Ignore:
Timestamp:
May 30, 2018, 12:41:51 AM (12 months ago)
Author:
xwa163
Message:
  1. Enable swizzled match copy in multiplexing lz4_grep for some special case
  2. Implement some lz4 AIO (all-in-one) pipeline and related kernel
File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.cpp

    r6047 r6059  
    5151#include <llvm/Support/raw_ostream.h>
    5252#include <llvm/Support/Debug.h>
    53 
     53#include <kernels/lz4/lz4_block_decoder.h>
     54#include <kernels/lz4/lz4_swizzled_aio.h>
    5455
    5556
     
    106107}
    107108
     109StreamSetBuffer * LZ4GrepGenerator::convertCompressedBitsStreamWithAioApproach(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix) {
     110    auto mGrepDriver = &mPxDriver;
     111    auto & iBuilder = mGrepDriver->getBuilder();
     112
     113    //// Decode Block Information
     114    StreamSetBuffer * const BlockData_IsCompressed = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(), 1);
     115    StreamSetBuffer * const BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(), 1);
     116    StreamSetBuffer * const BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(), 1);
     117
     118    //// Generate Helper Markers Extenders, FX, XF
     119    StreamSetBuffer * const Extenders = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks(), 1);
     120    mMatchOffsetMarker = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks());
     121    Kernel * extenderK = mPxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8);
     122    mPxDriver.makeKernelCall(extenderK, {mCompressedBasisBits}, {Extenders});
     123
     124
     125    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZ4BlockDecoderNewKernel>(iBuilder);
     126    blockDecoderK->setInitialArguments({iBuilder->CreateTrunc(mHasBlockChecksum, iBuilder->getInt1Ty()), mHeaderSize, mFileSize});
     127    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_IsCompressed, BlockData_BlockStart, BlockData_BlockEnd});
     128
     129
     130    // Produce unswizzled bit streams
     131    StreamSetBuffer * u16Swizzle0 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4), this->getInputBufferBlocks(), 1);
     132    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 4, 1, 1, 64, "source");
     133    mPxDriver.makeKernelCall(unSwizzleK, {compressedBitStream}, {u16Swizzle0});
     134
     135    StreamSetBuffer * decompressedSwizzled0 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4), this->getInputBufferBlocks(), 1);
     136
     137
     138    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4SwizzledAioKernel>(iBuilder, 4, 1, 4);
     139    lz4AioK->setInitialArguments({mFileSize});
     140    mPxDriver.makeKernelCall(
     141            lz4AioK,
     142            {
     143                    mCompressedByteStream,
     144                    Extenders,
     145
     146                    // Block Data
     147                    BlockData_IsCompressed,
     148                    BlockData_BlockStart,
     149                    BlockData_BlockEnd,
     150
     151                    u16Swizzle0,
     152            }, {
     153                    decompressedSwizzled0,
     154            });
     155
     156
     157
     158    StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getDecompressedBufferBlocks());
     159    // TODO deadloop when add unswizzled kernel
     160    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 4, 1, 1, 64, "dst");
     161    mPxDriver.makeKernelCall(unSwizzleK2, {decompressedSwizzled0}, {decompressionBitStream});
     162
     163    return decompressionBitStream;
     164
     165}
     166
     167StreamSetBuffer * LZ4GrepGenerator::convertCompressedBitsStreamWithSwizzledApproach(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix) {
     168    auto mGrepDriver = &mPxDriver;
     169    auto & idb = mGrepDriver->getBuilder();
     170
     171    StreamSetBuffer * u16Swizzle0 = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(4), this->getInputBufferBlocks(), 1);
     172    Kernel * delK = mPxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(idb, 4, 64);
     173    mPxDriver.makeKernelCall(delK, {mDeletionMarker, compressedBitStream}, {u16Swizzle0});
     174
     175    StreamSetBuffer * depositedSwizzle0 = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(4), this->getInputBufferBlocks(), 1);
     176    Kernel * multiplePdepK = mPxDriver.addKernelInstance<SwizzledMultiplePDEPkernel>(idb, 4, 1);
     177    mPxDriver.makeKernelCall(multiplePdepK, {mDepositMarker, u16Swizzle0}, {depositedSwizzle0});
     178
     179    StreamSetBuffer * matchCopiedSwizzle0 = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(4), this->getDecompressedBufferBlocks(), 1);
     180    Kernel * swizzledMatchCopyK = mPxDriver.addKernelInstance<LZ4SwizzledMatchCopyKernel>(idb, 4, 1, 4);
     181    mPxDriver.makeKernelCall(swizzledMatchCopyK, {mMatchOffsetMarker, mM0Marker, mCompressedByteStream, depositedSwizzle0}, {matchCopiedSwizzle0});
     182
     183    // Produce unswizzled bit streams
     184    StreamSetBuffer * matchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks());
     185    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(idb, 4, 1, 1);
     186    mPxDriver.makeKernelCall(unSwizzleK, {matchCopiedSwizzle0}, {matchCopiedBits});
     187
     188    return matchCopiedBits;
     189}
    108190parabix::StreamSetBuffer * LZ4GrepGenerator::convertCompressedBitsStream(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix) {
     191    if (numberOfStream == 4) {
     192        return this->convertCompressedBitsStreamWithSwizzledApproach(compressedBitStream, numberOfStream, prefix);
     193    }
     194
    109195    auto mGrepDriver = &mPxDriver;
    110196    auto & idb = mGrepDriver->getBuilder();
     
    147233}
    148234
    149 
    150 std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> LZ4GrepGenerator::multiplexingGrepPipeline(std::vector<re::RE *> &REs) {
    151 
     235void LZ4GrepGenerator::generateMultiplexingCompressedBitStream(std::vector<re::RE *> &REs) {
    152236    this->initREs(REs);
    153237    auto mGrepDriver = &mPxDriver;
     
    159243    int MaxCountFlag = 0;
    160244
     245
    161246    //  Regular Expression Processing and Analysis Phase
    162247    const auto nREs = mREs.size();
    163248
    164249    std::vector<StreamSetBuffer *> MatchResultsBufs(nREs);
    165 
    166 
    167 
    168     if (mGrepRecordBreak == GrepRecordBreakKind::LF) {
    169     } else if (mGrepRecordBreak == GrepRecordBreakKind::Null) {
    170         // TODO fix here
    171 //        kernel::Kernel * breakK = mGrepDriver->addKernelInstance<kernel::ParabixCharacterClassKernelBuilder>(idb, "Null", std::vector<re::CC *>{mBreakCC}, 8);
    172 //        mGrepDriver->makeKernelCall(breakK, {matchCopiedBasisBits}, {LineBreakStream});
    173     }
    174250
    175251    std::map<std::string, StreamSetBuffer *> propertyStream;
     
    188264
    189265    kernel::Kernel * ccK = mGrepDriver->addKernelInstance<kernel::CharClassesKernel>(idb, std::move(mpx_basis));
    190     mGrepDriver->makeKernelCall(ccK, {mCompressedBasisBits}, {CharClasses}); //TODO get it from compression space
     266    mGrepDriver->makeKernelCall(ccK, {mCompressedBasisBits}, {CharClasses});
    191267
    192268    StreamSetBuffer * CompressedLineFeedStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
     
    197273    kernel::Kernel* streamCombineKernel = mPxDriver.addKernelInstance<StreamsCombineKernel>(idb, std::vector<unsigned>({1, (unsigned)numOfCharacterClasses}));
    198274    mPxDriver.makeKernelCall(streamCombineKernel, {CompressedLineFeedStream, CharClasses}, {combinedStream});
    199 
    200     StreamSetBuffer * decompressedCombinedStream = this->convertCompressedBitsStream(combinedStream, 1 + numOfCharacterClasses, "combined");
     275    StreamSetBuffer * decompressedCombinedStream = nullptr;
     276
     277    decompressedCombinedStream = this->convertCompressedBitsStreamWithAioApproach(combinedStream, 1 + numOfCharacterClasses, "combined");
     278
     279/*
    201280    StreamSetBuffer * LineBreakStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1), baseBufferSize);
    202281    StreamSetBuffer * decompressedCharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize);
     
    204283    mPxDriver.makeKernelCall(streamSplitKernel, {decompressedCombinedStream}, {LineBreakStream, decompressedCharClasses});
    205284
    206     /*
    207     StreamSetBuffer * LineBreakStream = this->convertCompressedBitsStream(CompressedLineFeedStream, 1, "LineFeed");
    208     StreamSetBuffer * decompressedCharClasses = this->convertCompressedBitsStream(CharClasses, numOfCharacterClasses, "mpx");
    209      */
     285
    210286
    211287    StreamSetBuffer * fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks());
     
    240316
    241317    return std::pair<StreamSetBuffer *, StreamSetBuffer *>(LineBreakStream, Matches);
     318    */
     319
    242320};
    243 
    244 std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> LZ4GrepGenerator::grepPipeline(
    245         std::vector<re::RE *> &REs, parabix::StreamSetBuffer *decompressedBasisBits) {
     321std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> LZ4GrepGenerator::multiplexingGrepPipeline(std::vector<re::RE *> &REs, bool useAio) {
    246322
    247323    this->initREs(REs);
     
    251327    // TODO: until we automate stream buffer sizing, use this calculation to determine how large our matches buffer needs to be.
    252328    const unsigned baseBufferSize = this->getInputBufferBlocks();
     329    bool CC_Multiplexing = true;
    253330    int MaxCountFlag = 0;
    254331
     
    256333    const auto nREs = mREs.size();
    257334
    258     StreamSetBuffer * LineBreakStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    259335    std::vector<StreamSetBuffer *> MatchResultsBufs(nREs);
    260336
    261     StreamSetBuffer * RequiredStreams = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    262     StreamSetBuffer * UnicodeLB = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    263 
    264     StreamSetBuffer * LineFeedStream = this->linefeedStreamFromDecompressedBits(decompressedBasisBits);
    265 
    266     kernel::Kernel * requiredStreamsK = mGrepDriver->addKernelInstance<kernel::RequiredStreams_UTF8>(idb);
    267     mGrepDriver->makeKernelCall(requiredStreamsK, {decompressedBasisBits, LineFeedStream}, {RequiredStreams, UnicodeLB});
     337
    268338
    269339    if (mGrepRecordBreak == GrepRecordBreakKind::LF) {
    270         LineBreakStream = LineFeedStream;
    271340    } else if (mGrepRecordBreak == GrepRecordBreakKind::Null) {
    272         kernel::Kernel * breakK = mGrepDriver->addKernelInstance<kernel::ParabixCharacterClassKernelBuilder>(idb, "Null", std::vector<re::CC *>{mBreakCC}, 8);
    273         mGrepDriver->makeKernelCall(breakK, {decompressedBasisBits}, {LineBreakStream});
     341        // TODO fix here
     342//        kernel::Kernel * breakK = mGrepDriver->addKernelInstance<kernel::ParabixCharacterClassKernelBuilder>(idb, "Null", std::vector<re::CC *>{mBreakCC}, 8);
     343//        mGrepDriver->makeKernelCall(breakK, {matchCopiedBasisBits}, {LineBreakStream});
     344    }
     345
     346    std::map<std::string, StreamSetBuffer *> propertyStream;
     347
     348    std::vector<std::string> externalStreamNames;
     349    std::set<re::Name *> UnicodeProperties;
     350
     351    const auto UnicodeSets = re::collectCCs(mREs[0], &cc::Unicode, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));
     352    StreamSetBuffer * const MatchResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
     353
     354    mpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSets);
     355    mREs[0] = transformCCs(mpx.get(), mREs[0]);
     356    std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
     357    auto numOfCharacterClasses = mpx_basis.size();
     358    StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize);
     359
     360    kernel::Kernel * ccK = mGrepDriver->addKernelInstance<kernel::CharClassesKernel>(idb, std::move(mpx_basis));
     361    mGrepDriver->makeKernelCall(ccK, {mCompressedBasisBits}, {CharClasses});
     362
     363    StreamSetBuffer * CompressedLineFeedStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
     364    kernel::Kernel * linefeedK = mPxDriver.addKernelInstance<kernel::LineFeedKernelBuilder>(idb, Binding{idb->getStreamSetTy(8), "basis", FixedRate(), Principal()});
     365    mPxDriver.makeKernelCall(linefeedK, {mCompressedBasisBits}, {CompressedLineFeedStream});
     366
     367    StreamSetBuffer * combinedStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses + 1), baseBufferSize);
     368    kernel::Kernel* streamCombineKernel = mPxDriver.addKernelInstance<StreamsCombineKernel>(idb, std::vector<unsigned>({1, (unsigned)numOfCharacterClasses}));
     369    mPxDriver.makeKernelCall(streamCombineKernel, {CompressedLineFeedStream, CharClasses}, {combinedStream});
     370    StreamSetBuffer * decompressedCombinedStream = nullptr;
     371
     372    if (useAio) {
     373        decompressedCombinedStream = this->convertCompressedBitsStreamWithAioApproach(combinedStream, 1 + numOfCharacterClasses, "combined");
    274374    } else {
    275         LineBreakStream = UnicodeLB;
    276     }
    277 
    278     std::map<std::string, StreamSetBuffer *> propertyStream;
    279 
    280     for(unsigned i = 0; i < nREs; ++i) {
    281         std::vector<std::string> externalStreamNames;
    282         std::vector<StreamSetBuffer *> icgrepInputSets = {decompressedBasisBits};
    283         if (mGrepRecordBreak == GrepRecordBreakKind::Unicode) {
    284             externalStreamNames.push_back("UTF8_LB");
    285             icgrepInputSets.push_back(LineBreakStream);
    286             externalStreamNames.push_back("UTF8_nonfinal");
    287             icgrepInputSets.push_back(RequiredStreams);
    288         }
    289         std::set<re::Name *> UnicodeProperties;
    290 
    291         StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    292         kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[i], externalStreamNames);
    293         mGrepDriver->makeKernelCall(icgrepK, icgrepInputSets, {MatchResults});
    294         MatchResultsBufs[i] = MatchResults;
    295     }
     375        decompressedCombinedStream = this->convertCompressedBitsStream(combinedStream, 1 + numOfCharacterClasses, "combined");
     376    }
     377
     378    StreamSetBuffer * LineBreakStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1), baseBufferSize);
     379    StreamSetBuffer * decompressedCharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize);
     380    kernel::Kernel* streamSplitKernel = mPxDriver.addKernelInstance<StreamsSplitKernel>(idb, std::vector<unsigned>({1, (unsigned)numOfCharacterClasses}));
     381    mPxDriver.makeKernelCall(streamSplitKernel, {decompressedCombinedStream}, {LineBreakStream, decompressedCharClasses});
     382
     383    /*
     384    StreamSetBuffer * LineBreakStream = this->convertCompressedBitsStream(CompressedLineFeedStream, 1, "LineFeed");
     385    StreamSetBuffer * decompressedCharClasses = this->convertCompressedBitsStream(CharClasses, numOfCharacterClasses, "mpx");
     386     */
     387
     388    StreamSetBuffer * fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks());
     389    Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<LZ4FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, 8);
     390    mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits});
     391
     392    kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[0], externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()});
     393    mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {MatchResults});
     394    MatchResultsBufs[0] = MatchResults;
    296395
    297396    StreamSetBuffer * MergedResults = MatchResultsBufs[0];
     
    318417
    319418    return std::pair<StreamSetBuffer *, StreamSetBuffer *>(LineBreakStream, Matches);
     419};
     420
     421std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> LZ4GrepGenerator::grepPipeline(
     422        std::vector<re::RE *> &REs, parabix::StreamSetBuffer *decompressedBasisBits) {
     423
     424    this->initREs(REs);
     425    auto mGrepDriver = &mPxDriver;
     426
     427    auto & idb = mGrepDriver->getBuilder();
     428    // TODO: until we automate stream buffer sizing, use this calculation to determine how large our matches buffer needs to be.
     429    const unsigned baseBufferSize = this->getInputBufferBlocks();
     430    int MaxCountFlag = 0;
     431
     432    //  Regular Expression Processing and Analysis Phase
     433    const auto nREs = mREs.size();
     434
     435    std::vector<StreamSetBuffer *> MatchResultsBufs(nREs);
     436
     437    StreamSetBuffer * LineBreakStream = LineBreakStream = this->linefeedStreamFromDecompressedBits(decompressedBasisBits);
     438
     439
     440    std::map<std::string, StreamSetBuffer *> propertyStream;
     441
     442    for(unsigned i = 0; i < nREs; ++i) {
     443        std::vector<std::string> externalStreamNames;
     444        std::vector<StreamSetBuffer *> icgrepInputSets = {decompressedBasisBits};
     445
     446        std::set<re::Name *> UnicodeProperties;
     447
     448        StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
     449        kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[i], externalStreamNames);
     450        mGrepDriver->makeKernelCall(icgrepK, icgrepInputSets, {MatchResults});
     451        MatchResultsBufs[i] = MatchResults;
     452    }
     453
     454    StreamSetBuffer * MergedResults = MatchResultsBufs[0];
     455    if (mREs.size() > 1) {
     456        MergedResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
     457        kernel::Kernel * streamsMergeK = mGrepDriver->addKernelInstance<kernel::StreamsMerge>(idb, 1, mREs.size());
     458        mGrepDriver->makeKernelCall(streamsMergeK, MatchResultsBufs, {MergedResults});
     459    }
     460    StreamSetBuffer * Matches = MergedResults;
     461    if (mMoveMatchesToEOL) {
     462        StreamSetBuffer * OriginalMatches = Matches;
     463        kernel::Kernel * matchedLinesK = mGrepDriver->addKernelInstance<kernel::MatchedLinesKernel>(idb);
     464        Matches = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
     465        mGrepDriver->makeKernelCall(matchedLinesK, {OriginalMatches, LineBreakStream}, {Matches});
     466    }
     467
     468    if (MaxCountFlag > 0) {
     469        kernel::Kernel * untilK = mGrepDriver->addKernelInstance<kernel::UntilNkernel>(idb);
     470        untilK->setInitialArguments({idb->getSize(MaxCountFlag)});
     471        StreamSetBuffer * const AllMatches = Matches;
     472        Matches = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
     473        mGrepDriver->makeKernelCall(untilK, {AllMatches}, {Matches});
     474    }
     475
     476    return std::pair<StreamSetBuffer *, StreamSetBuffer *>(LineBreakStream, Matches);
    320477
    321478}
     
    380537    mPxDriver.finalizeObject();
    381538}
     539
     540void LZ4GrepGenerator::generateMultiplexingSwizzledAioPipeline(re::RE* regex) {
     541    auto & iBuilder = mPxDriver.getBuilder();
     542    this->generateMainFunc(iBuilder);
     543
     544    // GeneratePipeline
     545    this->generateLoadByteStreamAndBitStream(iBuilder);
     546
     547    std::vector<re::RE*> res = {regex};
     548    this->generateMultiplexingCompressedBitStream(res);
     549
     550    mPxDriver.generatePipelineIR();
     551    mPxDriver.deallocateBuffers();
     552
     553    iBuilder->CreateRetVoid();
     554
     555    mPxDriver.finalizeObject();
     556}
     557
     558void LZ4GrepGenerator::generateMultiplexingSwizzledAioPipeline2(re::RE* regex) {
     559    auto & iBuilder = mPxDriver.getBuilder();
     560    this->generateMainFunc(iBuilder);
     561
     562    // GeneratePipeline
     563    this->generateLoadByteStreamAndBitStream(iBuilder);
     564//    this->generateExtractAndDepositMarkers(iBuilder);
     565
     566    StreamSetBuffer * LineBreakStream;
     567    StreamSetBuffer * Matches;
     568    std::vector<re::RE*> res = {regex};
     569    std::tie(LineBreakStream, Matches) = multiplexingGrepPipeline(res, true);
     570
     571    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
     572    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
     573    mPxDriver.generatePipelineIR();
     574
     575    iBuilder->setKernel(matchCountK);
     576    Value * matchedLineCount = iBuilder->getAccumulator("countResult");
     577    matchedLineCount = iBuilder->CreateZExt(matchedLineCount, iBuilder->getInt64Ty());
     578    iBuilder->CallPrintInt("aaa", matchedLineCount);
     579
     580    mPxDriver.deallocateBuffers();
     581
     582    // TODO return matchedLineCount
     583//        idb->CreateRet(matchedLineCount);
     584
     585    iBuilder->CreateRetVoid();
     586
     587    mPxDriver.finalizeObject();
     588}
     589
     590void LZ4GrepGenerator::generateSwizzledAioPipeline(re::RE* regex) {
     591    auto & iBuilder = mPxDriver.getBuilder();
     592    this->generateMainFunc(iBuilder);
     593
     594    // GeneratePipeline
     595    this->generateLoadByteStreamAndBitStream(iBuilder);
     596
     597    StreamSetBuffer * const decompressionBitStream = this->generateSwizzledAIODecompression(iBuilder);
     598
     599    StreamSetBuffer * LineBreakStream;
     600    StreamSetBuffer * Matches;
     601    std::vector<re::RE*> res = {regex};
     602    std::tie(LineBreakStream, Matches) = grepPipeline(res, decompressionBitStream);
     603/*
     604    StreamSetBuffer * const decompressionByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     605    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     606    mPxDriver.makeKernelCall(p2sK, {decompressionBitStream}, {decompressionByteStream});
     607
     608    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     609    outK->setInitialArguments({iBuilder->GetString("/Users/wxy325/developer/LZ4-sample-files/workspace/lz4d-normal/8k_.txt")});
     610    mPxDriver.makeKernelCall(outK, {decompressionByteStream}, {});
     611*/
     612    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
     613
     614    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
     615    mPxDriver.generatePipelineIR();
     616
     617    iBuilder->setKernel(matchCountK);
     618    Value * matchedLineCount = iBuilder->getAccumulator("countResult");
     619    matchedLineCount = iBuilder->CreateZExt(matchedLineCount, iBuilder->getInt64Ty());
     620    iBuilder->CallPrintInt("aaa", matchedLineCount);
     621    mPxDriver.deallocateBuffers();
     622
     623    // TODO return matchedLineCount
     624//        idb->CreateRet(matchedLineCount);
     625
     626    iBuilder->CreateRetVoid();
     627
     628    mPxDriver.finalizeObject();
     629}
     630
     631void LZ4GrepGenerator::generateAioPipeline(re::RE *regex) {
     632    auto & iBuilder = mPxDriver.getBuilder();
     633    this->generateMainFunc(iBuilder);
     634
     635    // GeneratePipeline
     636    this->generateLoadByteStreamAndBitStream(iBuilder);
     637    parabix::StreamSetBuffer * decompressedByteStream = this->generateAIODecompression(iBuilder);
     638
     639
     640    StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getDecompressedBufferBlocks());
     641    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, /*aligned = */ true, "a");
     642    mPxDriver.makeKernelCall(s2pk, {decompressedByteStream}, {decompressionBitStream});
     643
     644
     645    StreamSetBuffer * LineBreakStream;
     646    StreamSetBuffer * Matches;
     647    std::vector<re::RE*> res = {regex};
     648    std::tie(LineBreakStream, Matches) = grepPipeline(res, decompressionBitStream);
     649
     650
     651//    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     652//    outK->setInitialArguments({iBuilder->GetString("/Users/wxy325/developer/LZ4-sample-files/workspace/lz4d-normal/8k_.txt")});
     653//    mPxDriver.makeKernelCall(outK, {decompressedStream}, {});
     654
     655    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
     656    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
     657    mPxDriver.generatePipelineIR();
     658
     659    iBuilder->setKernel(matchCountK);
     660    Value * matchedLineCount = iBuilder->getAccumulator("countResult");
     661    matchedLineCount = iBuilder->CreateZExt(matchedLineCount, iBuilder->getInt64Ty());
     662    iBuilder->CallPrintInt("aaa", matchedLineCount);
     663
     664    mPxDriver.deallocateBuffers();
     665
     666    // TODO return matchedLineCount
     667//        idb->CreateRet(matchedLineCount);
     668
     669    iBuilder->CreateRetVoid();
     670
     671    mPxDriver.finalizeObject();
     672
     673}
     674
    382675
    383676void LZ4GrepGenerator::generateCountOnlyGrepPipeline(re::RE *regex, bool enableGather) {
Note: See TracChangeset for help on using the changeset viewer.