Ignore:
Timestamp:
Jul 25, 2018, 2:50:09 AM (7 months ago)
Author:
xwa163
Message:
  1. Add sourceCC in multiplexed CC
  2. Remove workaround FakeBasisBits? from ICGrep
  3. Implement Swizzled version of LZParabix
  4. Init checkin for SwizzleByGather? Kernel
File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGrepGenerator.cpp

    r6130 r6133  
    119119}
    120120
    121 void LZParabixGrepGenerator::generateCountOnlyAioPipeline(re::RE *regex) {
     121void LZParabixGrepGenerator::generateCountOnlyAioPipeline(re::RE *regex, bool swizzledDecompression ) {
    122122    auto & iBuilder = mPxDriver.getBuilder();
    123123    this->generateCountOnlyMainFunc(iBuilder);
     
    132132        std::tie(LineBreakStream, Matches) = multiplexingGrepPipeline(res);
    133133    } else {
    134         std::tie(LineBreakStream, Matches) = grepPipeline(res);
     134        std::tie(LineBreakStream, Matches) = grepPipeline(res, swizzledDecompression);
    135135    }
    136136
     
    201201    mGrepDriver->makeKernelCall(ccK, {LiteralBitStream}, {CharClasses});
    202202
    203 
    204203    StreamSetBuffer * newLineBreak = nullptr;
     204
     205
     206    StreamSetBuffer * decompressedCharClasses = nullptr;
     207    StreamSetBuffer * u8NoFinalStream = nullptr;
     208    StreamSetBuffer * fakeMatchCopiedBits = nullptr;
     209
    205210    if (allCcByteLength) {
    206211        auto ret = this->generateBitStreamDecompression(idb, {CharClasses});
    207         StreamSetBuffer * decompressedCharClasses = ret[0];
    208         StreamSetBuffer * fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
    209         Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, 8);
    210         mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits});
    211 
    212         newLineBreak = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getInputBufferBlocks(idb));
    213         kernel::Kernel * lineFeedGrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), linefeedCC), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian, true);
    214         mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {newLineBreak});
    215 
    216         kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[0], externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian, true);
    217         mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {MatchResults});
    218         MatchResultsBufs[0] = MatchResults;
     212        decompressedCharClasses = ret[0];
     213        fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
     214        u8NoFinalStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1), this->getInputBufferBlocks(idb), 1);
     215
     216        Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, std::vector<unsigned>({8, 1}));
     217        mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits, u8NoFinalStream});
    219218    } else {
    220219        re::RE* nonFinalName = re::makeAlt({re::makeByte(0xC2, 0xF4),
     
    227226        auto ret = this->generateBitStreamDecompression(idb, {CharClasses, compressedU8NoFinalStream});
    228227
    229         StreamSetBuffer * decompressedCharClasses = ret[0];
    230         StreamSetBuffer * u8NoFinalStream = ret[1];
    231 
    232         StreamSetBuffer * fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
     228        decompressedCharClasses = ret[0];
     229        u8NoFinalStream = ret[1];
     230
     231        fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
    233232        Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, 8);
    234233        mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits});
    235234
    236         newLineBreak = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getInputBufferBlocks(idb));
    237         kernel::Kernel * lineFeedGrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), linefeedCC), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian, true);
    238         mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {newLineBreak});
    239 
    240         externalStreamNames.push_back("UTF8_nonfinal");
    241 
    242         kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[0], externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
    243         mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, u8NoFinalStream, decompressedCharClasses}, {MatchResults});
    244         MatchResultsBufs[0] = MatchResults;
    245     }
     235    }
     236
     237
     238    newLineBreak = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getInputBufferBlocks(idb));
     239    kernel::Kernel * lineFeedGrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), linefeedCC), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
     240    mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {newLineBreak});
     241
     242    externalStreamNames.push_back("UTF8_nonfinal");
     243
     244    kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[0], externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
     245    mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, u8NoFinalStream, decompressedCharClasses}, {MatchResults});
     246    MatchResultsBufs[0] = MatchResults;
     247
    246248
    247249    StreamSetBuffer * MergedResults = MatchResultsBufs[0];
     
    272274
    273275std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *>
    274 LZParabixGrepGenerator::grepPipeline(std::vector<re::RE *> &REs) {
     276LZParabixGrepGenerator::grepPipeline(std::vector<re::RE *> &REs, bool swizzledDecompression) {
    275277
    276278    this->initREs(REs);
     
    293295
    294296
    295     auto ret = this->generateBitStreamDecompression(idb, {LiteralBitStream/*, compressedLineBreakStream*/});
    296     StreamSetBuffer * decompressedBasisBits = ret[0];
    297 //    StreamSetBuffer * LineBreakStream = ret[1];
    298 
    299 //    StreamSetBuffer * decompressedBasisBits = this->generateAioBitStreamDecompressoin(idb, {mCompressedBasisBits})[0];
     297    StreamSetBuffer * decompressedBasisBits = nullptr;
     298    if (swizzledDecompression) {
     299        decompressedBasisBits = this->generateSwizzledBitStreamDecompression(idb, LiteralBitStream);
     300    } else {
     301        auto ret = this->generateBitStreamDecompression(idb, {LiteralBitStream});
     302        decompressedBasisBits = ret[0];
     303    }
     304
    300305    StreamSetBuffer * LineBreakStream = this->linefeedStreamFromDecompressedBits(decompressedBasisBits);
    301306
Note: See TracChangeset for help on using the changeset viewer.