Ignore:
Timestamp:
Aug 1, 2018, 12:22:09 AM (9 months ago)
Author:
xwa163
Message:
  1. LZ4 ScanMatch? pipeline
  2. Refactor LZ4 Generator
  3. Adjust some naming
File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGrepGenerator.cpp

    r6135 r6137  
    135135    }
    136136
    137 //    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
    138 //    outK->setInitialArguments({iBuilder->GetString("/Users/wxy325/developer/LZ4-sample-files/workspace/lz4d-normal/8k_.txt")});
    139 //    mPxDriver.makeKernelCall(outK, {decompressedStream}, {});
    140137
    141138    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
     
    202199
    203200
    204     StreamSetBuffer * decompressedCharClasses = nullptr;
     201    StreamSetBuffer * uncompressedCharClasses = nullptr;
    205202    StreamSetBuffer * u8NoFinalStream = nullptr;
    206203    StreamSetBuffer * fakeMatchCopiedBits = nullptr;
     
    210207    if (allCcByteLength) {
    211208        auto ret = this->generateBitStreamDecompression(idb, {CharClasses});
    212         decompressedCharClasses = ret[0];
     209        uncompressedCharClasses = ret[0];
    213210        fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
    214211        u8NoFinalStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1), this->getInputBufferBlocks(idb), 1);
    215212
    216213        Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, std::vector<unsigned>({8, 1}));
    217         mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits, u8NoFinalStream});
     214        mPxDriver.makeKernelCall(fakeStreamGeneratorK, {uncompressedCharClasses}, {fakeMatchCopiedBits, u8NoFinalStream});
    218215    } else {
    219216        re::RE* nonFinalName = re::makeAlt({re::makeByte(0xC2, 0xF4),
     
    226223        auto ret = this->generateBitStreamDecompression(idb, {CharClasses, compressedU8NoFinalStream});
    227224
    228         decompressedCharClasses = ret[0];
     225        uncompressedCharClasses = ret[0];
    229226        u8NoFinalStream = ret[1];
    230227
    231228        fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
    232229        Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, 8);
    233         mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits});
     230        mPxDriver.makeKernelCall(fakeStreamGeneratorK, {uncompressedCharClasses}, {fakeMatchCopiedBits});
    234231
    235232    }
     
    238235    newLineBreak = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getInputBufferBlocks(idb));
    239236    kernel::Kernel * lineFeedGrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), linefeedCC), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
    240     mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {newLineBreak});
     237    mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, uncompressedCharClasses}, {newLineBreak});
    241238
    242239    externalStreamNames.push_back("UTF8_nonfinal");
    243240
    244241    kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[0], externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
    245     mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, u8NoFinalStream, decompressedCharClasses}, {MatchResults});
     242    mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, u8NoFinalStream, uncompressedCharClasses}, {MatchResults});
    246243    MatchResultsBufs[0] = MatchResults;
    247244
     
    292289    this->generateBlockData(idb);
    293290    StreamSetBuffer * const LiteralBitStream = this->extractLiteralBitStream(idb);
    294 //    auto compressedLineBreakStream = this->linefeedStreamFromDecompressedBits(LiteralBitStream);
    295 
    296 
    297     StreamSetBuffer * decompressedBasisBits = nullptr;
     291//    auto compressedLineBreakStream = this->linefeedStreamFromUncompressedBits(LiteralBitStream);
     292
     293
     294    StreamSetBuffer * uncompressedBasisBits = nullptr;
    298295    if (swizzledDecompression) {
    299         decompressedBasisBits = this->generateSwizzledBitStreamDecompression(idb, LiteralBitStream);
     296        uncompressedBasisBits = this->generateSwizzledBitStreamDecompression(idb, LiteralBitStream);
    300297    } else {
    301298        auto ret = this->generateBitStreamDecompression(idb, {LiteralBitStream});
    302         decompressedBasisBits = ret[0];
    303     }
    304 
    305     StreamSetBuffer * LineBreakStream = this->linefeedStreamFromDecompressedBits(decompressedBasisBits);
     299        uncompressedBasisBits = ret[0];
     300    }
     301
     302    StreamSetBuffer * LineBreakStream = this->linefeedStreamFromUncompressedBits(uncompressedBasisBits);
    306303
    307304    std::map<std::string, StreamSetBuffer *> propertyStream;
     
    309306    for(unsigned i = 0; i < nREs; ++i) {
    310307        std::vector<std::string> externalStreamNames;
    311         std::vector<StreamSetBuffer *> icgrepInputSets = {decompressedBasisBits};
     308        std::vector<StreamSetBuffer *> icgrepInputSets = {uncompressedBasisBits};
    312309
    313310        std::set<re::Name *> UnicodeProperties;
     
    345342
    346343parabix::StreamSetBuffer *
    347 LZParabixGrepGenerator::linefeedStreamFromDecompressedBits(parabix::StreamSetBuffer *decompressedBasisBits) {
     344LZParabixGrepGenerator::linefeedStreamFromUncompressedBits(parabix::StreamSetBuffer *uncompressedBasisBits) {
    348345    auto & idb = mPxDriver.getBuilder();
    349346    const unsigned baseBufferSize = this->getInputBufferBlocks(idb);
    350347    StreamSetBuffer * LineFeedStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    351348    kernel::Kernel * linefeedK = mPxDriver.addKernelInstance<kernel::LineFeedKernelBuilder>(idb, Binding{idb->getStreamSetTy(8), "basis", FixedRate(), Principal()}, cc::BitNumbering::BigEndian);
    352     mPxDriver.makeKernelCall(linefeedK, {decompressedBasisBits}, {LineFeedStream});
     349    mPxDriver.makeKernelCall(linefeedK, {uncompressedBasisBits}, {LineFeedStream});
    353350    return LineFeedStream;
    354351}
Note: See TracChangeset for help on using the changeset viewer.