Ignore:
Timestamp:
Jul 25, 2018, 2:50:09 AM (11 months ago)
Author:
xwa163
Message:
  1. Add sourceCC in multiplexed CC
  2. Remove workaround FakeBasisBits? from ICGrep
  3. Implement Swizzled version of LZParabix
  4. Init checkin for SwizzleByGather? Kernel
Location:
icGREP/icgrep-devel/icgrep/lzparabix
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.cpp

    r6123 r6133  
    1 //
    2 // Created by wxy325 on 2018/6/18.
    3 //
    41
    52#include "LZParabixGenerator.h"
     
    2219#include <kernels/swizzled_multiple_pdep_kernel.h>
    2320#include <kernels/lzparabix/decoder/LZParabixBlockDecoder.h>
    24 #include <kernels/lzparabix/decoder/LZParabixAioKernel.h>
     21#include <kernels/lzparabix/decoder/LZParabixBitStreamAioKernel.h>
     22#include <kernels/lzparabix/decoder/LZParabixSwizzledAioKernel.h>
    2523#include <kernels/lzparabix/decoder/LZParabixLiteralDecoderKernel.h>
    2624
     
    8381    mPxDriver.makeKernelCall(literalDecoderK, {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd}, {LiteralBitStream});
    8482    return LiteralBitStream;
     83}
     84
     85
     86
     87parabix::StreamSetBuffer* LZParabixGenerator::generateSwizzledBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, parabix::StreamSetBuffer* inputBitStreams) {
     88
     89    Kernel * swizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 2, 1);
     90
     91    StreamSetBuffer* inputSwizzled1 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
     92    StreamSetBuffer* inputSwizzled2 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
     93    mPxDriver.makeKernelCall(swizzleK, {inputBitStreams}, {inputSwizzled1, inputSwizzled2});
     94
     95
     96    StreamSetBuffer* outputSwizzled1 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
     97    StreamSetBuffer* outputSwizzled2 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
     98    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixSwizzledAioKernel>(iBuilder);
     99    aioK->setInitialArguments({mFileSize});
     100    mPxDriver.makeKernelCall(aioK, {
     101            mCompressedByteStream,
     102            BlockData_BlockStart,
     103            BlockData_BlockEnd,
     104
     105//            inputBitStreams,
     106            inputSwizzled1,
     107            inputSwizzled2
     108    }, {
     109            outputSwizzled1,
     110            outputSwizzled2
     111    });
     112
     113    StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getDecompressedBufferBlocks(iBuilder));
     114
     115    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
     116//    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleByGather>(iBuilder);
     117    mPxDriver.makeKernelCall(unSwizzleK2, {outputSwizzled1, outputSwizzled2}, {decompressionBitStream});
     118    return decompressionBitStream;
    85119}
    86120
     
    101135    }
    102136
    103     Kernel * aioK = mPxDriver.addKernelInstance<LZParabixAioKernel>(iBuilder, numbersOfStreams);
     137    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixBitStreamAioKernel>(iBuilder, numbersOfStreams);
    104138    aioK->setInitialArguments({mFileSize});
    105139    mPxDriver.makeKernelCall(aioK, inputStreams, outputStreams);
     
    133167    }
    134168
    135     Kernel * aioK = mPxDriver.addKernelInstance<LZParabixAioKernel>(iBuilder, numsOfStreams);
     169    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixBitStreamAioKernel>(iBuilder, numsOfStreams);
    136170    aioK->setInitialArguments({mFileSize});
    137171
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.h

    r6123 r6133  
    6767    parabix::StreamSetBuffer* extractLiteralBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    6868
     69
     70    parabix::StreamSetBuffer* generateSwizzledBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, parabix::StreamSetBuffer* inputBitStreams);
    6971    std::vector<parabix::StreamSetBuffer*> generateBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, std::vector<parabix::StreamSetBuffer*> inputBitStreams);
    7072};
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGrepGenerator.cpp

    r6130 r6133  
    119119}
    120120
    121 void LZParabixGrepGenerator::generateCountOnlyAioPipeline(re::RE *regex) {
     121void LZParabixGrepGenerator::generateCountOnlyAioPipeline(re::RE *regex, bool swizzledDecompression ) {
    122122    auto & iBuilder = mPxDriver.getBuilder();
    123123    this->generateCountOnlyMainFunc(iBuilder);
     
    132132        std::tie(LineBreakStream, Matches) = multiplexingGrepPipeline(res);
    133133    } else {
    134         std::tie(LineBreakStream, Matches) = grepPipeline(res);
     134        std::tie(LineBreakStream, Matches) = grepPipeline(res, swizzledDecompression);
    135135    }
    136136
     
    201201    mGrepDriver->makeKernelCall(ccK, {LiteralBitStream}, {CharClasses});
    202202
    203 
    204203    StreamSetBuffer * newLineBreak = nullptr;
     204
     205
     206    StreamSetBuffer * decompressedCharClasses = nullptr;
     207    StreamSetBuffer * u8NoFinalStream = nullptr;
     208    StreamSetBuffer * fakeMatchCopiedBits = nullptr;
     209
    205210    if (allCcByteLength) {
    206211        auto ret = this->generateBitStreamDecompression(idb, {CharClasses});
    207         StreamSetBuffer * decompressedCharClasses = ret[0];
    208         StreamSetBuffer * fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
    209         Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, 8);
    210         mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits});
    211 
    212         newLineBreak = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getInputBufferBlocks(idb));
    213         kernel::Kernel * lineFeedGrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), linefeedCC), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian, true);
    214         mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {newLineBreak});
    215 
    216         kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[0], externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian, true);
    217         mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {MatchResults});
    218         MatchResultsBufs[0] = MatchResults;
     212        decompressedCharClasses = ret[0];
     213        fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
     214        u8NoFinalStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1), this->getInputBufferBlocks(idb), 1);
     215
     216        Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, std::vector<unsigned>({8, 1}));
     217        mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits, u8NoFinalStream});
    219218    } else {
    220219        re::RE* nonFinalName = re::makeAlt({re::makeByte(0xC2, 0xF4),
     
    227226        auto ret = this->generateBitStreamDecompression(idb, {CharClasses, compressedU8NoFinalStream});
    228227
    229         StreamSetBuffer * decompressedCharClasses = ret[0];
    230         StreamSetBuffer * u8NoFinalStream = ret[1];
    231 
    232         StreamSetBuffer * fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
     228        decompressedCharClasses = ret[0];
     229        u8NoFinalStream = ret[1];
     230
     231        fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
    233232        Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, 8);
    234233        mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits});
    235234
    236         newLineBreak = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getInputBufferBlocks(idb));
    237         kernel::Kernel * lineFeedGrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), linefeedCC), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian, true);
    238         mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {newLineBreak});
    239 
    240         externalStreamNames.push_back("UTF8_nonfinal");
    241 
    242         kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[0], externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
    243         mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, u8NoFinalStream, decompressedCharClasses}, {MatchResults});
    244         MatchResultsBufs[0] = MatchResults;
    245     }
     235    }
     236
     237
     238    newLineBreak = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getInputBufferBlocks(idb));
     239    kernel::Kernel * lineFeedGrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), linefeedCC), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
     240    mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {newLineBreak});
     241
     242    externalStreamNames.push_back("UTF8_nonfinal");
     243
     244    kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[0], externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
     245    mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, u8NoFinalStream, decompressedCharClasses}, {MatchResults});
     246    MatchResultsBufs[0] = MatchResults;
     247
    246248
    247249    StreamSetBuffer * MergedResults = MatchResultsBufs[0];
     
    272274
    273275std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *>
    274 LZParabixGrepGenerator::grepPipeline(std::vector<re::RE *> &REs) {
     276LZParabixGrepGenerator::grepPipeline(std::vector<re::RE *> &REs, bool swizzledDecompression) {
    275277
    276278    this->initREs(REs);
     
    293295
    294296
    295     auto ret = this->generateBitStreamDecompression(idb, {LiteralBitStream/*, compressedLineBreakStream*/});
    296     StreamSetBuffer * decompressedBasisBits = ret[0];
    297 //    StreamSetBuffer * LineBreakStream = ret[1];
    298 
    299 //    StreamSetBuffer * decompressedBasisBits = this->generateAioBitStreamDecompressoin(idb, {mCompressedBasisBits})[0];
     297    StreamSetBuffer * decompressedBasisBits = nullptr;
     298    if (swizzledDecompression) {
     299        decompressedBasisBits = this->generateSwizzledBitStreamDecompression(idb, LiteralBitStream);
     300    } else {
     301        auto ret = this->generateBitStreamDecompression(idb, {LiteralBitStream});
     302        decompressedBasisBits = ret[0];
     303    }
     304
    300305    StreamSetBuffer * LineBreakStream = this->linefeedStreamFromDecompressedBits(decompressedBasisBits);
    301306
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGrepGenerator.h

    r6114 r6133  
    2323public:
    2424    LZParabixGrepGenerator(bool enableMultiplexing = false);
    25     void generateCountOnlyAioPipeline(re::RE* regex);
     25    void generateCountOnlyAioPipeline(re::RE* regex, bool swizzledDecompression = false);
    2626    CountOnlyGrepMainFunctionType getCountOnlyGrepMainFunction();
    2727protected:
     
    3636    void initREs(std::vector<re::RE *> & REs);
    3737
    38     std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(std::vector<re::RE *> &REs);
     38    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(std::vector<re::RE *> &REs, bool swizzledDecompression);
    3939    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> multiplexingGrepPipeline(std::vector<re::RE *> &REs);
    4040
Note: See TracChangeset for help on using the changeset viewer.