Changeset 6144


Ignore:
Timestamp:
Aug 11, 2018, 1:17:11 AM (7 days ago)
Author:
xwa163
Message:

lz4_grep: Init checkin for utf8 character class multiplexing

Location:
icGREP/icgrep-devel/icgrep
Files:
10 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/kernels/lz4/twist_kernel.cpp

    r6140 r6144  
    4242    }
    4343
    44 
    45 
    46 
    4744    TwistByPDEPKernel::TwistByPDEPKernel(const std::unique_ptr <kernel::KernelBuilder> &b, unsigned numberOfInputStream, unsigned twistWidth)
    4845            : BlockOrientedKernel("TwistByPDEPKernel",
     
    7572
    7673    TwistMultipleByPDEPKernel::TwistMultipleByPDEPKernel(const std::unique_ptr<kernel::KernelBuilder> &b,
    77                                                          unsigned numberOfInputStreamSet, unsigned twistWidth)
     74                     std::vector<unsigned> numberOfInputStreams, unsigned twistWidth)
    7875            : BlockOrientedKernel("TwistMultipleByPDEPKernel",
    7976                                  {},
    8077                                  {Binding{b->getStreamSetTy(1, twistWidth), "byteStream"}},
    8178                                  {}, {}, {}),
    82               mNumberOfInputStreamSet(numberOfInputStreamSet),
     79              mNumberOfInputStreams(numberOfInputStreams),
    8380              mTwistWidth(twistWidth)
    8481    {
    85         for (unsigned i = 0; i < numberOfInputStreamSet; i++) {
    86             mStreamSetInputs.push_back(Binding{b->getStreamSetTy(1), "basisBits_" + std::to_string(i) });
     82        assert(twistWidth == 2 || twistWidth == 4);
     83        size_t totalNumOfStreams = 0;
     84        for (unsigned i = 0; i < numberOfInputStreams.size(); i++) {
     85            totalNumOfStreams += numberOfInputStreams[i];
     86            mStreamSetInputs.push_back(Binding{b->getStreamSetTy(numberOfInputStreams[i]), "basisBits_" + std::to_string(i) });
    8787        }
     88        assert(totalNumOfStreams < twistWidth);
    8889    }
    8990
     
    9192        Value *inputBlocks[mTwistWidth];
    9293
    93         for (unsigned i = 0; i < mTwistWidth; i++) {
    94             if (i < mNumberOfInputStreamSet) {
    95                 inputBlocks[i] = b->loadInputStreamBlock("basisBits_" + std::to_string(i), b->getInt32(0));
    96                 if (i == 0) {
    97                     b->CallPrintRegister("basisBits_" + std::to_string(i), inputBlocks[i]);
    98                 }
    99             } else {
    100                 inputBlocks[i] = ConstantVector::getNullValue(b->getBitBlockType());
     94        unsigned iStreamIndex = 0;
     95        for (unsigned i = 0; i < mNumberOfInputStreams.size(); i++) {
     96            for (unsigned j = 0; j < mNumberOfInputStreams[i]; j++) {
     97                inputBlocks[iStreamIndex] = b->loadInputStreamBlock("basisBits_" + std::to_string(i), b->getInt32(j));
     98                ++iStreamIndex;
    10199            }
    102100        }
     101
     102        for (; iStreamIndex < mTwistWidth; iStreamIndex++) {
     103            inputBlocks[iStreamIndex] = ConstantVector::getNullValue(b->getBitBlockType());
     104        }
     105
    103106
    104107        Value *outputBasePtr = b->CreatePointerCast(b->getOutputStreamBlockPtr("byteStream", b->getSize(0)),
  • icGREP/icgrep-devel/icgrep/kernels/lz4/twist_kernel.h

    r6140 r6144  
    2525    class TwistMultipleByPDEPKernel final : public BlockOrientedKernel {
    2626    public:
    27         TwistMultipleByPDEPKernel(const std::unique_ptr <kernel::KernelBuilder> &b, unsigned numberOfInputStreamSet, unsigned twistWidth);
     27        TwistMultipleByPDEPKernel(const std::unique_ptr <kernel::KernelBuilder> &b, std::vector<unsigned> numberOfInputStreams, unsigned twistWidth);
    2828
    2929    private:
    30         const unsigned mNumberOfInputStreamSet;
     30        std::vector<unsigned> mNumberOfInputStreams;
    3131        const unsigned mTwistWidth;
    3232
  • icGREP/icgrep-devel/icgrep/kernels/lz4/untwist_kernel.cpp

    r6143 r6144  
    66
    77namespace kernel {
    8     UntwistByPEXTKernel::UntwistByPEXTKernel(const std::unique_ptr<kernel::KernelBuilder> &b, unsigned numberOfOutputStream, unsigned twistWidth)
    9             :BlockOrientedKernel("UntwistByPEXTKernel",
    10                                  {
    11                                          Binding{b->getStreamSetTy(1, twistWidth), "byteStream", FixedRate(), Principal()}
    12                                  },
    13                                  {
    14                                          Binding{b->getStreamSetTy(numberOfOutputStream, 1), "basisBits"}
    15                                  }, {}, {}, {}),
    16              mNumberOfOutputStream(numberOfOutputStream),
    17              mTwistWidth(twistWidth)
    18     {
    19         assert(twistWidth == 2 || twistWidth == 4);
    20         assert(numberOfInputStream <= twistWidth);
    218
    22     }
    23 
    24     void UntwistByPEXTKernel::generateDoBlockMethod(const std::unique_ptr<KernelBuilder> &b) {
     9    void untwistByPEXT(const std::unique_ptr <kernel::KernelBuilder> &b, Value* inputBasePtr, unsigned mTwistWidth, Value *outputBlocks[]){
    2510        Function* PEXT_func = Intrinsic::getDeclaration(b->getModule(), Intrinsic::x86_bmi_pext_64);
    26 
    2711        uint64_t pextBaseMask = mTwistWidth == 2 ? 0x5555555555555555: 0x1111111111111111;
    2812
    29         Value* inputBasePtr = b->CreatePointerCast(b->getInputStreamBlockPtr("byteStream", b->getSize(0)), b->getInt64Ty()->getPointerTo());
    30 
    31         Value* outputBlocks[mTwistWidth];
    3213        for (unsigned i = 0; i < mTwistWidth; i++) {
    3314            outputBlocks[i] = ConstantVector::getNullValue(b->getBitBlockType());
     
    6243            }
    6344        }
     45    }
     46
     47
     48
     49
     50    UntwistByPEXTKernel::UntwistByPEXTKernel(const std::unique_ptr<kernel::KernelBuilder> &b, unsigned numberOfOutputStream, unsigned twistWidth)
     51            :BlockOrientedKernel("UntwistByPEXTKernel",
     52                                 {
     53                                         Binding{b->getStreamSetTy(1, twistWidth), "byteStream", FixedRate(), Principal()}
     54                                 },
     55                                 {
     56                                         Binding{b->getStreamSetTy(numberOfOutputStream, 1), "basisBits"}
     57                                 }, {}, {}, {}),
     58             mNumberOfOutputStream(numberOfOutputStream),
     59             mTwistWidth(twistWidth)
     60    {
     61        assert(twistWidth == 2 || twistWidth == 4);
     62        assert(numberOfInputStream <= twistWidth);
     63
     64    }
     65
     66    void UntwistByPEXTKernel::generateDoBlockMethod(const std::unique_ptr<KernelBuilder> &b) {
     67        Value* inputBasePtr = b->CreatePointerCast(b->getInputStreamBlockPtr("byteStream", b->getSize(0)), b->getInt64Ty()->getPointerTo());
     68        Value* outputBlocks[mTwistWidth];
     69
     70        untwistByPEXT(b, inputBasePtr, mTwistWidth, outputBlocks);
    6471
    6572        for (unsigned i = 0; i < mNumberOfOutputStream; i++) {
    6673            b->storeOutputStreamBlock("basisBits", b->getInt32(i), outputBlocks[i]);
     74        }
     75    }
     76
     77
     78    UntwistMultipleByPEXTKernel::UntwistMultipleByPEXTKernel(const std::unique_ptr<kernel::KernelBuilder> &b,
     79                                                             std::vector<unsigned> numberOfOutputStreams,
     80                                                             unsigned twistWidth) :
     81            BlockOrientedKernel("UntwistMultipleByPEXTKernel",
     82                                {
     83                                        Binding{b->getStreamSetTy(1, twistWidth), "byteStream", FixedRate(),
     84                                                Principal()}
     85                                },
     86                                {
     87                                        //Binding{b->getStreamSetTy(numberOfOutputStream, 1), "basisBits"}
     88                                }, {}, {}, {}),
     89            mNumberOfOutputStreams(numberOfOutputStreams),
     90            mTwistWidth(twistWidth) {
     91        assert(twistWidth == 2 || twistWidth == 4);
     92
     93        size_t totalNumOfStreams = 0;
     94        for (unsigned i = 0; i < numberOfOutputStreams.size(); i++) {
     95            totalNumOfStreams += numberOfOutputStreams[i];
     96            mStreamSetOutputs.push_back(Binding{b->getStreamSetTy(numberOfOutputStreams[i]), "basisBits_" + std::to_string(i) });
     97        }
     98        assert(totalNumOfStreams < twistWidth);
     99    }
     100
     101    void UntwistMultipleByPEXTKernel::generateDoBlockMethod(const std::unique_ptr<KernelBuilder> &b) {
     102        Value* inputBasePtr = b->CreatePointerCast(b->getInputStreamBlockPtr("byteStream", b->getSize(0)), b->getInt64Ty()->getPointerTo());
     103        Value* outputBlocks[mTwistWidth];
     104        untwistByPEXT(b, inputBasePtr, mTwistWidth, outputBlocks);
     105
     106        unsigned iStreamIndex = 0;
     107        for (unsigned i = 0; i < mNumberOfOutputStreams.size(); i++) {
     108            for (unsigned j = 0; j < mNumberOfOutputStreams[i]; j++) {
     109                b->storeOutputStreamBlock("basisBits_" + std::to_string(i), b->getInt32(j), outputBlocks[iStreamIndex]);
     110                iStreamIndex++;
     111            }
    67112        }
    68113    }
  • icGREP/icgrep-devel/icgrep/kernels/lz4/untwist_kernel.h

    r6143 r6144  
    1313    class UntwistByPEXTKernel final : public BlockOrientedKernel{
    1414    public:
    15         UntwistByPEXTKernel(const std::unique_ptr<kernel::KernelBuilder> & b, unsigned numberOfOutputStream = 4, unsigned twistWidth = 2);
     15        UntwistByPEXTKernel(const std::unique_ptr<kernel::KernelBuilder> & b, unsigned numberOfOutputStream, unsigned twistWidth);
    1616    protected:
    1717        const size_t mNumberOfOutputStream;
     
    1919        void generateDoBlockMethod(const std::unique_ptr<KernelBuilder> & b) override;
    2020    };
     21
     22    class UntwistMultipleByPEXTKernel final : public BlockOrientedKernel{
     23    public:
     24        UntwistMultipleByPEXTKernel(const std::unique_ptr<kernel::KernelBuilder> & b, std::vector<unsigned> numberOfOutputStreams, unsigned twistWidth);
     25    protected:
     26        const std::vector<unsigned> mNumberOfOutputStreams;
     27        const size_t mTwistWidth;
     28        void generateDoBlockMethod(const std::unique_ptr<KernelBuilder> & b) override;
     29    };
     30
    2131
    2232    class StreamCompareKernel final : public BlockOrientedKernel{
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_base_generator.cpp

    r6143 r6144  
    4545#include <re/re_seq.h>
    4646#include <kernels/kernel_builder.h>
     47#include <re/re_alt.h>
    4748
    4849namespace re { class CC; }
     
    5253using namespace kernel;
    5354using namespace grep;
    54 
    55 LZ4GrepBaseGenerator::LZ4GrepBaseGenerator(): LZ4BaseGenerator() {
     55using namespace re;
     56
     57LZ4GrepBaseGenerator::LZ4GrepBaseGenerator()
     58        : LZ4BaseGenerator(),
     59          u8NonFinalRe(makeAlt({makeByte(0xC2, 0xF4),
     60                                makeSeq({makeByte(0xE0, 0xF4), makeByte(0x80, 0xBF)}),
     61                                makeSeq({makeByte(0xF0, 0xF4), makeByte(0x80, 0xBF), makeByte(0x80, 0xBF)})}))
     62{
    5663    mGrepRecordBreak = grep::GrepRecordBreakKind::LF;
    5764    mMoveMatchesToEOL = true;
     
    167174    std::set<re::Name *> UnicodeProperties;
    168175
    169 
    170     re::Seq* seq = re::makeSeq();
    171     re::RE* targetRe = mRE;
    172 
     176    StreamSetBuffer* fakeMatchCopiedBits = nullptr;
     177    StreamSetBuffer* u8NoFinalStream = nullptr;
     178    StreamSetBuffer * uncompressedCharClasses = nullptr;
    173179
    174180    re::CC* linefeedCC = nullptr;
     181
     182
    175183    if (utf8CC) {
     184        re::Seq* seq = re::makeSeq();
     185        re::RE* targetRe = mRE;
     186
    176187        linefeedCC = re::makeCC(0x0A);
     188
     189        seq->push_back(targetRe);
     190        seq->push_back(std::move(linefeedCC));
     191
     192        std::vector<re::CC*> UnicodeSets = re::collectCCs(seq, &cc::Unicode, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));;
     193
     194        mpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSets);
     195        mRE = transformCCs(mpx.get(), targetRe);
     196
     197
     198        std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
     199        auto numOfCharacterClasses = mpx_basis.size();
     200        StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize, 1);
     201
     202        kernel::Kernel * ccK = mGrepDriver->addKernelInstance<kernel::CharClassesKernel>(idb, std::move(mpx_basis), false, cc::BitNumbering::BigEndian);
     203        mGrepDriver->makeKernelCall(ccK, {compressedBitStream}, {CharClasses});
     204
     205
     206
     207        StreamSetBuffer* compressedNonFinalStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
     208        kernel::Kernel * nonFinalK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, u8NonFinalRe, externalStreamNames, std::vector<cc::Alphabet *>(), cc::BitNumbering::BigEndian);
     209        mGrepDriver->makeKernelCall(nonFinalK, {compressedBitStream}, {compressedNonFinalStream});
     210
     211
     212
     213        auto decompressedStreams = this->decompressBitStreams(compressedByteStream, {CharClasses, compressedNonFinalStream});
     214
     215//        uncompressedCharClasses = this->decompressBitStream(compressedByteStream, CharClasses);
     216        uncompressedCharClasses = decompressedStreams[0];
     217        u8NoFinalStream = decompressedStreams[1];
     218
     219
     220
     221        auto fakeStreams = this->generateFakeStreams(idb, uncompressedCharClasses, std::vector<unsigned>{8});
     222        fakeMatchCopiedBits = fakeStreams[0];
     223
    177224    } else {
     225        re::Seq* seq = re::makeSeq();
     226        re::RE* targetRe = mRE;
     227        targetRe = re::toUTF8(targetRe, true);
     228
    178229        linefeedCC = re::makeByte(0x0A);
    179         targetRe = re::toUTF8(targetRe, true);
    180     }
    181 
    182     seq->push_back(targetRe);
    183     seq->push_back(std::move(linefeedCC));
    184 
    185     std::vector<re::CC*> UnicodeSets;
    186     if (utf8CC) {
    187         UnicodeSets = re::collectCCs(seq, &cc::Unicode, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));
    188     } else {
    189         UnicodeSets = re::collectCCs(seq, &cc::Byte, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));
    190     }
     230
     231        seq->push_back(targetRe);
     232        seq->push_back(std::move(linefeedCC));
     233
     234        std::vector<re::CC*> UnicodeSets = re::collectCCs(seq, &cc::Byte, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));
     235
     236        mpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSets);
     237        mRE = transformCCs(mpx.get(), targetRe);
     238
     239        std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
     240        auto numOfCharacterClasses = mpx_basis.size();
     241        StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize, 1);
     242
     243        kernel::Kernel * ccK = mGrepDriver->addKernelInstance<kernel::ByteClassesKernel>(idb, std::move(mpx_basis), false, cc::BitNumbering::BigEndian);
     244        mGrepDriver->makeKernelCall(ccK, {compressedBitStream}, {CharClasses});
     245
     246        uncompressedCharClasses = this->decompressBitStream(compressedByteStream, CharClasses);
     247        auto fakeStreams = this->generateFakeStreams(idb, uncompressedCharClasses, std::vector<unsigned>{8, 1});
     248        fakeMatchCopiedBits = fakeStreams[0];
     249        u8NoFinalStream = fakeStreams[1];
     250    }
     251
     252
     253
     254
    191255
    192256    StreamSetBuffer * const MatchResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
    193257
    194     mpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSets);
    195     mRE = transformCCs(mpx.get(), targetRe);
    196 
    197 
    198     std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
    199     auto numOfCharacterClasses = mpx_basis.size();
    200     StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize, 1);
    201 
    202     kernel::Kernel * ccK = nullptr;
    203     if (utf8CC) {
    204         ccK = mGrepDriver->addKernelInstance<kernel::CharClassesKernel>(idb, std::move(mpx_basis), false, cc::BitNumbering::BigEndian);
    205     } else {
    206         ccK = mGrepDriver->addKernelInstance<kernel::ByteClassesKernel>(idb, std::move(mpx_basis), false, cc::BitNumbering::BigEndian);
    207     }
    208     mGrepDriver->makeKernelCall(ccK, {compressedBitStream}, {CharClasses});
    209 
    210     StreamSetBuffer * uncompressedCharClasses = this->decompressBitStream(compressedByteStream, CharClasses);
    211 
    212 
    213     StreamSetBuffer * fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8),
    214                                                                               this->getDefaultBufferBlocks(), 1);
    215     StreamSetBuffer * u8NoFinalStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1),
    216                                                                           this->getDefaultBufferBlocks(), 1);
    217 
    218     Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, std::vector<unsigned>({8, 1}));
    219     mPxDriver.makeKernelCall(fakeStreamGeneratorK, {uncompressedCharClasses}, {fakeMatchCopiedBits, u8NoFinalStream});
    220258
    221259    StreamSetBuffer * LineBreakStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getDefaultBufferBlocks(), 1);
     
    295333            std::set<re::Name *> UnicodeProperties;
    296334
     335
     336
     337
     338            StreamSetBuffer* nonFinalStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
     339            kernel::Kernel * nonFinalK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, u8NonFinalRe, externalStreamNames, std::vector<cc::Alphabet *>(), cc::BitNumbering::BigEndian);
     340            mGrepDriver->makeKernelCall(nonFinalK, icgrepInputSets, {nonFinalStream});
     341            icgrepInputSets.push_back(nonFinalStream);
     342            externalStreamNames.push_back("UTF8_nonfinal");
     343
     344
     345
     346
     347
    297348            StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
    298349            kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mRE, externalStreamNames, std::vector<cc::Alphabet *>(), cc::BitNumbering::BigEndian);
     
    452503    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
    453504}
     505
     506std::vector<parabix::StreamSetBuffer *>
     507LZ4GrepBaseGenerator::generateFakeStreams(const std::unique_ptr<kernel::KernelBuilder> &idb,
     508                                          parabix::StreamSetBuffer *refStream, std::vector<unsigned> numOfStreams) {
     509
     510    std::vector<StreamSetBuffer *> outputStreams;
     511    for (unsigned i = 0; i < numOfStreams.size(); i++) {
     512        outputStreams.push_back(mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfStreams[i]),
     513                                                                  this->getDefaultBufferBlocks(), 1));
     514    }
     515    Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, refStream->getNumOfStreams(), numOfStreams);
     516    mPxDriver.makeKernelCall(fakeStreamGeneratorK, {refStream}, outputStreams);
     517    return outputStreams;
     518}
     519
     520std::vector<parabix::StreamSetBuffer *>
     521LZ4GrepBaseGenerator::decompressBitStreams(parabix::StreamSetBuffer *compressedByteStream,
     522                                           std::vector<parabix::StreamSetBuffer *> compressedBitStreams) {
     523    // Default implementation here will be slow
     524    std::vector<parabix::StreamSetBuffer *> retVec;
     525    for (unsigned i = 0; i < compressedBitStreams.size(); i++) {
     526        retVec.push_back(this->decompressBitStream(compressedByteStream, compressedBitStreams[i]));
     527    }
     528    return retVec;
     529}
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_base_generator.h

    r6143 r6144  
    3838    virtual parabix::StreamSetBuffer* generateUncompressedBitStreams() = 0;
    3939    virtual parabix::StreamSetBuffer* decompressBitStream(parabix::StreamSetBuffer* compressedByteStream, parabix::StreamSetBuffer* compressedBitStream) = 0;
     40    virtual std::vector<parabix::StreamSetBuffer*> decompressBitStreams(parabix::StreamSetBuffer* compressedByteStream, std::vector<parabix::StreamSetBuffer*> compressedBitStreams);
     41
    4042
    4143private:
     
    4850    std::set<re::Name *> mUnicodeProperties;
    4951    bool mMoveMatchesToEOL;
     52    re::RE* u8NonFinalRe;
    5053
    5154
     
    7578    );
    7679    std::unique_ptr<cc::MultiplexedAlphabet> mpx;
     80
     81
     82    std::vector<parabix::StreamSetBuffer*> generateFakeStreams(
     83            const std::unique_ptr<kernel::KernelBuilder> & iBuilder,
     84            parabix::StreamSetBuffer* refStream,
     85            std::vector<unsigned> numOfStreams
     86    );
     87
    7788};
    7889
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_bitstream_generator.cpp

    r6143 r6144  
    2222    return ret[0];
    2323}
     24
     25std::vector<parabix::StreamSetBuffer *>
     26LZ4GrepBitStreamGenerator::decompressBitStreams(parabix::StreamSetBuffer *compressedByteStream,
     27                                                std::vector<parabix::StreamSetBuffer *> compressedBitStreams) {
     28    return this->convertCompressedBitsStreamWithBitStreamAioApproach(compressedByteStream, compressedBitStreams);
     29}
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_bitstream_generator.h

    r6143 r6144  
    1010    virtual parabix::StreamSetBuffer* generateUncompressedBitStreams() override;
    1111    virtual parabix::StreamSetBuffer* decompressBitStream(parabix::StreamSetBuffer* compressedByteStream, parabix::StreamSetBuffer* compressedBitStream) override;
     12    virtual std::vector<parabix::StreamSetBuffer*> decompressBitStreams(parabix::StreamSetBuffer* compressedByteStream, std::vector<parabix::StreamSetBuffer*> compressedBitStreams) override;
    1213};
    1314
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_bytestream_generator.cpp

    r6143 r6144  
    6868        StreamSetBuffer* twistedCharClasses = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 2),
    6969                                                                                this->getDefaultBufferBlocks(), 1);
    70         kernel::Kernel* twistK = mPxDriver.addKernelInstance<kernel::TwistByPDEPKernel>(b, numOfStreams, 2);
     70        kernel::Kernel* twistK = mPxDriver.addKernelInstance<kernel::TwistMultipleByPDEPKernel>(b, std::vector<unsigned>{numOfStreams}, 2);
    7171        mPxDriver.makeKernelCall(twistK, {compressedBitStream}, {twistedCharClasses});
    7272
     
    9090        StreamSetBuffer* untwistedCharClasses = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(numOfStreams),
    9191                                                                                  this->getDefaultBufferBlocks(), 1);
    92         kernel::Kernel* untwistK = mPxDriver.addKernelInstance<kernel::UntwistByPEXTKernel>(b, numOfStreams, 2);
     92        kernel::Kernel* untwistK = mPxDriver.addKernelInstance<kernel::UntwistMultipleByPEXTKernel>(b, std::vector<unsigned>{numOfStreams}, 2);
    9393        mPxDriver.makeKernelCall(untwistK, {uncompressedTwistedCharClasses}, {untwistedCharClasses});
    9494        return untwistedCharClasses;
     
    9797        StreamSetBuffer* twistedCharClasses = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 4),
    9898                                                                                this->getDefaultBufferBlocks(), 1);
    99         kernel::Kernel* twistK = mPxDriver.addKernelInstance<kernel::TwistByPDEPKernel>(b, numOfStreams, 4);
     99        kernel::Kernel* twistK = mPxDriver.addKernelInstance<kernel::TwistMultipleByPDEPKernel>(b, std::vector<unsigned>{numOfStreams}, 4);
    100100        mPxDriver.makeKernelCall(twistK, {compressedBitStream}, {twistedCharClasses});
    101101
     
    120120        StreamSetBuffer* untwistedCharClasses = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(numOfStreams),
    121121                                                                                  this->getDefaultBufferBlocks(), 1);
    122         kernel::Kernel* untwistK = mPxDriver.addKernelInstance<kernel::UntwistByPEXTKernel>(b, numOfStreams, 4);
     122        kernel::Kernel* untwistK = mPxDriver.addKernelInstance<kernel::UntwistMultipleByPEXTKernel>(b, std::vector<unsigned>{numOfStreams}, 4);
    123123        mPxDriver.makeKernelCall(untwistK, {uncompressedTwistedCharClasses}, {untwistedCharClasses});
    124124        return untwistedCharClasses;
     
    154154    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(b, cc::BitNumbering::BigEndian, true, prefix, numOfStreams);
    155155    mPxDriver.makeKernelCall(s2pk, {decompressionMtxByteStream}, {uncompressedMtxBitStream});
    156     this->s2p(decompressionMtxByteStream);
    157156    return uncompressedMtxBitStream;
    158157}
     158
     159std::vector<parabix::StreamSetBuffer *>
     160LZ4GrepByteStreamGenerator::decompressBitStreams(parabix::StreamSetBuffer *compressedByteStream,
     161                                                 std::vector<parabix::StreamSetBuffer *> compressedBitStreams) {
     162
     163
     164    auto & b = mPxDriver.getBuilder();
     165    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
     166
     167    unsigned totalStreamNum = 0;
     168    std::vector<unsigned> numOfStreams;
     169    std::vector<StreamSetBuffer*> retStreams;
     170    for (unsigned i = 0; i < compressedBitStreams.size(); i++) {
     171        unsigned n = compressedBitStreams[i]->getNumOfStreams();
     172
     173        numOfStreams.push_back(n);
     174        totalStreamNum += n;
     175        retStreams.push_back(mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(n), this->getDefaultBufferBlocks(), 1));
     176    }
     177
     178    if (totalStreamNum == 1) {
     179        // TODO
     180    } else if (totalStreamNum <= 4) {
     181        unsigned twistWidth = totalStreamNum == 2 ? 2 : 4;
     182
     183        StreamSetBuffer* twistedStream = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, twistWidth),
     184                                                                                this->getDefaultBufferBlocks(), 1);
     185        kernel::Kernel* twistK = mPxDriver.addKernelInstance<kernel::TwistMultipleByPDEPKernel>(b, numOfStreams, twistWidth);
     186        mPxDriver.makeKernelCall(twistK, compressedBitStreams, {twistedStream});
     187
     188        StreamSetBuffer* uncompressedTwistedStream = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, twistWidth),
     189                                                                                            this->getDefaultBufferBlocks(), 1);
     190        Kernel* lz4I4AioK = mPxDriver.addKernelInstance<LZ4TwistDecompressionKernel>(b, twistWidth);
     191        lz4I4AioK->setInitialArguments({mFileSize});
     192        mPxDriver.makeKernelCall(lz4I4AioK, {
     193                compressedByteStream,
     194
     195                blockInfo.isCompress,
     196                blockInfo.blockStart,
     197                blockInfo.blockEnd,
     198
     199                twistedStream
     200        }, {
     201                uncompressedTwistedStream
     202                                 });
     203
     204        kernel::Kernel* untwistK = mPxDriver.addKernelInstance<kernel::UntwistMultipleByPEXTKernel>(b, numOfStreams, twistWidth);
     205        mPxDriver.makeKernelCall(untwistK, {uncompressedTwistedStream}, retStreams);
     206    } else {
     207        // TODO
     208    }
     209    return retStreams;
     210}
  • icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_bytestream_generator.h

    r6143 r6144  
    99    virtual parabix::StreamSetBuffer* generateUncompressedBitStreams() override;
    1010    virtual parabix::StreamSetBuffer* decompressBitStream(parabix::StreamSetBuffer* compressedByteStream, parabix::StreamSetBuffer* compressedBitStream) override;
     11    virtual std::vector<parabix::StreamSetBuffer*> decompressBitStreams(parabix::StreamSetBuffer* compressedByteStream, std::vector<parabix::StreamSetBuffer*> compressedBitStreams) override;
    1112
    1213private:
     
    1617            std::string prefix
    1718    );
     19
     20
    1821};
    1922
Note: See TracChangeset for help on using the changeset viewer.