Ignore:
Timestamp:
Jun 27, 2018, 3:48:18 AM (12 months ago)
Author:
xwa163
Message:

Implement bitstream decompression version of LZ4 Aio Kernel

Location:
icGREP/icgrep-devel/icgrep/lz4
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.cpp

    r6111 r6118  
    2626#include <kernels/lz4/aio/lz4_parallel_bytestream_aio.h>
    2727#include <kernels/lz4/aio/lz4_swizzled_aio.h>
     28#include <kernels/lz4/aio/lz4_bitstream_aio.h>
    2829#include <kernels/bitstream_pdep_kernel.h>
    2930#include <kernels/lz4/lz4_bitstream_not_kernel.h>
     
    405406}
    406407
     408StreamSetBuffer * LZ4Generator::generateBitStreamAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     409//// Decode Block Information
     410    StreamSetBuffer * const BlockData_IsCompressed = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder), 1);
     411    StreamSetBuffer * const BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
     412    StreamSetBuffer * const BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
     413
     414    //// Generate Helper Markers Extenders, FX, XF
     415//    StreamSetBuffer * const Extenders = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks(iBuilder), 1);
     416//    mMatchOffsetMarker = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks(iBuilder));
     417//    Kernel * extenderK = mPxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8);
     418//    mPxDriver.makeKernelCall(extenderK, {mCompressedBasisBits}, {Extenders});
     419
     420
     421    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZ4BlockDecoderKernel>(iBuilder);
     422    blockDecoderK->setInitialArguments({iBuilder->CreateTrunc(mHasBlockChecksum, iBuilder->getInt1Ty()), mHeaderSize, mFileSize});
     423    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_IsCompressed, BlockData_BlockStart, BlockData_BlockEnd});
     424
     425    StreamSetBuffer* decompressedBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getDecompressedBufferBlocks(iBuilder));
     426
     427    Kernel* lz4BitStreamAioK = mPxDriver.addKernelInstance<LZ4BitStreamAioKernel>(iBuilder);
     428    lz4BitStreamAioK->setInitialArguments({mFileSize});
     429    mPxDriver.makeKernelCall(
     430            lz4BitStreamAioK,
     431            {
     432                    mCompressedByteStream,
     433
     434//                    Extenders,
     435
     436                    // Block Data
     437                    BlockData_IsCompressed,
     438                    BlockData_BlockStart,
     439                    BlockData_BlockEnd,
     440
     441                    mCompressedBasisBits
     442            }, {
     443                    decompressedBitStream
     444            });
     445    return decompressedBitStream;
     446
     447}
     448
    407449StreamSetBuffer * LZ4Generator::generateSwizzledAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    408450    //// Decode Block Information
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.h

    r6111 r6118  
    4444    virtual parabix::StreamSetBuffer * generateAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    4545    virtual parabix::StreamSetBuffer * generateSwizzledAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     46    virtual parabix::StreamSetBuffer * generateBitStreamAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     47
    4648    virtual std::pair<parabix::StreamSetBuffer*, parabix::StreamSetBuffer*> generateSwizzleExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    4749    virtual parabix::StreamSetBuffer* generateBitStreamExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
  • icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.cpp

    r6112 r6118  
    551551}
    552552
     553void LZ4GrepGenerator::generateBitStreamAioPipeline(re::RE* regex) {
     554    auto & iBuilder = mPxDriver.getBuilder();
     555    this->generateCountOnlyMainFunc(iBuilder);
     556
     557    // GeneratePipeline
     558    this->generateLoadByteStreamAndBitStream(iBuilder);
     559    StreamSetBuffer * const decompressionBitStream = this->generateBitStreamAIODecompression(iBuilder);
     560
     561    StreamSetBuffer * LineBreakStream;
     562    StreamSetBuffer * Matches;
     563    std::vector<re::RE*> res = {regex};
     564    std::tie(LineBreakStream, Matches) = grepPipeline(res, decompressionBitStream);
     565
     566    /*
     567    StreamSetBuffer * const decompressionByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks(iBuilder));
     568    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     569    mPxDriver.makeKernelCall(p2sK, {decompressionBitStream}, {decompressionByteStream});
     570
     571    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     572    outK->setInitialArguments({iBuilder->GetString("/Users/wxy325/developer/LZ4-sample-files/workspace/lz4d-normal/8k_.txt")});
     573    mPxDriver.makeKernelCall(outK, {decompressionByteStream}, {});
     574    */
     575    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
     576
     577    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
     578    mPxDriver.generatePipelineIR();
     579
     580    iBuilder->setKernel(matchCountK);
     581    Value * matchedLineCount = iBuilder->getAccumulator("countResult");
     582    matchedLineCount = iBuilder->CreateZExt(matchedLineCount, iBuilder->getInt64Ty());
     583    mPxDriver.deallocateBuffers();
     584
     585    iBuilder->CreateRet(matchedLineCount);
     586
     587    mPxDriver.finalizeObject();
     588}
     589
    553590void LZ4GrepGenerator::generateSwizzledAioPipeline(re::RE* regex) {
    554591    auto & iBuilder = mPxDriver.getBuilder();
  • icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.h

    r6111 r6118  
    3636
    3737    void generateSwizzledAioPipeline(re::RE* regex);
     38    void generateBitStreamAioPipeline(re::RE* regex);
    3839
    3940    void generateAioPipeline(re::RE* regex);
Note: See TracChangeset for help on using the changeset viewer.