Ignore:
Timestamp:
May 14, 2018, 12:25:34 PM (13 months ago)
Author:
xwa163
Message:

Init checkin for bitstream version of lz4d match copy

Location:
icGREP/icgrep-devel/icgrep/lz4
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.cpp

    r6029 r6039  
    2020#include <kernels/swizzled_multiple_pdep_kernel.h>
    2121#include <kernels/lz4/lz4_swizzled_match_copy_kernel.h>
     22#include <kernels/lz4/lz4_bitstream_match_copy_kernel.h>
    2223#include <kernels/lz4/lz4_block_decoder.h>
    2324#include <kernels/lz4/lz4_index_builder.h>
    2425#include <kernels/bitstream_pdep_kernel.h>
     26#include <kernels/lz4/lz4_bitstream_not_kernel.h>
    2527
    2628namespace re { class CC; }
     
    3840}
    3941
    40 
    41 
    42 void LZ4Generator::generateExtractOnlyPipeline(const std::string& outputFile) {
    43     auto & iBuilder = mPxDriver.getBuilder();
    44     this->generateMainFunc(iBuilder);
    45 
    46     StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
    47 
    48     // GeneratePipeline
    49     this->generateLoadByteStreamAndBitStream(iBuilder);
    50 
    51 
    52     this->generateExtractAndDepositMarkers(iBuilder);
    53 
    54 
     42void LZ4Generator::generateExtractOnlyPipeline(const std::string &outputFile) {
     43    auto & iBuilder = mPxDriver.getBuilder();
     44    this->generateMainFunc(iBuilder);
     45
     46    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     47
     48    // GeneratePipeline
     49    this->generateLoadByteStreamAndBitStream(iBuilder);
     50    this->generateExtractAndDepositMarkers(iBuilder);
     51    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
     52
     53    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     54    mPxDriver.makeKernelCall(p2sK, {extractedBits}, {DecompressedByteStream});
     55
     56    // --------------------------------------------------------
     57    // End
     58    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     59
     60    outK->setInitialArguments({iBuilder->GetString(outputFile)});
     61    mPxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     62
     63    mPxDriver.generatePipelineIR();
     64    mPxDriver.deallocateBuffers();
     65
     66    iBuilder->CreateRetVoid();
     67
     68    mPxDriver.finalizeObject();
     69}
     70
     71void LZ4Generator::generateSwizzledExtractOnlyPipeline(const std::string &outputFile) {
     72    auto & iBuilder = mPxDriver.getBuilder();
     73    this->generateMainFunc(iBuilder);
     74
     75    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     76
     77    // GeneratePipeline
     78    this->generateLoadByteStreamAndBitStream(iBuilder);
     79    this->generateExtractAndDepositMarkers(iBuilder);
    5580    auto swizzle = this->generateSwizzleExtractData(iBuilder);
    5681
     
    90115    this->generateLoadByteStreamAndBitStream(iBuilder);
    91116    this->generateExtractAndDepositMarkers(iBuilder);
    92 
    93     auto swizzle = this->generateSwizzleExtractData(iBuilder); // TODO: use compression kernel instead
    94 
    95     StreamSetBuffer * extractedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
    96     Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    97     mPxDriver.makeKernelCall(unSwizzleK, {swizzle.first, swizzle.second}, {extractedBits});
     117    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
    98118
    99119    StreamSetBuffer * depositedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getDecompressedBufferBlocks());
     
    101121    mPxDriver.makeKernelCall(bitStreamPDEPk, {mDepositMarker, extractedBits}, {depositedBits});
    102122
    103     /*
     123    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     124    mPxDriver.makeKernelCall(p2sK, {depositedBits}, {DecompressedByteStream});
     125
     126    // --------------------------------------------------------
     127    // End
     128    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     129
     130    outK->setInitialArguments({iBuilder->GetString(outputFile)});
     131    mPxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     132
     133    mPxDriver.generatePipelineIR();
     134    mPxDriver.deallocateBuffers();
     135
     136    iBuilder->CreateRetVoid();
     137
     138    mPxDriver.finalizeObject();
     139}
     140
     141void LZ4Generator::generateSwizzledExtractAndDepositOnlyPipeline(const std::string &outputFile) {
     142    auto & iBuilder = mPxDriver.getBuilder();
     143    this->generateMainFunc(iBuilder);
     144
     145    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     146
     147    // GeneratePipeline
     148    this->generateLoadByteStreamAndBitStream(iBuilder);
     149    this->generateExtractAndDepositMarkers(iBuilder);
     150
     151    auto swizzle = this->generateSwizzleExtractData(iBuilder);
    104152
    105153    StreamSetBuffer * depositedSwizzle0 = mPxDriver.addBuffer<CircularCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), this->getInputBufferBlocks(), 1);
     
    113161    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    114162    mPxDriver.makeKernelCall(unSwizzleK, {depositedSwizzle0, depositedSwizzle1}, {depositedBits});
    115      */
    116163
    117164    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     
    132179}
    133180
    134 void LZ4Generator::generatePipeline(const std::string& outputFile) {
     181void LZ4Generator::generatePipeline(const std::string &outputFile) {
     182    auto & iBuilder = mPxDriver.getBuilder();
     183    this->generateMainFunc(iBuilder);
     184
     185    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     186
     187    // GeneratePipeline
     188    this->generateLoadByteStreamAndBitStream(iBuilder);
     189    this->generateExtractAndDepositMarkers(iBuilder);
     190    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
     191
     192    StreamSetBuffer * depositedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getDecompressedBufferBlocks());
     193    Kernel * bitStreamPDEPk = mPxDriver.addKernelInstance<BitStreamPDEPKernel>(iBuilder, 8);
     194    mPxDriver.makeKernelCall(bitStreamPDEPk, {mDepositMarker, extractedBits}, {depositedBits});
     195
     196    StreamSetBuffer * matchCopiedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     197    Kernel * bitStreamMatchCopyK = mPxDriver.addKernelInstance<LZ4BitStreamMatchCopyKernel>(iBuilder, 8);
     198    mPxDriver.makeKernelCall(bitStreamMatchCopyK, {mMatchOffsetMarker, mM0Marker, mCompressedByteStream, depositedBits}, {matchCopiedBits});
     199
     200    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     201    mPxDriver.makeKernelCall(p2sK, {matchCopiedBits}, {DecompressedByteStream});
     202
     203    // --------------------------------------------------------
     204    // End
     205    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     206
     207    outK->setInitialArguments({iBuilder->GetString(outputFile)});
     208    mPxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     209
     210    mPxDriver.generatePipelineIR();
     211    mPxDriver.deallocateBuffers();
     212
     213    iBuilder->CreateRetVoid();
     214
     215    mPxDriver.finalizeObject();
     216}
     217
     218void LZ4Generator::generateSwizzledPipeline(const std::string &outputFile) {
    135219    auto & iBuilder = mPxDriver.getBuilder();
    136220    this->generateMainFunc(iBuilder);
     
    158242
    159243    // Produce unswizzled bit streams
    160     StreamSetBuffer * extractedbits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     244    StreamSetBuffer * matchCopiedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
    161245    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    162     mPxDriver.makeKernelCall(unSwizzleK, {matchCopiedSwizzle0, matchCopiedSwizzle1}, {extractedbits});
    163 
    164 
    165     Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
    166     mPxDriver.makeKernelCall(p2sK, {extractedbits}, {DecompressedByteStream});
     246    mPxDriver.makeKernelCall(unSwizzleK, {matchCopiedSwizzle0, matchCopiedSwizzle1}, {matchCopiedBits});
     247
     248
     249    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     250    mPxDriver.makeKernelCall(p2sK, {matchCopiedBits}, {DecompressedByteStream});
    167251
    168252    // --------------------------------------------------------
     
    304388}
    305389
     390parabix::StreamSetBuffer* LZ4Generator::generateBitStreamExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     391    StreamSetBuffer * const compressionMarker = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks());
     392    Kernel * bitstreamNotK = mPxDriver.addKernelInstance<LZ4BitStreamNotKernel>(iBuilder);
     393    mPxDriver.makeKernelCall(bitstreamNotK, {mDeletionMarker}, {compressionMarker});
     394
     395    // Deletion
     396    StreamSetBuffer * deletedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     397    StreamSetBuffer * deletionCounts = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     398
     399    Kernel * delK = mPxDriver.addKernelInstance<PEXTFieldCompressKernel>(iBuilder, 64, 8);
     400    mPxDriver.makeKernelCall(delK, {mCompressedBasisBits, compressionMarker}, {deletedBits, deletionCounts});
     401
     402    StreamSetBuffer * compressedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     403    Kernel * streamCompressionK = mPxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
     404    mPxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
     405
     406    return compressedBits;
     407}
     408
    306409int LZ4Generator::get4MbBufferBlocks() {
    307410    return 4 * 1024 * 1024 / codegen::BlockSize;
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.h

    r6026 r6039  
    2626    MainFunctionType getMainFunc();
    2727
    28     void generatePipeline(const std::string& outputFile);
    29     void generateExtractOnlyPipeline(const std::string& outputFile);
    30     void generateExtractAndDepositOnlyPipeline(const std::string& outputFile);
     28    void generatePipeline(const std::string &outputFile);
     29    void generateSwizzledPipeline(const std::string &outputFile);
     30    void generateExtractOnlyPipeline(const std::string &outputFile);
     31    void generateSwizzledExtractOnlyPipeline(const std::string &outputFile);
     32    void generateExtractAndDepositOnlyPipeline(const std::string &outputFile);
     33    void generateSwizzledExtractAndDepositOnlyPipeline(const std::string &outputFile);
    3134
    3235protected:
     
    3740    virtual void generateExtractAndDepositMarkers(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3841    virtual std::pair<parabix::StreamSetBuffer*, parabix::StreamSetBuffer*> generateSwizzleExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     42    virtual parabix::StreamSetBuffer* generateBitStreamExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3943
    4044    // BufferSize related Helper Function
  • icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.cpp

    r6026 r6039  
    2020#include <kernels/swizzled_multiple_pdep_kernel.h>
    2121#include <kernels/lz4/lz4_swizzled_match_copy_kernel.h>
     22#include <kernels/lz4/lz4_bitstream_match_copy_kernel.h>
     23#include <kernels/lz4/lz4_bitstream_not_kernel.h>
     24#include <kernels/bitstream_pdep_kernel.h>
    2225#include <re/re_toolchain.h>
    2326
     
    4649#include <llvm/Support/raw_ostream.h>
    4750#include <llvm/Support/Debug.h>
     51
    4852
    4953
     
    549553}
    550554
    551 
    552 
    553 
    554555void LZ4GrepGenerator::invokeScanMatchGrep(char* fileBuffer, size_t blockStart, size_t blockEnd, bool hasBlockChecksum) {
    555556    auto main = this->getScanMatchGrepMainFunction();
     
    585586    mPxDriver.makeKernelCall(swizzledMatchCopyK, {mMatchOffsetMarker, mM0Marker, mCompressedByteStream, depositedSwizzle0, depositedSwizzle1}, {matchCopiedSwizzle0, matchCopiedSwizzle1});
    586587
    587 
    588588    // Produce unswizzled bit streams
    589589    StreamSetBuffer * extractedbits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
    590590    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    591591    mPxDriver.makeKernelCall(unSwizzleK, {matchCopiedSwizzle0, matchCopiedSwizzle1}, {extractedbits});
    592 
    593 
    594592
    595593    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     
    615613}
    616614
    617 void LZ4GrepGenerator::generateCountOnlyGrepPipeline(re::RE* regex) {
     615void LZ4GrepGenerator::generateCountOnlyGrepPipeline(re::RE *regex) {
     616    auto & iBuilder = mPxDriver.getBuilder();
     617    this->generateMainFunc(iBuilder);
     618
     619    // GeneratePipeline
     620    this->generateLoadByteStreamAndBitStream(iBuilder);
     621    this->generateExtractAndDepositMarkers(iBuilder);
     622
     623    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
     624
     625    StreamSetBuffer * depositedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getDecompressedBufferBlocks());
     626    Kernel * bitStreamPDEPk = mPxDriver.addKernelInstance<BitStreamPDEPKernel>(iBuilder, 8);
     627    mPxDriver.makeKernelCall(bitStreamPDEPk, {mDepositMarker, extractedBits}, {depositedBits});
     628
     629    StreamSetBuffer * matchCopiedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     630    Kernel * bitStreamMatchCopyK = mPxDriver.addKernelInstance<LZ4BitStreamMatchCopyKernel>(iBuilder, 8);
     631    mPxDriver.makeKernelCall(bitStreamMatchCopyK, {mMatchOffsetMarker, mM0Marker, mCompressedByteStream, depositedBits}, {matchCopiedBits});
     632
     633    StreamSetBuffer * LineBreakStream;
     634    StreamSetBuffer * Matches;
     635    std::vector<re::RE*> res = {regex};
     636    if (mEnableMultiplexing) {
     637        std::tie(LineBreakStream, Matches) = multiplexingGrepPipeline(res, matchCopiedBits);
     638    } else {
     639        std::tie(LineBreakStream, Matches) = grepPipeline(res, matchCopiedBits);
     640    };
     641
     642    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
     643    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
     644    mPxDriver.generatePipelineIR();
     645
     646    iBuilder->setKernel(matchCountK);
     647    Value * matchedLineCount = iBuilder->getAccumulator("countResult");
     648    matchedLineCount = iBuilder->CreateZExt(matchedLineCount, iBuilder->getInt64Ty());
     649    iBuilder->CallPrintInt("aaa", matchedLineCount);
     650
     651    mPxDriver.deallocateBuffers();
     652
     653    // TODO return matchedLineCount
     654//        idb->CreateRet(matchedLineCount);
     655
     656    iBuilder->CreateRetVoid();
     657
     658    mPxDriver.finalizeObject();
     659}
     660
     661
     662void LZ4GrepGenerator::generateSwizzledCountOnlyGrepPipeline(re::RE *regex) {
    618663    auto & iBuilder = mPxDriver.getBuilder();
    619664    this->generateMainFunc(iBuilder);
     
    662707    };
    663708
    664 
    665 
    666709    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
    667710    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
    668711    mPxDriver.generatePipelineIR();
    669 
    670712
    671713    iBuilder->setKernel(matchCountK);
     
    678720    // TODO return matchedLineCount
    679721//        idb->CreateRet(matchedLineCount);
    680 
    681722
    682723    iBuilder->CreateRetVoid();
  • icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.h

    r6026 r6039  
    1818public:
    1919    LZ4GrepGenerator(bool enableMultiplexing = false);
    20     void generateCountOnlyGrepPipeline(re::RE* regex);
     20    void generateSwizzledCountOnlyGrepPipeline(re::RE *regex);
     21    void generateCountOnlyGrepPipeline(re::RE *regex);
     22
    2123    void generateScanMatchGrepPipeline(re::RE* regex);
    2224    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(std::vector<re::RE *> &REs,
Note: See TracChangeset for help on using the changeset viewer.