Ignore:
Timestamp:
May 14, 2018, 12:25:34 PM (15 months ago)
Author:
xwa163
Message:

Init checkin for bitstream version of lz4d match copy

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.cpp

    r6029 r6039  
    2020#include <kernels/swizzled_multiple_pdep_kernel.h>
    2121#include <kernels/lz4/lz4_swizzled_match_copy_kernel.h>
     22#include <kernels/lz4/lz4_bitstream_match_copy_kernel.h>
    2223#include <kernels/lz4/lz4_block_decoder.h>
    2324#include <kernels/lz4/lz4_index_builder.h>
    2425#include <kernels/bitstream_pdep_kernel.h>
     26#include <kernels/lz4/lz4_bitstream_not_kernel.h>
    2527
    2628namespace re { class CC; }
     
    3840}
    3941
    40 
    41 
    42 void LZ4Generator::generateExtractOnlyPipeline(const std::string& outputFile) {
    43     auto & iBuilder = mPxDriver.getBuilder();
    44     this->generateMainFunc(iBuilder);
    45 
    46     StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
    47 
    48     // GeneratePipeline
    49     this->generateLoadByteStreamAndBitStream(iBuilder);
    50 
    51 
    52     this->generateExtractAndDepositMarkers(iBuilder);
    53 
    54 
     42void LZ4Generator::generateExtractOnlyPipeline(const std::string &outputFile) {
     43    auto & iBuilder = mPxDriver.getBuilder();
     44    this->generateMainFunc(iBuilder);
     45
     46    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     47
     48    // GeneratePipeline
     49    this->generateLoadByteStreamAndBitStream(iBuilder);
     50    this->generateExtractAndDepositMarkers(iBuilder);
     51    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
     52
     53    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     54    mPxDriver.makeKernelCall(p2sK, {extractedBits}, {DecompressedByteStream});
     55
     56    // --------------------------------------------------------
     57    // End
     58    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     59
     60    outK->setInitialArguments({iBuilder->GetString(outputFile)});
     61    mPxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     62
     63    mPxDriver.generatePipelineIR();
     64    mPxDriver.deallocateBuffers();
     65
     66    iBuilder->CreateRetVoid();
     67
     68    mPxDriver.finalizeObject();
     69}
     70
     71void LZ4Generator::generateSwizzledExtractOnlyPipeline(const std::string &outputFile) {
     72    auto & iBuilder = mPxDriver.getBuilder();
     73    this->generateMainFunc(iBuilder);
     74
     75    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     76
     77    // GeneratePipeline
     78    this->generateLoadByteStreamAndBitStream(iBuilder);
     79    this->generateExtractAndDepositMarkers(iBuilder);
    5580    auto swizzle = this->generateSwizzleExtractData(iBuilder);
    5681
     
    90115    this->generateLoadByteStreamAndBitStream(iBuilder);
    91116    this->generateExtractAndDepositMarkers(iBuilder);
    92 
    93     auto swizzle = this->generateSwizzleExtractData(iBuilder); // TODO: use compression kernel instead
    94 
    95     StreamSetBuffer * extractedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
    96     Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    97     mPxDriver.makeKernelCall(unSwizzleK, {swizzle.first, swizzle.second}, {extractedBits});
     117    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
    98118
    99119    StreamSetBuffer * depositedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getDecompressedBufferBlocks());
     
    101121    mPxDriver.makeKernelCall(bitStreamPDEPk, {mDepositMarker, extractedBits}, {depositedBits});
    102122
    103     /*
     123    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     124    mPxDriver.makeKernelCall(p2sK, {depositedBits}, {DecompressedByteStream});
     125
     126    // --------------------------------------------------------
     127    // End
     128    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     129
     130    outK->setInitialArguments({iBuilder->GetString(outputFile)});
     131    mPxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     132
     133    mPxDriver.generatePipelineIR();
     134    mPxDriver.deallocateBuffers();
     135
     136    iBuilder->CreateRetVoid();
     137
     138    mPxDriver.finalizeObject();
     139}
     140
     141void LZ4Generator::generateSwizzledExtractAndDepositOnlyPipeline(const std::string &outputFile) {
     142    auto & iBuilder = mPxDriver.getBuilder();
     143    this->generateMainFunc(iBuilder);
     144
     145    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     146
     147    // GeneratePipeline
     148    this->generateLoadByteStreamAndBitStream(iBuilder);
     149    this->generateExtractAndDepositMarkers(iBuilder);
     150
     151    auto swizzle = this->generateSwizzleExtractData(iBuilder);
    104152
    105153    StreamSetBuffer * depositedSwizzle0 = mPxDriver.addBuffer<CircularCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), this->getInputBufferBlocks(), 1);
     
    113161    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    114162    mPxDriver.makeKernelCall(unSwizzleK, {depositedSwizzle0, depositedSwizzle1}, {depositedBits});
    115      */
    116163
    117164    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     
    132179}
    133180
    134 void LZ4Generator::generatePipeline(const std::string& outputFile) {
     181void LZ4Generator::generatePipeline(const std::string &outputFile) {
     182    auto & iBuilder = mPxDriver.getBuilder();
     183    this->generateMainFunc(iBuilder);
     184
     185    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     186
     187    // GeneratePipeline
     188    this->generateLoadByteStreamAndBitStream(iBuilder);
     189    this->generateExtractAndDepositMarkers(iBuilder);
     190    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
     191
     192    StreamSetBuffer * depositedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getDecompressedBufferBlocks());
     193    Kernel * bitStreamPDEPk = mPxDriver.addKernelInstance<BitStreamPDEPKernel>(iBuilder, 8);
     194    mPxDriver.makeKernelCall(bitStreamPDEPk, {mDepositMarker, extractedBits}, {depositedBits});
     195
     196    StreamSetBuffer * matchCopiedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     197    Kernel * bitStreamMatchCopyK = mPxDriver.addKernelInstance<LZ4BitStreamMatchCopyKernel>(iBuilder, 8);
     198    mPxDriver.makeKernelCall(bitStreamMatchCopyK, {mMatchOffsetMarker, mM0Marker, mCompressedByteStream, depositedBits}, {matchCopiedBits});
     199
     200    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     201    mPxDriver.makeKernelCall(p2sK, {matchCopiedBits}, {DecompressedByteStream});
     202
     203    // --------------------------------------------------------
     204    // End
     205    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     206
     207    outK->setInitialArguments({iBuilder->GetString(outputFile)});
     208    mPxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     209
     210    mPxDriver.generatePipelineIR();
     211    mPxDriver.deallocateBuffers();
     212
     213    iBuilder->CreateRetVoid();
     214
     215    mPxDriver.finalizeObject();
     216}
     217
     218void LZ4Generator::generateSwizzledPipeline(const std::string &outputFile) {
    135219    auto & iBuilder = mPxDriver.getBuilder();
    136220    this->generateMainFunc(iBuilder);
     
    158242
    159243    // Produce unswizzled bit streams
    160     StreamSetBuffer * extractedbits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     244    StreamSetBuffer * matchCopiedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
    161245    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    162     mPxDriver.makeKernelCall(unSwizzleK, {matchCopiedSwizzle0, matchCopiedSwizzle1}, {extractedbits});
    163 
    164 
    165     Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
    166     mPxDriver.makeKernelCall(p2sK, {extractedbits}, {DecompressedByteStream});
     246    mPxDriver.makeKernelCall(unSwizzleK, {matchCopiedSwizzle0, matchCopiedSwizzle1}, {matchCopiedBits});
     247
     248
     249    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     250    mPxDriver.makeKernelCall(p2sK, {matchCopiedBits}, {DecompressedByteStream});
    167251
    168252    // --------------------------------------------------------
     
    304388}
    305389
     390parabix::StreamSetBuffer* LZ4Generator::generateBitStreamExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     391    StreamSetBuffer * const compressionMarker = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks());
     392    Kernel * bitstreamNotK = mPxDriver.addKernelInstance<LZ4BitStreamNotKernel>(iBuilder);
     393    mPxDriver.makeKernelCall(bitstreamNotK, {mDeletionMarker}, {compressionMarker});
     394
     395    // Deletion
     396    StreamSetBuffer * deletedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     397    StreamSetBuffer * deletionCounts = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     398
     399    Kernel * delK = mPxDriver.addKernelInstance<PEXTFieldCompressKernel>(iBuilder, 64, 8);
     400    mPxDriver.makeKernelCall(delK, {mCompressedBasisBits, compressionMarker}, {deletedBits, deletionCounts});
     401
     402    StreamSetBuffer * compressedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     403    Kernel * streamCompressionK = mPxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
     404    mPxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
     405
     406    return compressedBits;
     407}
     408
    306409int LZ4Generator::get4MbBufferBlocks() {
    307410    return 4 * 1024 * 1024 / codegen::BlockSize;
Note: See TracChangeset for help on using the changeset viewer.