Changeset 6123


Ignore:
Timestamp:
Jul 1, 2018, 3:53:34 PM (2 weeks ago)
Author:
xwa163
Message:

Encode BitStream? directly in LZParabix compressed file

Location:
icGREP/icgrep-devel/icgrep
Files:
10 added
6 deleted
5 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/CMakeLists.txt

    r6118 r6123  
    104104add_library(GrepEngine  ${GREP_CORE_SRC} grep/grep_engine.cpp kernels/cc_kernel.cpp kernels/cc_scan_kernel.cpp kernels/charclasses.cpp kernels/streams_merge.cpp kernels/until_n.cpp kernels/UCD_property_kernel.cpp kernels/grapheme_kernel.cpp)
    105105add_library(LZ4_Lib lz4FrameDecoder.cpp kernels/cc_kernel.cpp kernels/lz4/lz4_deposit_uncompressed.cpp kernels/lz4/lz4_generate_deposit_stream.cpp kernels/pdep_kernel.cpp lz4/LZ4Generator.cpp kernels/lz4/lz4_block_decoder.cpp kernels/lz4/lz4_index_builder.cpp kernels/lz4/lz4_swizzled_match_copy_kernel.cpp kernels/bitstream_pdep_kernel.cpp kernels/bitstream_gather_pdep_kernel.cpp kernels/swizzled_multiple_pdep_kernel.cpp kernels/lz4/lz4_bitstream_not_kernel.cpp kernels/lz4/lz4_bitstream_match_copy_kernel.cpp kernels/fake_stream_generating_kernel.cpp kernels/lz4/aio/lz4_bytestream_aio.cpp kernels/lz4/aio/lz4_swizzled_aio.cpp kernels/lz4/aio/lz4_parallel_bytestream_aio.cpp kernels/lz4/aio/lz4_sequential_aio_base.cpp kernels/lz4/aio/lz4_sequential_aio_base.h kernels/lz4/aio/lz4_bitstream_aio.cpp kernels/lz4/aio/lz4_bitstream_aio.h)
    106 add_library(LZParabix_Lib lzparabix/LZParabixGenerator.cpp kernels/lzparabix/LZParabixBlockDecoder.cpp kernels/lzparabix/LZParabixBlockDecoder.h kernels/lzparabix/LZParabixAioKernel.cpp kernels/lzparabix/LZParabixAioKernel.h lzparabix/LZParabixGrepGenerator.cpp lzparabix/LZParabixGrepGenerator.h kernels/fake_stream_generating_kernel.cpp kernels/lzparabix/LZParabixCompressionKernel.cpp kernels/lzparabix/LZParabixCompressionKernel.h)
     106add_library(LZParabix_Lib lzparabix/LZParabixGenerator.cpp kernels/lzparabix/decoder/LZParabixBlockDecoder.cpp kernels/lzparabix/decoder/LZParabixBlockDecoder.h kernels/lzparabix/decoder/LZParabixAioKernel.cpp kernels/lzparabix/decoder/LZParabixAioKernel.h lzparabix/LZParabixGrepGenerator.cpp lzparabix/LZParabixGrepGenerator.h kernels/fake_stream_generating_kernel.cpp kernels/lzparabix/encoder/LZParabixCompressionKernel.cpp kernels/lzparabix/encoder/LZParabixCompressionKernel.h kernels/lzparabix/decoder/LZParabixLiteralDecoderKernel.cpp kernels/lzparabix/decoder/LZParabixLiteralDecoderKernel.h)
    107107
    108108# force the compiler to compile the object cache to ensure that the versioning information is up to date
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixCompressorGenerator.cpp

    r6114 r6123  
    1717#include <kernels/stdout_kernel.h>
    1818#include <kernels/kernel_builder.h>
    19 #include <kernels/lzparabix/LZParabixCompressionKernel.h>
     19#include <kernels/lzparabix/encoder/LZParabixCompressionKernel.h>
    2020
    2121
     
    4343
    4444    parabix::StreamSetBuffer* outputStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
     45
     46    parabix::StreamSetBuffer* literalByteBuffer = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
     47    parabix::StreamSetBuffer* tokenByteBuffer = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
     48
     49    parabix::StreamSetBuffer* strToBlockIndex = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), HASHNBCELLS4);
     50    parabix::StreamSetBuffer* strToMatchPos = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 32), HASHNBCELLS4);
     51
    4552    Kernel * compressionKernel = mPxDriver.addKernelInstance<LZParabixCompressionKernel>(iBuilder);
    4653    compressionKernel->setInitialArguments({mFileSize});
    47     mPxDriver.makeKernelCall(compressionKernel, {mCompressedByteStream}, {outputStream});
     54    mPxDriver.makeKernelCall(compressionKernel, {mCompressedByteStream, mCompressedBasisBits}, {outputStream, strToBlockIndex, strToMatchPos, literalByteBuffer, tokenByteBuffer});
    4855
    4956    // --------------------------------------------------------
     
    7077    sourceK->setInitialArguments({mInputStream, mFileSize});
    7178    mPxDriver.makeKernelCall(sourceK, {}, {mCompressedByteStream});
    72 //    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::LittleEndian);
    73 //    mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
     79    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::BigEndian);
     80    mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
    7481
    7582}
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.cpp

    r6119 r6123  
    2121#include <kernels/pdep_kernel.h>
    2222#include <kernels/swizzled_multiple_pdep_kernel.h>
    23 #include <kernels/lzparabix/LZParabixBlockDecoder.h>
    24 #include <kernels/lzparabix/LZParabixAioKernel.h>
     23#include <kernels/lzparabix/decoder/LZParabixBlockDecoder.h>
     24#include <kernels/lzparabix/decoder/LZParabixAioKernel.h>
     25#include <kernels/lzparabix/decoder/LZParabixLiteralDecoderKernel.h>
    2526
    2627namespace re { class CC; }
     
    4546
    4647    this->generateLoadByteStreamAndBitStream(iBuilder);
    47     auto decompressedBitStream = this->generateAioBitStreamDecompressoin(iBuilder, {mCompressedBasisBits})[0];
     48
     49    auto decompressedBitStream = this->generateFullBitStreamDecompression(iBuilder);
     50//    auto decompressedBitStream = this->generateAioBitStreamDecompressoin(iBuilder, {mCompressedBasisBits})[0];
    4851
    4952    auto decompressedByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
    50     Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     53    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder, cc::BitNumbering::BigEndian);
    5154    mPxDriver.makeKernelCall(p2sK, {decompressedBitStream}, {decompressedByteStream});
    5255
     
    6568}
    6669
     70
     71void LZParabixGenerator::generateBlockData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     72    BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder));
     73    BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder));
     74    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZParabixBlockDecoderKernel>(iBuilder);
     75    blockDecoderK->setInitialArguments({mFileSize});
     76    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_BlockStart, BlockData_BlockEnd});
     77}
     78
     79parabix::StreamSetBuffer* LZParabixGenerator::extractLiteralBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     80    StreamSetBuffer * const LiteralBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
     81    Kernel* literalDecoderK = mPxDriver.addKernelInstance<LZParabixLiteralDecoderKernel>(iBuilder);
     82    literalDecoderK->setInitialArguments({mFileSize});
     83    mPxDriver.makeKernelCall(literalDecoderK, {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd}, {LiteralBitStream});
     84    return LiteralBitStream;
     85}
     86
     87std::vector<parabix::StreamSetBuffer*> LZParabixGenerator::generateBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, std::vector<parabix::StreamSetBuffer*> inputBitStreams) {
     88    std::vector<unsigned> numbersOfStreams;
     89    std::vector<StreamSetBuffer*> inputStreams = {
     90            mCompressedByteStream,
     91            BlockData_BlockStart,
     92            BlockData_BlockEnd,
     93    };;
     94    std::vector<StreamSetBuffer*> outputStreams;
     95    for (unsigned i = 0; i < inputBitStreams.size(); i++) {
     96        unsigned numOfStream = inputBitStreams[i]->getNumOfStreams();
     97
     98        numbersOfStreams.push_back(numOfStream);
     99        inputStreams.push_back(inputBitStreams[i]);
     100        outputStreams.push_back(mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(numOfStream, 1), this->getInputBufferBlocks(iBuilder)));
     101    }
     102
     103    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixAioKernel>(iBuilder, numbersOfStreams);
     104    aioK->setInitialArguments({mFileSize});
     105    mPxDriver.makeKernelCall(aioK, inputStreams, outputStreams);
     106    return outputStreams;
     107
     108}
     109
     110parabix::StreamSetBuffer* LZParabixGenerator::generateFullBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & b) {
     111    this->generateBlockData(b);
     112    StreamSetBuffer * const LiteralBitStream = this->extractLiteralBitStream(b);
     113    return this->generateBitStreamDecompression(b, {LiteralBitStream})[0];
     114}
     115
     116
    67117std::vector<parabix::StreamSetBuffer*> LZParabixGenerator::generateAioBitStreamDecompressoin(
    68118        const std::unique_ptr<kernel::KernelBuilder> & iBuilder,
    69119        std::vector<parabix::StreamSetBuffer*> bitStreamSets
    70120) {
    71     StreamSetBuffer * const BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
    72     StreamSetBuffer * const BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
    73     Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZParabixBlockDecoderKernel>(iBuilder);
    74     blockDecoderK->setInitialArguments({mFileSize});
    75     mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_BlockStart, BlockData_BlockEnd});
     121    this->generateBlockData(iBuilder);
    76122
    77123    std::vector<parabix::StreamSetBuffer*> inputStreamSetParams = {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd};
     
    97143void LZParabixGenerator::generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    98144    mCompressedByteStream = mPxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
    99     mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
     145//    mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
    100146
    101147    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
    102148    sourceK->setInitialArguments({mInputStream, mFileSize});
    103149    mPxDriver.makeKernelCall(sourceK, {}, {mCompressedByteStream});
    104     Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::BigEndian);
    105     mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
     150//    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::BigEndian);
     151//    mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
    106152}
    107153
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.h

    r6114 r6123  
    3030    virtual void generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3131
     32
     33    parabix::StreamSetBuffer* generateFullBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3234    std::vector<parabix::StreamSetBuffer*> generateAioBitStreamDecompressoin(
    3335            const std::unique_ptr<kernel::KernelBuilder> & iBuilder,
     
    5759    parabix::StreamSetBuffer * mCompressedByteStream;
    5860    parabix::StreamSetBuffer * mCompressedBasisBits;
     61
     62    // Block Data
     63    void generateBlockData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     64    parabix::StreamSetBuffer * BlockData_BlockStart;
     65    parabix::StreamSetBuffer * BlockData_BlockEnd;
     66
     67    parabix::StreamSetBuffer* extractLiteralBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     68
     69    std::vector<parabix::StreamSetBuffer*> generateBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, std::vector<parabix::StreamSetBuffer*> inputBitStreams);
    5970};
    6071
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGrepGenerator.cpp

    r6119 r6123  
    176176    StreamSetBuffer * const MatchResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    177177
     178
     179    this->generateBlockData(idb);
     180    StreamSetBuffer * const LiteralBitStream = this->extractLiteralBitStream(idb);
     181
    178182    mpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSets);
    179183    mREs[0] = transformCCs(mpx.get(), mREs[0]);
     
    183187
    184188    kernel::Kernel * ccK = mGrepDriver->addKernelInstance<kernel::CharClassesKernel>(idb, std::move(mpx_basis), false, cc::BitNumbering::BigEndian);
    185     mGrepDriver->makeKernelCall(ccK, {mCompressedBasisBits}, {CharClasses});
     189    mGrepDriver->makeKernelCall(ccK, {LiteralBitStream}, {CharClasses});
    186190
    187191    StreamSetBuffer * CompressedLineFeedStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    188192    kernel::Kernel * linefeedK = mPxDriver.addKernelInstance<kernel::LineFeedKernelBuilder>(idb, Binding{idb->getStreamSetTy(8), "basis", FixedRate(), Principal()}, cc::BitNumbering::BigEndian);
    189     mPxDriver.makeKernelCall(linefeedK, {mCompressedBasisBits}, {CompressedLineFeedStream});
    190 
    191     auto ret = this->generateAioBitStreamDecompressoin(idb, {CharClasses, CompressedLineFeedStream});
     193    mPxDriver.makeKernelCall(linefeedK, {LiteralBitStream}, {CompressedLineFeedStream});
     194
     195    auto ret = this->generateBitStreamDecompression(idb, {CharClasses, CompressedLineFeedStream});
     196//    auto ret = this->generateAioBitStreamDecompressoin(idb, {CharClasses, CompressedLineFeedStream});
    192197
    193198    StreamSetBuffer * decompressedCharClasses = ret[0];
     
    245250    std::vector<StreamSetBuffer *> MatchResultsBufs(nREs);
    246251
    247     StreamSetBuffer * CompressedLineBreakStream = this->linefeedStreamFromDecompressedBits(mCompressedBasisBits);
    248     auto ret = this->generateAioBitStreamDecompressoin(idb, {mCompressedBasisBits, CompressedLineBreakStream});
     252
     253    this->generateBlockData(idb);
     254    StreamSetBuffer * const LiteralBitStream = this->extractLiteralBitStream(idb);
     255    auto compressedLineBreakStream = this->linefeedStreamFromDecompressedBits(LiteralBitStream);
     256
     257
     258    auto ret = this->generateBitStreamDecompression(idb, {LiteralBitStream, compressedLineBreakStream});
    249259    StreamSetBuffer * decompressedBasisBits = ret[0];
    250260    StreamSetBuffer * LineBreakStream = ret[1];
Note: See TracChangeset for help on using the changeset viewer.