Ignore:
Timestamp:
Jul 1, 2018, 3:53:34 PM (13 months ago)
Author:
xwa163
Message:

Encode BitStream? directly in LZParabix compressed file

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.cpp

    r6119 r6123  
    2121#include <kernels/pdep_kernel.h>
    2222#include <kernels/swizzled_multiple_pdep_kernel.h>
    23 #include <kernels/lzparabix/LZParabixBlockDecoder.h>
    24 #include <kernels/lzparabix/LZParabixAioKernel.h>
     23#include <kernels/lzparabix/decoder/LZParabixBlockDecoder.h>
     24#include <kernels/lzparabix/decoder/LZParabixAioKernel.h>
     25#include <kernels/lzparabix/decoder/LZParabixLiteralDecoderKernel.h>
    2526
    2627namespace re { class CC; }
     
    4546
    4647    this->generateLoadByteStreamAndBitStream(iBuilder);
    47     auto decompressedBitStream = this->generateAioBitStreamDecompressoin(iBuilder, {mCompressedBasisBits})[0];
     48
     49    auto decompressedBitStream = this->generateFullBitStreamDecompression(iBuilder);
     50//    auto decompressedBitStream = this->generateAioBitStreamDecompressoin(iBuilder, {mCompressedBasisBits})[0];
    4851
    4952    auto decompressedByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
    50     Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     53    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder, cc::BitNumbering::BigEndian);
    5154    mPxDriver.makeKernelCall(p2sK, {decompressedBitStream}, {decompressedByteStream});
    5255
     
    6568}
    6669
     70
     71void LZParabixGenerator::generateBlockData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     72    BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder));
     73    BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder));
     74    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZParabixBlockDecoderKernel>(iBuilder);
     75    blockDecoderK->setInitialArguments({mFileSize});
     76    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_BlockStart, BlockData_BlockEnd});
     77}
     78
     79parabix::StreamSetBuffer* LZParabixGenerator::extractLiteralBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     80    StreamSetBuffer * const LiteralBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
     81    Kernel* literalDecoderK = mPxDriver.addKernelInstance<LZParabixLiteralDecoderKernel>(iBuilder);
     82    literalDecoderK->setInitialArguments({mFileSize});
     83    mPxDriver.makeKernelCall(literalDecoderK, {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd}, {LiteralBitStream});
     84    return LiteralBitStream;
     85}
     86
     87std::vector<parabix::StreamSetBuffer*> LZParabixGenerator::generateBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, std::vector<parabix::StreamSetBuffer*> inputBitStreams) {
     88    std::vector<unsigned> numbersOfStreams;
     89    std::vector<StreamSetBuffer*> inputStreams = {
     90            mCompressedByteStream,
     91            BlockData_BlockStart,
     92            BlockData_BlockEnd,
     93    };;
     94    std::vector<StreamSetBuffer*> outputStreams;
     95    for (unsigned i = 0; i < inputBitStreams.size(); i++) {
     96        unsigned numOfStream = inputBitStreams[i]->getNumOfStreams();
     97
     98        numbersOfStreams.push_back(numOfStream);
     99        inputStreams.push_back(inputBitStreams[i]);
     100        outputStreams.push_back(mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(numOfStream, 1), this->getInputBufferBlocks(iBuilder)));
     101    }
     102
     103    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixAioKernel>(iBuilder, numbersOfStreams);
     104    aioK->setInitialArguments({mFileSize});
     105    mPxDriver.makeKernelCall(aioK, inputStreams, outputStreams);
     106    return outputStreams;
     107
     108}
     109
     110parabix::StreamSetBuffer* LZParabixGenerator::generateFullBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & b) {
     111    this->generateBlockData(b);
     112    StreamSetBuffer * const LiteralBitStream = this->extractLiteralBitStream(b);
     113    return this->generateBitStreamDecompression(b, {LiteralBitStream})[0];
     114}
     115
     116
    67117std::vector<parabix::StreamSetBuffer*> LZParabixGenerator::generateAioBitStreamDecompressoin(
    68118        const std::unique_ptr<kernel::KernelBuilder> & iBuilder,
    69119        std::vector<parabix::StreamSetBuffer*> bitStreamSets
    70120) {
    71     StreamSetBuffer * const BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
    72     StreamSetBuffer * const BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
    73     Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZParabixBlockDecoderKernel>(iBuilder);
    74     blockDecoderK->setInitialArguments({mFileSize});
    75     mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_BlockStart, BlockData_BlockEnd});
     121    this->generateBlockData(iBuilder);
    76122
    77123    std::vector<parabix::StreamSetBuffer*> inputStreamSetParams = {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd};
     
    97143void LZParabixGenerator::generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    98144    mCompressedByteStream = mPxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
    99     mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
     145//    mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
    100146
    101147    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
    102148    sourceK->setInitialArguments({mInputStream, mFileSize});
    103149    mPxDriver.makeKernelCall(sourceK, {}, {mCompressedByteStream});
    104     Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::BigEndian);
    105     mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
     150//    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::BigEndian);
     151//    mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
    106152}
    107153
Note: See TracChangeset for help on using the changeset viewer.