Ignore:
Timestamp:
Jun 28, 2018, 1:54:52 AM (9 months ago)
Author:
xwa163
Message:
  1. Add some BasisSetNumbering? option to fix bug of multiplexing
  2. Use BiigEndian? BitNumbering? for lz4 and lzparabix related pipeline
  3. Support multiplexing in LZ4BitStreamAio pipeline
File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.cpp

    r6118 r6119  
    402402    sourceK->setInitialArguments({mInputStream, mFileSize});
    403403    mPxDriver.makeKernelCall(sourceK, {}, {mCompressedByteStream});
    404     Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::LittleEndian);
     404    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::BigEndian);
    405405    mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
    406406}
    407407
    408408StreamSetBuffer * LZ4Generator::generateBitStreamAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    409 //// Decode Block Information
     409    return this->convertCompressedBitsStreamWithBitStreamAioApproach({mCompressedBasisBits}, "combined")[0];
     410}
     411
     412std::vector<StreamSetBuffer*> LZ4Generator::convertCompressedBitsStreamWithBitStreamAioApproach(
     413        std::vector<StreamSetBuffer*> compressedBitStreams, std::string prefix) {
     414    auto mGrepDriver = &mPxDriver;
     415    auto & iBuilder = mGrepDriver->getBuilder();
     416
     417    //// Decode Block Information
    410418    StreamSetBuffer * const BlockData_IsCompressed = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder), 1);
    411419    StreamSetBuffer * const BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
    412420    StreamSetBuffer * const BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
    413421
    414     //// Generate Helper Markers Extenders, FX, XF
    415 //    StreamSetBuffer * const Extenders = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks(iBuilder), 1);
    416 //    mMatchOffsetMarker = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks(iBuilder));
    417 //    Kernel * extenderK = mPxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8);
    418 //    mPxDriver.makeKernelCall(extenderK, {mCompressedBasisBits}, {Extenders});
    419 
    420422
    421423    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZ4BlockDecoderKernel>(iBuilder);
     
    423425    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_IsCompressed, BlockData_BlockStart, BlockData_BlockEnd});
    424426
    425     StreamSetBuffer* decompressedBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getDecompressedBufferBlocks(iBuilder));
    426 
    427     Kernel* lz4BitStreamAioK = mPxDriver.addKernelInstance<LZ4BitStreamAioKernel>(iBuilder);
    428     lz4BitStreamAioK->setInitialArguments({mFileSize});
    429     mPxDriver.makeKernelCall(
    430             lz4BitStreamAioK,
    431             {
    432                     mCompressedByteStream,
    433 
    434 //                    Extenders,
    435 
    436                     // Block Data
    437                     BlockData_IsCompressed,
    438                     BlockData_BlockStart,
    439                     BlockData_BlockEnd,
    440 
    441                     mCompressedBasisBits
    442             }, {
    443                     decompressedBitStream
    444             });
    445     return decompressedBitStream;
    446 
    447 }
     427
     428    std::vector<StreamSetBuffer *> inputStreams = {
     429            mCompressedByteStream,
     430
     431            // Block Data
     432            BlockData_IsCompressed,
     433            BlockData_BlockStart,
     434            BlockData_BlockEnd
     435    };
     436
     437    std::vector<StreamSetBuffer *> outputStream;
     438    std::vector<unsigned> numbersOfStreams;
     439
     440    for (unsigned i = 0; i < compressedBitStreams.size(); i++) {
     441        unsigned numOfStreams = compressedBitStreams[i]->getNumOfStreams();
     442        numbersOfStreams.push_back(numOfStreams);
     443        inputStreams.push_back(compressedBitStreams[i]);
     444        outputStream.push_back(mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(numOfStreams, 1), this->getInputBufferBlocks(iBuilder)));
     445    }
     446
     447    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4BitStreamAioKernel>(iBuilder, numbersOfStreams);
     448    lz4AioK->setInitialArguments({mFileSize});
     449    mPxDriver.makeKernelCall(lz4AioK, inputStreams, outputStream);
     450
     451    return outputStream;
     452}
     453
    448454
    449455StreamSetBuffer * LZ4Generator::generateSwizzledAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
Note: See TracChangeset for help on using the changeset viewer.