Ignore:
Timestamp:
Jun 4, 2018, 3:20:10 AM (16 months ago)
Author:
xwa163
Message:

Init checkin for LZ4ParallelByteStreamAioKernel and related pipeline

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.cpp

    r6059 r6064  
    2525#include <kernels/lz4/lz4_index_builder_new.h>
    2626#include <kernels/lz4/lz4_bytestream_aio.h>
     27#include <kernels/lz4/lz4_parallel_bytestream_aio.h>
    2728#include <kernels/lz4/lz4_swizzled_aio.h>
    2829#include <kernels/bitstream_pdep_kernel.h>
     
    388389}
    389390
     391void LZ4Generator::generateLoadByteStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     392    mCompressedByteStream = mPxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
     393    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
     394    sourceK->setInitialArguments({mInputStream, mFileSize});
     395    mPxDriver.makeKernelCall(sourceK, {}, {mCompressedByteStream});
     396}
    390397void LZ4Generator::generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    391398    mCompressedByteStream = mPxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
     
    458465}
    459466
    460 
     467parabix::StreamSetBuffer * LZ4Generator::generateParallelAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     468    //// Decode Block Information
     469    StreamSetBuffer * const BlockData_IsCompressed = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(), 1);
     470    StreamSetBuffer * const BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(), 1);
     471    StreamSetBuffer * const BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(), 1);
     472
     473    //// Generate Helper Markers Extenders
     474    StreamSetBuffer * const Extenders = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks(), 1);
     475    mMatchOffsetMarker = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks());
     476    Kernel * extenderK = mPxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8);
     477    mPxDriver.makeKernelCall(extenderK, {mCompressedBasisBits}, {Extenders});
     478
     479    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZ4BlockDecoderNewKernel>(iBuilder);
     480    blockDecoderK->setInitialArguments({iBuilder->CreateTrunc(mHasBlockChecksum, iBuilder->getInt1Ty()), mHeaderSize, mFileSize});
     481    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_IsCompressed, BlockData_BlockStart, BlockData_BlockEnd});
     482
     483
     484    StreamSetBuffer * const decompressionByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks(), 1);
     485
     486    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4ParallelByteStreamAioKernel>(iBuilder);
     487    lz4AioK->setInitialArguments({mFileSize});
     488    mPxDriver.makeKernelCall(
     489            lz4AioK,
     490            {
     491                    mCompressedByteStream,
     492                    Extenders,
     493
     494                    // Block Data
     495                    BlockData_IsCompressed,
     496                    BlockData_BlockStart,
     497                    BlockData_BlockEnd
     498            }, {
     499                    decompressionByteStream
     500            });
     501
     502    return decompressionByteStream;
     503
     504}
    461505
    462506StreamSetBuffer * LZ4Generator::generateAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     
    466510    StreamSetBuffer * const BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(), 1);
    467511
    468     //// Generate Helper Markers Extenders, FX, XF
     512
     513    //// Generate Helper Markers Extenders
     514    /*
    469515    StreamSetBuffer * const Extenders = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks(), 1);
    470516    mMatchOffsetMarker = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks());
    471517    Kernel * extenderK = mPxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8);
    472518    mPxDriver.makeKernelCall(extenderK, {mCompressedBasisBits}, {Extenders});
     519    */
    473520
    474521
     
    478525
    479526
    480     StreamSetBuffer * const decompressionByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     527    StreamSetBuffer * const decompressionByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks(), 1);
    481528
    482529    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4ByteStreamAioKernel>(iBuilder);
     
    486533            {
    487534                    mCompressedByteStream,
    488                     Extenders,
     535//                    Extenders,
    489536
    490537                    // Block Data
     
    590637
    591638int LZ4Generator::get4MbBufferBlocks() {
    592     return 4 * 1024 * 1024 / codegen::BlockSize;
     639    return 4 * 1024 * 1024 / codegen::BlockSize * 4; // TODO * 4 here in the end is a workaround for parallel aio pipeline
    593640}
    594641
Note: See TracChangeset for help on using the changeset viewer.