Ignore:
Timestamp:
Jul 23, 2018, 4:56:33 AM (7 months ago)
Author:
xwa163
Message:
  1. More experiment on lz4 grep
  2. Improve performance of lzparabix grep
File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.cpp

    r6119 r6132  
    2727#include <kernels/lz4/aio/lz4_swizzled_aio.h>
    2828#include <kernels/lz4/aio/lz4_bitstream_aio.h>
     29#include <kernels/lz4/aio/lz4_i4_bytestream_aio.h>
    2930#include <kernels/bitstream_pdep_kernel.h>
    3031#include <kernels/lz4/lz4_bitstream_not_kernel.h>
     
    426427
    427428
     429
     430    if (compressedBitStreams[0]->getNumOfStreams() == 4) {
     431        StreamSetBuffer* twistedCharClasses = mGrepDriver->addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 4), this->getInputBufferBlocks(iBuilder));
     432        kernel::Kernel* twistK = mGrepDriver->addKernelInstance<kernel::P2S4StreamByPDEP>(iBuilder);
     433        mGrepDriver->makeKernelCall(twistK, {compressedBitStreams[0]}, {twistedCharClasses});
     434
     435
     436        StreamSetBuffer* uncompressedTwistedCharClasses = mGrepDriver->addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 4), this->getInputBufferBlocks(iBuilder));
     437        Kernel* lz4I4AioK = mPxDriver.addKernelInstance<LZ4I4ByteStreamAioKernel>(iBuilder);
     438        lz4I4AioK->setInitialArguments({mFileSize});
     439        mGrepDriver->makeKernelCall(lz4I4AioK, {
     440                mCompressedByteStream,
     441
     442                // Block Data
     443                BlockData_IsCompressed,
     444                BlockData_BlockStart,
     445                BlockData_BlockEnd,
     446
     447                twistedCharClasses
     448        }, {
     449                                            uncompressedTwistedCharClasses
     450                                    });
     451
     452        StreamSetBuffer* untwistedCharClasses = mGrepDriver->addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4), this->getInputBufferBlocks(iBuilder));
     453        kernel::Kernel* untwistK = mGrepDriver->addKernelInstance<kernel::S2P4StreamByPEXTKernel>(iBuilder);
     454        mGrepDriver->makeKernelCall(untwistK, {uncompressedTwistedCharClasses}, {untwistedCharClasses});
     455        return {untwistedCharClasses};
     456    }
     457
     458
     459
     460
    428461    std::vector<StreamSetBuffer *> inputStreams = {
    429462            mCompressedByteStream,
     
    554587
    555588StreamSetBuffer * LZ4Generator::generateAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    556     //// Decode Block Information
    557     StreamSetBuffer * const BlockData_IsCompressed = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder), 1);
    558     StreamSetBuffer * const BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
    559     StreamSetBuffer * const BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
    560 
    561 
    562     //// Generate Helper Markers Extenders
    563 //    StreamSetBuffer * const Extenders = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks(iBuilder), 1);
    564 //    mMatchOffsetMarker = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks(iBuilder));
    565 //    Kernel * extenderK = mPxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8);
    566 //    mPxDriver.makeKernelCall(extenderK, {mCompressedBasisBits}, {Extenders});
    567 
    568 
    569     Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZ4BlockDecoderKernel>(iBuilder);
    570     blockDecoderK->setInitialArguments({iBuilder->CreateTrunc(mHasBlockChecksum, iBuilder->getInt1Ty()), mHeaderSize, mFileSize});
    571     mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_IsCompressed, BlockData_BlockStart, BlockData_BlockEnd});
    572 
     589    LZ4BlockInfo blockInfo = this->getBlockInfo(iBuilder);
    573590
    574591    StreamSetBuffer * const decompressionByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks(iBuilder), 1);
    575 
    576592    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4ByteStreamAioKernel>(iBuilder);
    577593    lz4AioK->setInitialArguments({mFileSize});
     
    580596            {
    581597                    mCompressedByteStream,
    582 //                    Extenders,
    583598
    584599                    // Block Data
    585                     BlockData_IsCompressed,
    586                     BlockData_BlockStart,
    587                     BlockData_BlockEnd
     600                    blockInfo.isCompress,
     601                    blockInfo.blockStart,
     602                    blockInfo.blockEnd
    588603            }, {
    589604                    decompressionByteStream
     
    694709}
    695710
    696 
     711LZ4BlockInfo LZ4Generator::getBlockInfo(const std::unique_ptr<kernel::KernelBuilder> & b) {
     712    LZ4BlockInfo blockInfo;
     713    blockInfo.isCompress = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 8), this->getInputBufferBlocks(b), 1);
     714    blockInfo.blockStart = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 64), this->getInputBufferBlocks(b), 1);
     715    blockInfo.blockEnd = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 64), this->getInputBufferBlocks(b), 1);
     716
     717    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZ4BlockDecoderKernel>(b);
     718    blockDecoderK->setInitialArguments({b->CreateTrunc(mHasBlockChecksum, b->getInt1Ty()), mHeaderSize, mFileSize});
     719    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {blockInfo.isCompress, blockInfo.blockStart, blockInfo.blockEnd});
     720
     721    return blockInfo;
     722}
    697723
    698724
Note: See TracChangeset for help on using the changeset viewer.