Changeset 6039


Ignore:
Timestamp:
May 14, 2018, 12:25:34 PM (7 days ago)
Author:
xwa163
Message:

Init checkin for bitstream version of lz4d match copy

Location:
icGREP/icgrep-devel
Files:
4 added
9 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/QA/lz4d_ext_dep/main.py

    r5948 r6039  
    1010failure_count = 0
    1111
     12# For bitstream approach, there are still some bugs in some special small file
     13# (For now, it will have bug in 29k.txt.lz4)
    1214test_options = [
    1315    # ('extract_only', '-extract-only', {'extract_only' : True}),
    1416    # ('extract_and_deposit', '-extract-and-deposit-only', {'extract_and_deposit_only': True}),
    15     ('normal', '', {}),
     17    ('normal', '-swizzled-decompression', {}),
    1618]
    1719
  • icGREP/icgrep-devel/icgrep/CMakeLists.txt

    r6029 r6039  
    102102add_library(UCDlib UCD/CaseFolding.cpp utf8_encoder.cpp utf16_encoder.cpp UCD/ucd_compiler.cpp UCD/PropertyObjects.cpp UCD/resolve_properties.cpp)
    103103add_library(GrepEngine  ${GREP_CORE_SRC} grep/grep_engine.cpp kernels/cc_kernel.cpp kernels/cc_scan_kernel.cpp kernels/charclasses.cpp kernels/streams_merge.cpp kernels/until_n.cpp kernels/UCD_property_kernel.cpp kernels/grapheme_kernel.cpp)
    104 add_library(LZ4_Lib lz4FrameDecoder.cpp kernels/cc_kernel.cpp kernels/lz4/lz4_deposit_uncompressed.cpp kernels/lz4/lz4_generate_deposit_stream.cpp kernels/pdep_kernel.cpp lz4/LZ4Generator.cpp kernels/lz4/lz4_block_decoder.cpp kernels/lz4/lz4_index_builder.cpp lz4/LZ4GrepEngine.cpp kernels/lz4/lz4_swizzled_match_copy_kernel.cpp kernels/bitstream_pdep_kernel.cpp kernels/swizzled_multiple_pdep_kernel.cpp)
     104add_library(LZ4_Lib lz4FrameDecoder.cpp kernels/cc_kernel.cpp kernels/lz4/lz4_deposit_uncompressed.cpp kernels/lz4/lz4_generate_deposit_stream.cpp kernels/pdep_kernel.cpp lz4/LZ4Generator.cpp kernels/lz4/lz4_block_decoder.cpp kernels/lz4/lz4_index_builder.cpp lz4/LZ4GrepEngine.cpp kernels/lz4/lz4_swizzled_match_copy_kernel.cpp kernels/bitstream_pdep_kernel.cpp kernels/swizzled_multiple_pdep_kernel.cpp kernels/lz4/lz4_bitstream_not_kernel.cpp kernels/lz4/lz4_bitstream_match_copy_kernel.cpp)
    105105
    106106
     
    126126add_executable(core combine/core.cpp combine/regexGen.cpp combine/stringGen.cpp combine/propGen.cpp combine/icgrep-test/icgrep-test.cpp grep_interface.cpp grep/grep_engine.cpp kernels/scanmatchgen.cpp kernels/u8u32_kernel.cpp kernels/delmask_kernel.cpp kernels/cc_kernel.cpp kernels/cc_scan_kernel.cpp kernels/charclasses.cpp kernels/linebreak_kernel.cpp kernels/streams_merge.cpp kernels/grep_kernel.cpp kernels/until_n.cpp)
    127127add_executable(character_deletion character_deletion.cpp kernels/cc_kernel.cpp)
    128 add_executable(character_deposit character_deposit.cpp kernels/cc_kernel.cpp kernels/pdep_kernel.cpp)
     128add_executable(character_deposit character_deposit.cpp kernels/cc_kernel.cpp kernels/pdep_kernel.cpp kernels/bitstream_pdep_kernel.cpp)
    129129add_executable(lz4d_ext_dep lz4d_ext_dep.cpp)
    130130add_executable(lz4_grep grep_interface.cpp util/file_select.cpp lz4_grep.cpp lz4/LZ4GrepGenerator.cpp)
  • icGREP/icgrep-devel/icgrep/kernels/lz4/lz4_index_builder.cpp

    r6029 r6039  
    1414
    1515namespace kernel{
     16
     17    // TODO IndexBuilderKernel is responsible to clear the output buffer for final produced block
     18    // e.g. when produce item count is 0x120, IndexBuilderKernel needs to set 0x121 ~ 0x200 to 0
    1619
    1720    LZ4IndexBuilderKernel::LZ4IndexBuilderKernel(const std::unique_ptr<kernel::KernelBuilder> &b)
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.cpp

    r6029 r6039  
    2020#include <kernels/swizzled_multiple_pdep_kernel.h>
    2121#include <kernels/lz4/lz4_swizzled_match_copy_kernel.h>
     22#include <kernels/lz4/lz4_bitstream_match_copy_kernel.h>
    2223#include <kernels/lz4/lz4_block_decoder.h>
    2324#include <kernels/lz4/lz4_index_builder.h>
    2425#include <kernels/bitstream_pdep_kernel.h>
     26#include <kernels/lz4/lz4_bitstream_not_kernel.h>
    2527
    2628namespace re { class CC; }
     
    3840}
    3941
    40 
    41 
    42 void LZ4Generator::generateExtractOnlyPipeline(const std::string& outputFile) {
    43     auto & iBuilder = mPxDriver.getBuilder();
    44     this->generateMainFunc(iBuilder);
    45 
    46     StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
    47 
    48     // GeneratePipeline
    49     this->generateLoadByteStreamAndBitStream(iBuilder);
    50 
    51 
    52     this->generateExtractAndDepositMarkers(iBuilder);
    53 
    54 
     42void LZ4Generator::generateExtractOnlyPipeline(const std::string &outputFile) {
     43    auto & iBuilder = mPxDriver.getBuilder();
     44    this->generateMainFunc(iBuilder);
     45
     46    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     47
     48    // GeneratePipeline
     49    this->generateLoadByteStreamAndBitStream(iBuilder);
     50    this->generateExtractAndDepositMarkers(iBuilder);
     51    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
     52
     53    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     54    mPxDriver.makeKernelCall(p2sK, {extractedBits}, {DecompressedByteStream});
     55
     56    // --------------------------------------------------------
     57    // End
     58    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     59
     60    outK->setInitialArguments({iBuilder->GetString(outputFile)});
     61    mPxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     62
     63    mPxDriver.generatePipelineIR();
     64    mPxDriver.deallocateBuffers();
     65
     66    iBuilder->CreateRetVoid();
     67
     68    mPxDriver.finalizeObject();
     69}
     70
     71void LZ4Generator::generateSwizzledExtractOnlyPipeline(const std::string &outputFile) {
     72    auto & iBuilder = mPxDriver.getBuilder();
     73    this->generateMainFunc(iBuilder);
     74
     75    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     76
     77    // GeneratePipeline
     78    this->generateLoadByteStreamAndBitStream(iBuilder);
     79    this->generateExtractAndDepositMarkers(iBuilder);
    5580    auto swizzle = this->generateSwizzleExtractData(iBuilder);
    5681
     
    90115    this->generateLoadByteStreamAndBitStream(iBuilder);
    91116    this->generateExtractAndDepositMarkers(iBuilder);
    92 
    93     auto swizzle = this->generateSwizzleExtractData(iBuilder); // TODO: use compression kernel instead
    94 
    95     StreamSetBuffer * extractedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
    96     Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    97     mPxDriver.makeKernelCall(unSwizzleK, {swizzle.first, swizzle.second}, {extractedBits});
     117    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
    98118
    99119    StreamSetBuffer * depositedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getDecompressedBufferBlocks());
     
    101121    mPxDriver.makeKernelCall(bitStreamPDEPk, {mDepositMarker, extractedBits}, {depositedBits});
    102122
    103     /*
     123    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     124    mPxDriver.makeKernelCall(p2sK, {depositedBits}, {DecompressedByteStream});
     125
     126    // --------------------------------------------------------
     127    // End
     128    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     129
     130    outK->setInitialArguments({iBuilder->GetString(outputFile)});
     131    mPxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     132
     133    mPxDriver.generatePipelineIR();
     134    mPxDriver.deallocateBuffers();
     135
     136    iBuilder->CreateRetVoid();
     137
     138    mPxDriver.finalizeObject();
     139}
     140
     141void LZ4Generator::generateSwizzledExtractAndDepositOnlyPipeline(const std::string &outputFile) {
     142    auto & iBuilder = mPxDriver.getBuilder();
     143    this->generateMainFunc(iBuilder);
     144
     145    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     146
     147    // GeneratePipeline
     148    this->generateLoadByteStreamAndBitStream(iBuilder);
     149    this->generateExtractAndDepositMarkers(iBuilder);
     150
     151    auto swizzle = this->generateSwizzleExtractData(iBuilder);
    104152
    105153    StreamSetBuffer * depositedSwizzle0 = mPxDriver.addBuffer<CircularCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), this->getInputBufferBlocks(), 1);
     
    113161    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    114162    mPxDriver.makeKernelCall(unSwizzleK, {depositedSwizzle0, depositedSwizzle1}, {depositedBits});
    115      */
    116163
    117164    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     
    132179}
    133180
    134 void LZ4Generator::generatePipeline(const std::string& outputFile) {
     181void LZ4Generator::generatePipeline(const std::string &outputFile) {
     182    auto & iBuilder = mPxDriver.getBuilder();
     183    this->generateMainFunc(iBuilder);
     184
     185    StreamSetBuffer * const DecompressedByteStream = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getDecompressedBufferBlocks());
     186
     187    // GeneratePipeline
     188    this->generateLoadByteStreamAndBitStream(iBuilder);
     189    this->generateExtractAndDepositMarkers(iBuilder);
     190    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
     191
     192    StreamSetBuffer * depositedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getDecompressedBufferBlocks());
     193    Kernel * bitStreamPDEPk = mPxDriver.addKernelInstance<BitStreamPDEPKernel>(iBuilder, 8);
     194    mPxDriver.makeKernelCall(bitStreamPDEPk, {mDepositMarker, extractedBits}, {depositedBits});
     195
     196    StreamSetBuffer * matchCopiedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     197    Kernel * bitStreamMatchCopyK = mPxDriver.addKernelInstance<LZ4BitStreamMatchCopyKernel>(iBuilder, 8);
     198    mPxDriver.makeKernelCall(bitStreamMatchCopyK, {mMatchOffsetMarker, mM0Marker, mCompressedByteStream, depositedBits}, {matchCopiedBits});
     199
     200    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     201    mPxDriver.makeKernelCall(p2sK, {matchCopiedBits}, {DecompressedByteStream});
     202
     203    // --------------------------------------------------------
     204    // End
     205    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
     206
     207    outK->setInitialArguments({iBuilder->GetString(outputFile)});
     208    mPxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
     209
     210    mPxDriver.generatePipelineIR();
     211    mPxDriver.deallocateBuffers();
     212
     213    iBuilder->CreateRetVoid();
     214
     215    mPxDriver.finalizeObject();
     216}
     217
     218void LZ4Generator::generateSwizzledPipeline(const std::string &outputFile) {
    135219    auto & iBuilder = mPxDriver.getBuilder();
    136220    this->generateMainFunc(iBuilder);
     
    158242
    159243    // Produce unswizzled bit streams
    160     StreamSetBuffer * extractedbits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     244    StreamSetBuffer * matchCopiedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
    161245    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    162     mPxDriver.makeKernelCall(unSwizzleK, {matchCopiedSwizzle0, matchCopiedSwizzle1}, {extractedbits});
    163 
    164 
    165     Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
    166     mPxDriver.makeKernelCall(p2sK, {extractedbits}, {DecompressedByteStream});
     246    mPxDriver.makeKernelCall(unSwizzleK, {matchCopiedSwizzle0, matchCopiedSwizzle1}, {matchCopiedBits});
     247
     248
     249    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     250    mPxDriver.makeKernelCall(p2sK, {matchCopiedBits}, {DecompressedByteStream});
    167251
    168252    // --------------------------------------------------------
     
    304388}
    305389
     390parabix::StreamSetBuffer* LZ4Generator::generateBitStreamExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     391    StreamSetBuffer * const compressionMarker = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->getInputBufferBlocks());
     392    Kernel * bitstreamNotK = mPxDriver.addKernelInstance<LZ4BitStreamNotKernel>(iBuilder);
     393    mPxDriver.makeKernelCall(bitstreamNotK, {mDeletionMarker}, {compressionMarker});
     394
     395    // Deletion
     396    StreamSetBuffer * deletedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     397    StreamSetBuffer * deletionCounts = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     398
     399    Kernel * delK = mPxDriver.addKernelInstance<PEXTFieldCompressKernel>(iBuilder, 64, 8);
     400    mPxDriver.makeKernelCall(delK, {mCompressedBasisBits, compressionMarker}, {deletedBits, deletionCounts});
     401
     402    StreamSetBuffer * compressedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     403    Kernel * streamCompressionK = mPxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
     404    mPxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
     405
     406    return compressedBits;
     407}
     408
    306409int LZ4Generator::get4MbBufferBlocks() {
    307410    return 4 * 1024 * 1024 / codegen::BlockSize;
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.h

    r6026 r6039  
    2626    MainFunctionType getMainFunc();
    2727
    28     void generatePipeline(const std::string& outputFile);
    29     void generateExtractOnlyPipeline(const std::string& outputFile);
    30     void generateExtractAndDepositOnlyPipeline(const std::string& outputFile);
     28    void generatePipeline(const std::string &outputFile);
     29    void generateSwizzledPipeline(const std::string &outputFile);
     30    void generateExtractOnlyPipeline(const std::string &outputFile);
     31    void generateSwizzledExtractOnlyPipeline(const std::string &outputFile);
     32    void generateExtractAndDepositOnlyPipeline(const std::string &outputFile);
     33    void generateSwizzledExtractAndDepositOnlyPipeline(const std::string &outputFile);
    3134
    3235protected:
     
    3740    virtual void generateExtractAndDepositMarkers(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3841    virtual std::pair<parabix::StreamSetBuffer*, parabix::StreamSetBuffer*> generateSwizzleExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     42    virtual parabix::StreamSetBuffer* generateBitStreamExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3943
    4044    // BufferSize related Helper Function
  • icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.cpp

    r6026 r6039  
    2020#include <kernels/swizzled_multiple_pdep_kernel.h>
    2121#include <kernels/lz4/lz4_swizzled_match_copy_kernel.h>
     22#include <kernels/lz4/lz4_bitstream_match_copy_kernel.h>
     23#include <kernels/lz4/lz4_bitstream_not_kernel.h>
     24#include <kernels/bitstream_pdep_kernel.h>
    2225#include <re/re_toolchain.h>
    2326
     
    4649#include <llvm/Support/raw_ostream.h>
    4750#include <llvm/Support/Debug.h>
     51
    4852
    4953
     
    549553}
    550554
    551 
    552 
    553 
    554555void LZ4GrepGenerator::invokeScanMatchGrep(char* fileBuffer, size_t blockStart, size_t blockEnd, bool hasBlockChecksum) {
    555556    auto main = this->getScanMatchGrepMainFunction();
     
    585586    mPxDriver.makeKernelCall(swizzledMatchCopyK, {mMatchOffsetMarker, mM0Marker, mCompressedByteStream, depositedSwizzle0, depositedSwizzle1}, {matchCopiedSwizzle0, matchCopiedSwizzle1});
    586587
    587 
    588588    // Produce unswizzled bit streams
    589589    StreamSetBuffer * extractedbits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
    590590    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
    591591    mPxDriver.makeKernelCall(unSwizzleK, {matchCopiedSwizzle0, matchCopiedSwizzle1}, {extractedbits});
    592 
    593 
    594592
    595593    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
     
    615613}
    616614
    617 void LZ4GrepGenerator::generateCountOnlyGrepPipeline(re::RE* regex) {
     615void LZ4GrepGenerator::generateCountOnlyGrepPipeline(re::RE *regex) {
     616    auto & iBuilder = mPxDriver.getBuilder();
     617    this->generateMainFunc(iBuilder);
     618
     619    // GeneratePipeline
     620    this->generateLoadByteStreamAndBitStream(iBuilder);
     621    this->generateExtractAndDepositMarkers(iBuilder);
     622
     623    StreamSetBuffer * const extractedBits = this->generateBitStreamExtractData(iBuilder);
     624
     625    StreamSetBuffer * depositedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getDecompressedBufferBlocks());
     626    Kernel * bitStreamPDEPk = mPxDriver.addKernelInstance<BitStreamPDEPKernel>(iBuilder, 8);
     627    mPxDriver.makeKernelCall(bitStreamPDEPk, {mDepositMarker, extractedBits}, {depositedBits});
     628
     629    StreamSetBuffer * matchCopiedBits = mPxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), this->getInputBufferBlocks());
     630    Kernel * bitStreamMatchCopyK = mPxDriver.addKernelInstance<LZ4BitStreamMatchCopyKernel>(iBuilder, 8);
     631    mPxDriver.makeKernelCall(bitStreamMatchCopyK, {mMatchOffsetMarker, mM0Marker, mCompressedByteStream, depositedBits}, {matchCopiedBits});
     632
     633    StreamSetBuffer * LineBreakStream;
     634    StreamSetBuffer * Matches;
     635    std::vector<re::RE*> res = {regex};
     636    if (mEnableMultiplexing) {
     637        std::tie(LineBreakStream, Matches) = multiplexingGrepPipeline(res, matchCopiedBits);
     638    } else {
     639        std::tie(LineBreakStream, Matches) = grepPipeline(res, matchCopiedBits);
     640    };
     641
     642    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
     643    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
     644    mPxDriver.generatePipelineIR();
     645
     646    iBuilder->setKernel(matchCountK);
     647    Value * matchedLineCount = iBuilder->getAccumulator("countResult");
     648    matchedLineCount = iBuilder->CreateZExt(matchedLineCount, iBuilder->getInt64Ty());
     649    iBuilder->CallPrintInt("aaa", matchedLineCount);
     650
     651    mPxDriver.deallocateBuffers();
     652
     653    // TODO return matchedLineCount
     654//        idb->CreateRet(matchedLineCount);
     655
     656    iBuilder->CreateRetVoid();
     657
     658    mPxDriver.finalizeObject();
     659}
     660
     661
     662void LZ4GrepGenerator::generateSwizzledCountOnlyGrepPipeline(re::RE *regex) {
    618663    auto & iBuilder = mPxDriver.getBuilder();
    619664    this->generateMainFunc(iBuilder);
     
    662707    };
    663708
    664 
    665 
    666709    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
    667710    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
    668711    mPxDriver.generatePipelineIR();
    669 
    670712
    671713    iBuilder->setKernel(matchCountK);
     
    678720    // TODO return matchedLineCount
    679721//        idb->CreateRet(matchedLineCount);
    680 
    681722
    682723    iBuilder->CreateRetVoid();
  • icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.h

    r6026 r6039  
    1818public:
    1919    LZ4GrepGenerator(bool enableMultiplexing = false);
    20     void generateCountOnlyGrepPipeline(re::RE* regex);
     20    void generateSwizzledCountOnlyGrepPipeline(re::RE *regex);
     21    void generateCountOnlyGrepPipeline(re::RE *regex);
     22
    2123    void generateScanMatchGrepPipeline(re::RE* regex);
    2224    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(std::vector<re::RE *> &REs,
  • icGREP/icgrep-devel/icgrep/lz4_grep.cpp

    r6036 r6039  
    4848static cl::opt<bool> enableMultiplexing("enable-multiplexing", cl::desc("Enable CC multiplexing."), cl::init(false), cl::cat(lz4GrepFlags));
    4949
    50 
    51 //static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(lz4GrepFlags));
    52 //static cl::opt<bool> overwriteOutput("f", cl::desc("Overwrite existing output file."), cl::init(false), cl::cat(lz4GrepFlags));
    53 
    54 
    5550static cl::OptionCategory lz4GrepDebugFlags("LZ4 Grep Debug Flags", "lz4d debug options");
    56 //static cl::opt<bool> extractOnly("extract-only", cl::desc("Only extract literal data to output file"), cl::init(false), cl::cat(lz4GrepDebugFlags));
    57 //static cl::opt<bool> extractAndDepositOnly("extract-and-deposit-only", cl::desc("Only extract and deposit literal data to output file"), cl::init(false), cl::cat(lz4GrepDebugFlags));
     51static cl::opt<bool> swizzledDecompression("swizzled-decompression", cl::desc("Use swizzle approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
    5852
    5953
     
    8175    LZ4GrepGenerator g(enableMultiplexing);
    8276    if (countOnly) {
    83         g.generateCountOnlyGrepPipeline(re_ast);
     77        if (swizzledDecompression) {
     78            g.generateSwizzledCountOnlyGrepPipeline(re_ast);
     79        } else {
     80            g.generateCountOnlyGrepPipeline(re_ast);
     81        }
     82
    8483        auto main = g.getMainFunc();
    8584        main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
  • icGREP/icgrep-devel/icgrep/lz4d_ext_dep.cpp

    r6008 r6039  
    4242static cl::opt<bool> extractOnly("extract-only", cl::desc("Only extract literal data to output file"), cl::init(false), cl::cat(lz4dDebugFlags));
    4343static cl::opt<bool> extractAndDepositOnly("extract-and-deposit-only", cl::desc("Only extract and deposit literal data to output file"), cl::init(false), cl::cat(lz4dDebugFlags));
    44 static cl::opt<bool> newApproach("new-approach", cl::desc("Use new approach"), cl::init(false), cl::cat(lz4dDebugFlags));
     44static cl::opt<bool> swizzledDecompression("swizzled-decompression", cl::desc("Use swizzle approach for decompression"), cl::init(false), cl::cat(lz4dDebugFlags));
    4545
    4646
     
    7676    LZ4Generator g;
    7777    if (extractOnly) {
    78         g.generateExtractOnlyPipeline(outputFile);
     78        if (swizzledDecompression) {
     79            g.generateSwizzledExtractOnlyPipeline(outputFile);
     80        } else {
     81            g.generateExtractOnlyPipeline(outputFile);
     82        }
    7983    } else if (extractAndDepositOnly) {
    80         g.generateExtractAndDepositOnlyPipeline(outputFile);
     84        if (swizzledDecompression) {
     85            g.generateSwizzledExtractAndDepositOnlyPipeline(outputFile);
     86        } else {
     87            g.generateExtractAndDepositOnlyPipeline(outputFile);
     88        }
    8189    } else {
    82         g.generatePipeline(outputFile);
     90        if (swizzledDecompression) {
     91            g.generateSwizzledPipeline(outputFile);
     92        } else {
     93            g.generatePipeline(outputFile);
     94        }
     95
    8396    }
    8497    auto main = g.getMainFunc();
Note: See TracChangeset for help on using the changeset viewer.