Changeset 6137


Ignore:
Timestamp:
Aug 1, 2018, 12:22:09 AM (5 months ago)
Author:
xwa163
Message:
  1. LZ4 ScanMatch? pipeline
  2. Refactor LZ4 Generator
  3. Adjust some naming
Location:
icGREP/icgrep-devel/icgrep
Files:
8 added
6 deleted
8 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/CMakeLists.txt

    r6136 r6137  
    103103add_library(UCDlib UCD/CaseFolding.cpp utf8_encoder.cpp utf16_encoder.cpp UCD/ucd_compiler.cpp UCD/PropertyObjects.cpp UCD/resolve_properties.cpp)
    104104add_library(GrepEngine  ${GREP_CORE_SRC} grep/grep_engine.cpp kernels/cc_kernel.cpp kernels/cc_scan_kernel.cpp kernels/charclasses.cpp kernels/streams_merge.cpp kernels/until_n.cpp kernels/UCD_property_kernel.cpp kernels/grapheme_kernel.cpp)
    105 add_library(LZ4_Lib lz4FrameDecoder.cpp kernels/cc_kernel.cpp kernels/pdep_kernel.cpp lz4/LZ4Generator.cpp kernels/lz4/lz4_block_decoder.cpp kernels/bitstream_pdep_kernel.cpp kernels/bitstream_gather_pdep_kernel.cpp kernels/swizzled_multiple_pdep_kernel.cpp kernels/fake_stream_generating_kernel.cpp kernels/lz4/decompression/lz4_bytestream_decompression.cpp kernels/lz4/decompression/lz4_swizzled_decompression.cpp kernels/lz4/decompression/lz4_parallel_bytestream_decompression.cpp kernels/lz4/decompression/lz4_sequential_decompression_base.cpp kernels/lz4/decompression/lz4_bitstream_decompression.cpp kernels/lz4/decompression/lz4_twist_decompression.cpp kernels/lz4/twist_kernel.cpp kernels/lz4/untwist_kernel.cpp)
     105add_library(LZ4_Lib lz4/lz4_frame_decoder.cpp kernels/cc_kernel.cpp kernels/pdep_kernel.cpp lz4/lz4_decompression_generator.cpp kernels/lz4/lz4_block_decoder.cpp kernels/bitstream_pdep_kernel.cpp kernels/bitstream_gather_pdep_kernel.cpp kernels/swizzled_multiple_pdep_kernel.cpp kernels/fake_stream_generating_kernel.cpp kernels/lz4/decompression/lz4_bytestream_decompression.cpp kernels/lz4/decompression/lz4_swizzled_decompression.cpp kernels/lz4/decompression/lz4_parallel_bytestream_decompression.cpp kernels/lz4/decompression/lz4_sequential_decompression_base.cpp kernels/lz4/decompression/lz4_bitstream_decompression.cpp kernels/lz4/decompression/lz4_twist_decompression.cpp kernels/lz4/twist_kernel.cpp kernels/lz4/untwist_kernel.cpp lz4/lz4_base_generator.cpp)
    106106add_library(LZParabix_Lib lzparabix/LZParabixGenerator.cpp kernels/lzparabix/decoder/LZParabixBlockDecoder.cpp kernels/lzparabix/decoder/LZParabixAioBaseKernel.cpp lzparabix/LZParabixGrepGenerator.cpp kernels/fake_stream_generating_kernel.cpp kernels/lzparabix/encoder/LZParabixCompressionKernel.cpp kernels/lzparabix/decoder/LZParabixLiteralDecoderKernel.cpp kernels/lzparabix/decoder/LZParabixBitStreamAioKernel.cpp kernels/lzparabix/decoder/LZParabixSwizzledAioKernel.cpp)
    107107
     
    125125add_executable(wc wc.cpp util/file_select.cpp)
    126126add_executable(editd editd/editd.cpp editd/pattern_compiler.cpp editd/editdscan_kernel.cpp editd/editd_gpu_kernel.cpp editd/editd_cpu_kernel.cpp kernels/streams_merge.cpp kernels/cc_kernel.cpp)
    127 add_executable(lz4d lz4d.cpp lz4FrameDecoder.cpp kernels/cc_kernel.cpp kernels/lz4/lz4_index_decoder.cpp kernels/lz4/lz4_bytestream_decoder.cpp)
     127add_executable(lz4d lz4d.cpp lz4/lz4_frame_decoder.cpp kernels/cc_kernel.cpp kernels/lz4/lz4_index_decoder.cpp kernels/lz4/lz4_bytestream_decoder.cpp)
    128128add_executable(core combine/core.cpp combine/regexGen.cpp combine/stringGen.cpp combine/propGen.cpp combine/icgrep-test/icgrep-test.cpp grep_interface.cpp grep/grep_engine.cpp kernels/scanmatchgen.cpp kernels/u8u32_kernel.cpp kernels/delmask_kernel.cpp kernels/cc_kernel.cpp kernels/cc_scan_kernel.cpp kernels/charclasses.cpp kernels/linebreak_kernel.cpp kernels/streams_merge.cpp kernels/grep_kernel.cpp kernels/until_n.cpp)
    129129add_executable(character_deletion character_deletion.cpp kernels/cc_kernel.cpp)
    130130add_executable(character_deposit character_deposit.cpp kernels/cc_kernel.cpp kernels/pdep_kernel.cpp kernels/bitstream_pdep_kernel.cpp)
    131131add_executable(lz4_decoder lz4_decoder.cpp)
    132 add_executable(lz4_grep grep_interface.cpp util/file_select.cpp lz4_grep.cpp lz4/LZ4GrepGenerator.cpp)
     132add_executable(lz4_grep grep_interface.cpp util/file_select.cpp lz4_grep.cpp lz4/lz4_grep_generator.cpp)
    133133add_executable(lzparabix_decoder lzparabix_decoder.cpp)
    134134add_executable(lzparabix_grep lzparabix_grep.cpp grep_interface.cpp util/file_select.cpp)
    135 add_executable(lzparabix_compressor lzparabix_compressor.cpp lzparabix/LZParabixCompressorGenerator.cpp lzparabix/LZParabixCompressorGenerator.h)
     135add_executable(lzparabix_compressor lzparabix_compressor.cpp lzparabix/LZParabixCompressorGenerator.cpp)
    136136add_executable(idisa_test idisa_test.cpp)
    137137
  • icGREP/icgrep-devel/icgrep/lz4_decoder.cpp

    r6136 r6137  
    55 */
    66
    7 
    8 #include <llvm/IR/Module.h>
    9 #include <llvm/Linker/Linker.h>
    107#include <llvm/Support/PrettyStackTrace.h>
    11 #include <llvm/Support/Signals.h>
    12 #include <llvm/Support/ManagedStatic.h>
    13 #include <IR_Gen/idisa_target.h>
    14 #include <boost/filesystem.hpp>
    15 #include <boost/iostreams/device/mapped_file.hpp>
    16 
    17 #include <lz4FrameDecoder.h>
    18 #include <cc/cc_compiler.h>
    19 #include <toolchain/toolchain.h>
    20 #include <kernels/cc_kernel.h>
    21 #include <kernels/streamset.h>
    22 #include <kernels/s2p_kernel.h>
    23 #include <kernels/kernel_builder.h>
    24 #include <toolchain/cpudriver.h>
    25 #include <llvm/Support/raw_ostream.h>
    26 #include <iostream>
    27 #include <lz4/LZ4Generator.h>
     8#include <lz4/lz4_decompression_generator.h>
    289
    2910namespace re { class CC; }
    3011
    3112using namespace llvm;
    32 using namespace parabix;
    33 using namespace kernel;
    3413
    3514static cl::OptionCategory lz4dFlags("Command Flags", "lz4d options");
     
    3918
    4019int main(int argc, char *argv[]) {
    41     // This boilerplate provides convenient stack traces and clean LLVM exit
    42     // handling. It also initializes the built in support for convenient
    43     // command line option handling.
    4420    sys::PrintStackTraceOnErrorSignal(argv[0]);
    4521    llvm::PrettyStackTraceProgram X(argc, argv);
    4622    llvm_shutdown_obj shutdown;
    4723    codegen::ParseCommandLineOptions(argc, argv, {&lz4dFlags, codegen::codegen_flags()});
    48     std::string fileName = inputFile;
    49     LZ4FrameDecoder lz4Frame(fileName);
    50     if (!lz4Frame.isValid()) {
    51         llvm::errs() << "Invalid LZ4 file.\n";
    52         return -1;
    53     }
    5424
    55     if (boost::filesystem::exists(outputFile)) {
    56         if (overwriteOutput) {
    57             boost::filesystem::remove(outputFile);
    58         } else {
    59             llvm::errs() << outputFile + " existed. Use -f argument to overwrite.\n";
    60             return -1;
    61         }
    62     }
    63 
    64     boost::iostreams::mapped_file_source mappedFile;
    65     mappedFile.open(fileName, lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
    66     char *fileBuffer = const_cast<char *>(mappedFile.data());
    67 
    68     LZ4Generator g;
    69     g.generateDecompressionPipeline(outputFile);
    70 
    71     auto main = g.getMainFunc();
    72     main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
    73     mappedFile.close();
    74 
    75     return 0;
     25    LZ4DecompressionGenerator g;
     26    return g.decompress(std::move(inputFile), std::move(outputFile), overwriteOutput);
    7627}
  • icGREP/icgrep-devel/icgrep/lz4_grep.cpp

    r6136 r6137  
    1515#include <boost/iostreams/device/mapped_file.hpp>
    1616
    17 #include <lz4FrameDecoder.h>
     17#include <lz4/lz4_frame_decoder.h>
    1818#include <cc/cc_compiler.h>
    1919#include <toolchain/toolchain.h>
     
    2525
    2626#include <iostream>
    27 #include <lz4/LZ4GrepGenerator.h>
     27#include <lz4/lz4_grep_generator.h>
    2828
    2929
  • icGREP/icgrep-devel/icgrep/lz4d.cpp

    r6089 r6137  
    1717#include <boost/iostreams/device/mapped_file.hpp>
    1818
    19 #include <lz4FrameDecoder.h>
     19#include <lz4/lz4_frame_decoder.h>
    2020#include <cc/alphabet.h>
    2121#include <cc/cc_compiler.h>
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.cpp

    r6133 r6137  
    4545    this->generateLoadByteStreamAndBitStream(iBuilder);
    4646
    47     auto decompressedBitStream = this->generateFullBitStreamDecompression(iBuilder);
    48 //    auto decompressedBitStream = this->generateAioBitStreamDecompressoin(iBuilder, {mCompressedBasisBits})[0];
    49 
    50     auto decompressedByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
     47    auto uncompressedBitStream = this->generateFullBitStreamDecompression(iBuilder);
     48
     49    auto uncompressedByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
    5150    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder, cc::BitNumbering::BigEndian);
    52     mPxDriver.makeKernelCall(p2sK, {decompressedBitStream}, {decompressedByteStream});
     51    mPxDriver.makeKernelCall(p2sK, {uncompressedBitStream}, {uncompressedByteStream});
    5352
    5453    // --------------------------------------------------------
     
    5655    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
    5756    outK->setInitialArguments({iBuilder->GetString(outputFile)});
    58     mPxDriver.makeKernelCall(outK, {decompressedByteStream}, {});
     57    mPxDriver.makeKernelCall(outK, {uncompressedByteStream}, {});
    5958
    6059    mPxDriver.generatePipelineIR();
     
    111110    });
    112111
    113     StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getDecompressedBufferBlocks(iBuilder));
     112    StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1),
     113                                                                                       this->getUncompressedBufferBlocks(
     114                                                                                               iBuilder));
    114115
    115116    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
     
    177178void LZParabixGenerator::generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    178179    mCompressedByteStream = mPxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
    179 //    mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
     180//    mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getDefaultBufferBlocks(iBuilder));
    180181
    181182    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
     
    221222    return this->get4MbBufferBlocks() * 2;
    222223}
    223 int LZParabixGenerator::getDecompressedBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & b) {
     224int LZParabixGenerator::getUncompressedBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> &b) {
    224225    return this->get4MbBufferBlocks() * 2;
    225226}
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.h

    r6133 r6137  
    4040    // BufferSize related Helper Function
    4141    virtual int getInputBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    42     virtual int getDecompressedBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     42    virtual int getUncompressedBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> &iBuilder);
    4343    int get4MbBufferBlocks();
    4444
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGrepGenerator.cpp

    r6135 r6137  
    135135    }
    136136
    137 //    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
    138 //    outK->setInitialArguments({iBuilder->GetString("/Users/wxy325/developer/LZ4-sample-files/workspace/lz4d-normal/8k_.txt")});
    139 //    mPxDriver.makeKernelCall(outK, {decompressedStream}, {});
    140137
    141138    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
     
    202199
    203200
    204     StreamSetBuffer * decompressedCharClasses = nullptr;
     201    StreamSetBuffer * uncompressedCharClasses = nullptr;
    205202    StreamSetBuffer * u8NoFinalStream = nullptr;
    206203    StreamSetBuffer * fakeMatchCopiedBits = nullptr;
     
    210207    if (allCcByteLength) {
    211208        auto ret = this->generateBitStreamDecompression(idb, {CharClasses});
    212         decompressedCharClasses = ret[0];
     209        uncompressedCharClasses = ret[0];
    213210        fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
    214211        u8NoFinalStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1), this->getInputBufferBlocks(idb), 1);
    215212
    216213        Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, std::vector<unsigned>({8, 1}));
    217         mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits, u8NoFinalStream});
     214        mPxDriver.makeKernelCall(fakeStreamGeneratorK, {uncompressedCharClasses}, {fakeMatchCopiedBits, u8NoFinalStream});
    218215    } else {
    219216        re::RE* nonFinalName = re::makeAlt({re::makeByte(0xC2, 0xF4),
     
    226223        auto ret = this->generateBitStreamDecompression(idb, {CharClasses, compressedU8NoFinalStream});
    227224
    228         decompressedCharClasses = ret[0];
     225        uncompressedCharClasses = ret[0];
    229226        u8NoFinalStream = ret[1];
    230227
    231228        fakeMatchCopiedBits = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(8), this->getInputBufferBlocks(idb), 1);
    232229        Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, numOfCharacterClasses, 8);
    233         mPxDriver.makeKernelCall(fakeStreamGeneratorK, {decompressedCharClasses}, {fakeMatchCopiedBits});
     230        mPxDriver.makeKernelCall(fakeStreamGeneratorK, {uncompressedCharClasses}, {fakeMatchCopiedBits});
    234231
    235232    }
     
    238235    newLineBreak = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getInputBufferBlocks(idb));
    239236    kernel::Kernel * lineFeedGrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), linefeedCC), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
    240     mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, decompressedCharClasses}, {newLineBreak});
     237    mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, uncompressedCharClasses}, {newLineBreak});
    241238
    242239    externalStreamNames.push_back("UTF8_nonfinal");
    243240
    244241    kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mREs[0], externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
    245     mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, u8NoFinalStream, decompressedCharClasses}, {MatchResults});
     242    mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, u8NoFinalStream, uncompressedCharClasses}, {MatchResults});
    246243    MatchResultsBufs[0] = MatchResults;
    247244
     
    292289    this->generateBlockData(idb);
    293290    StreamSetBuffer * const LiteralBitStream = this->extractLiteralBitStream(idb);
    294 //    auto compressedLineBreakStream = this->linefeedStreamFromDecompressedBits(LiteralBitStream);
    295 
    296 
    297     StreamSetBuffer * decompressedBasisBits = nullptr;
     291//    auto compressedLineBreakStream = this->linefeedStreamFromUncompressedBits(LiteralBitStream);
     292
     293
     294    StreamSetBuffer * uncompressedBasisBits = nullptr;
    298295    if (swizzledDecompression) {
    299         decompressedBasisBits = this->generateSwizzledBitStreamDecompression(idb, LiteralBitStream);
     296        uncompressedBasisBits = this->generateSwizzledBitStreamDecompression(idb, LiteralBitStream);
    300297    } else {
    301298        auto ret = this->generateBitStreamDecompression(idb, {LiteralBitStream});
    302         decompressedBasisBits = ret[0];
    303     }
    304 
    305     StreamSetBuffer * LineBreakStream = this->linefeedStreamFromDecompressedBits(decompressedBasisBits);
     299        uncompressedBasisBits = ret[0];
     300    }
     301
     302    StreamSetBuffer * LineBreakStream = this->linefeedStreamFromUncompressedBits(uncompressedBasisBits);
    306303
    307304    std::map<std::string, StreamSetBuffer *> propertyStream;
     
    309306    for(unsigned i = 0; i < nREs; ++i) {
    310307        std::vector<std::string> externalStreamNames;
    311         std::vector<StreamSetBuffer *> icgrepInputSets = {decompressedBasisBits};
     308        std::vector<StreamSetBuffer *> icgrepInputSets = {uncompressedBasisBits};
    312309
    313310        std::set<re::Name *> UnicodeProperties;
     
    345342
    346343parabix::StreamSetBuffer *
    347 LZParabixGrepGenerator::linefeedStreamFromDecompressedBits(parabix::StreamSetBuffer *decompressedBasisBits) {
     344LZParabixGrepGenerator::linefeedStreamFromUncompressedBits(parabix::StreamSetBuffer *uncompressedBasisBits) {
    348345    auto & idb = mPxDriver.getBuilder();
    349346    const unsigned baseBufferSize = this->getInputBufferBlocks(idb);
    350347    StreamSetBuffer * LineFeedStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    351348    kernel::Kernel * linefeedK = mPxDriver.addKernelInstance<kernel::LineFeedKernelBuilder>(idb, Binding{idb->getStreamSetTy(8), "basis", FixedRate(), Principal()}, cc::BitNumbering::BigEndian);
    352     mPxDriver.makeKernelCall(linefeedK, {decompressedBasisBits}, {LineFeedStream});
     349    mPxDriver.makeKernelCall(linefeedK, {uncompressedBasisBits}, {LineFeedStream});
    353350    return LineFeedStream;
    354351}
  • icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGrepGenerator.h

    r6133 r6137  
    3939    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> multiplexingGrepPipeline(std::vector<re::RE *> &REs);
    4040
    41     parabix::StreamSetBuffer * linefeedStreamFromDecompressedBits(parabix::StreamSetBuffer *decompressedBasisBits);
     41    parabix::StreamSetBuffer * linefeedStreamFromUncompressedBits(parabix::StreamSetBuffer *uncompressedBasisBits);
    4242
    4343    void generateCountOnlyMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
Note: See TracChangeset for help on using the changeset viewer.