Changeset 5921 for icGREP/icgrep-devel


Ignore:
Timestamp:
Mar 20, 2018, 4:05:54 PM (12 months ago)
Author:
xwa163
Message:
  1. Initial checkin for new approach for lz4 index decoder that always use 4MB buffer
  2. Add test case for new approach (for now test cases will fail when test file is larger than 4MB)
Location:
icGREP/icgrep-devel
Files:
6 added
9 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/QA/lz4d_ext_dep/main.py

    r5895 r5921  
    1414    ('extract_and_deposit', '-extract-and-deposit-only', {'extract_and_deposit_only': True}),
    1515    ('normal', '', {}),
     16    ('extract_only_new_approach', '-new-approach -extract-only', {'extract_only' : True}),
     17    ('extract_and_deposit_new_approach', '-new-approach -extract-and-deposit-only', {'extract_and_deposit_only': True}),
     18    ('normal_new_approach', '-new-approach', {}),
    1619]
     20
    1721
    1822def run_test(test_file, lz4_option, python_lz4_option, test_file_full_path, output_file_full_path, python_output_file_full_path):
  • icGREP/icgrep-devel/icgrep/CMakeLists.txt

    r5906 r5921  
    100100add_library(UCDlib UCD/CaseFolding.cpp utf8_encoder.cpp utf16_encoder.cpp UCD/ucd_compiler.cpp UCD/PropertyObjects.cpp UCD/resolve_properties.cpp)
    101101add_library(GrepEngine  ${GREP_CORE_SRC} grep/grep_engine.cpp kernels/cc_kernel.cpp kernels/cc_scan_kernel.cpp kernels/charclasses.cpp kernels/streams_merge.cpp kernels/until_n.cpp kernels/UCD_property_kernel.cpp kernels/grapheme_kernel.cpp)
     102add_library(LZ4_Lib lz4FrameDecoder.cpp kernels/cc_kernel.cpp kernels/lz4/lz4_block_decoder.cpp kernels/sequential_kernel.cpp kernels/lz4/lz4_extract_e_m0.cpp kernels/lz4/lz4_deposit_uncompressed.cpp kernels/lz4/lz4_generate_deposit_stream.cpp kernels/lz4/lz4_numbers_to_bitstream_kernel.cpp kernels/lz4/LZ4MarkerToMaskKernel.cpp kernels/lz4/lz4_bitstream_not_kernel.cpp kernels/pdep_kernel.cpp kernels/lz4/lz4_match_copy_kernel.cpp lz4/LZ4Generator.cpp kernels/lz4/lz4_multiple_pdep_kernel.cpp kernels/lz4/lz4_swizzled_match_copy_kernel.cpp kernels/lz4/lz4_block_decoder_new.cpp kernels/lz4/lz4_index_builder.cpp lz4/LZ4GeneratorNew.cpp)
    102103
    103104# force the compiler to compile the object cache to ensure that the versioning information is up to date
     
    123124add_executable(character_deletion character_deletion.cpp kernels/cc_kernel.cpp)
    124125add_executable(character_deposit character_deposit.cpp kernels/cc_kernel.cpp kernels/pdep_kernel.cpp kernels/lz4/lz4_multiple_pdep_kernel.cpp)
    125 add_executable(lz4d_ext_dep lz4d_ext_dep.cpp lz4FrameDecoder.cpp kernels/cc_kernel.cpp kernels/lz4/lz4_block_decoder.cpp kernels/sequential_kernel.cpp kernels/lz4/lz4_extract_e_m0.cpp kernels/lz4/lz4_deposit_uncompressed.cpp kernels/lz4/lz4_generate_deposit_stream.cpp kernels/lz4/lz4_numbers_to_bitstream_kernel.cpp kernels/lz4/LZ4MarkerToMaskKernel.cpp kernels/lz4/lz4_bitstream_not_kernel.cpp kernels/pdep_kernel.cpp kernels/lz4/lz4_match_copy_kernel.cpp lz4/LZ4Generator.cpp kernels/lz4/lz4_multiple_pdep_kernel.cpp kernels/lz4/lz4_swizzled_match_copy_kernel.cpp)
     126add_executable(lz4d_ext_dep lz4d_ext_dep.cpp)
     127
    126128
    127129#Exclude CoRE from default build.
     
    137139target_link_libraries (character_deletion PabloADT RegExpCompiler CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
    138140target_link_libraries (character_deposit PabloADT RegExpCompiler CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
    139 target_link_libraries (lz4d_ext_dep PabloADT RegExpCompiler CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
     141target_link_libraries (lz4d_ext_dep LZ4_Lib PabloADT CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
     142
    140143
    141144
  • icGREP/icgrep-devel/icgrep/kernels/attributes.h

    r5883 r5921  
    140140
    141141        // Whether the input streamset is in swizzled form
     142
     143        ConstantStrideLengthOne,
     144
     145        // TODO: Workaround here, the Pack Size is always one
    142146
    143147
     
    275279    friend Attribute ConditionalRegionEnd();
    276280    friend Attribute Swizzled();
     281    friend Attribute ConstantStrideLengthOne();
    277282    friend Attribute CanTerminateEarly();
    278283    friend Attribute MustExplicitlyTerminate();
     
    381386}
    382387
     388inline Attribute ConstantStrideLengthOne() {
     389    return Attribute(Attribute::KindId::ConstantStrideLengthOne, 0);
     390}
     391
     392
     393
    383394}
    384395
  • icGREP/icgrep-devel/icgrep/kernels/interface.h

    r5873 r5921  
    7272    bool isSwizzled() const {
    7373        return hasAttribute(AttributeId::Swizzled);
     74    }
     75
     76    bool isConstantStrideLengthOne() const {
     77        return hasAttribute(AttributeId::ConstantStrideLengthOne);
    7478    }
    7579
  • icGREP/icgrep-devel/icgrep/kernels/lz4/lz4_extract_e_m0.cpp

    r5905 r5921  
    66#include <iostream>
    77
    8 #define APPLY_64PACK_ACCELERATION
     8//#define APPLY_64PACK_ACCELERATION
    99// TODO May be we can change it to 256 PACK Acceleration based on SIMD instruction
    1010
     
    423423//    iBuilder->CallPrintInt("tokenPos", this->getCursorValue(iBuilder, "extender"));
    424424    Value* token = this->generateLoadSourceInputByte(iBuilder, "byteStream", this->getCursorValue(iBuilder, "extender"));
     425
     426    iBuilder->CallPrintInt("tokenPos", this->getCursorValue(iBuilder, "extender"));
    425427//    iBuilder->CallPrintInt("token", token);
    426 //    iBuilder->CallPrintInt("tokenPos", this->getCursorValue(iBuilder, "extender"));
    427428
    428429//    iBuilder->CreateAssert(iBuilder->CreateICmpULT(this->getCursorValue(iBuilder, "extender"), iBuilder->getSize(0xcb32a)), "ee");
     
    451452    Value* literalLengthEndPos = this->getCursorValue(iBuilder, "extender");
    452453    Value* literalExtensionSize = iBuilder->CreateSub(literalLengthEndPos, iBuilder->getScalarField("tokenPos"));
     454//    iBuilder->CallPrintInt("literalExtensionSize", literalExtensionSize);
    453455    Value* finalLengthByte = this->generateLoadSourceInputByte(iBuilder, "byteStream", this->getCursorValue(iBuilder, "extender"));
    454456
     
    471473
    472474
    473     Value* previousOffsetPos = iBuilder->getScalarField("offsetPos");
    474 
    475475    Value* offsetPos = iBuilder->CreateAdd(
    476476            iBuilder->CreateAdd(
     
    479479            iBuilder->getSize(1));
    480480    iBuilder->setScalarField("offsetPos", offsetPos);
     481//    iBuilder->CallPrintInt("offsetPos", offsetPos);
    481482//    iBuilder->CallPrintInt("literalStart", iBuilder->CreateAdd(literalLengthEndPos, iBuilder->getSize(1)));
    482483//    iBuilder->CallPrintInt("literalLength", literalLength);
     
    528529    matchLengthStartPos = iBuilder->getScalarField("matchLengthStartPos");
    529530    Value* oldMatchExtensionSize = iBuilder->CreateSub(this->getCursorValue(iBuilder, "extender"), matchLengthStartPos);
     531    iBuilder->CallPrintInt("aaa", oldMatchExtensionSize);
    530532
    531533    token = iBuilder->getScalarField("token");
     
    600602    this->increaseScalarField(iBuilder, "m0OutputPos", matchLength);
    601603    this->advanceCursor(iBuilder, "extender", iBuilder->getSize(1));
     604//    iBuilder->CallPrintInt("bbb", this->getCursorValue(iBuilder, "extender"));
    602605
    603606    iBuilder->CreateBr(compressedBlockLoopFinal);
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.cpp

    r5906 r5921  
    2020#include <kernels/lz4/lz4_generate_deposit_stream.h>
    2121#include <kernels/lz4/lz4_numbers_to_bitstream_kernel.h>
    22 //#include <kernels/LZ4MarkerToMaskKernel.h>
    2322#include <kernels/lz4/lz4_bitstream_not_kernel.h>
    2423#include <kernels/kernel_builder.h>
     
    3029#include <kernels/lz4/lz4_match_copy_kernel.h>
    3130#include <kernels/lz4/lz4_swizzled_match_copy_kernel.h>
     31#include <kernels/lz4/lz4_block_decoder_new.h>
     32#include <kernels/lz4/lz4_index_builder.h>
    3233
    3334namespace re { class CC; }
     
    113114
    114115//    pxDriver.makeKernelCall(unSwizzleK, {u16Swizzle0, u16Swizzle1}, {extractedbits});
    115 
    116     // TODO MatchCopy before p2s
    117116
    118117    Kernel * p2sK = pxDriver.addKernelInstance<P2SKernel>(iBuilder);
     
    170169//    pxDriver.makeKernelCall(unSwizzleK, {u16Swizzle0, u16Swizzle1}, {extractedbits});
    171170
    172     // TODO MatchCopy before p2s
    173171
    174172    Kernel * p2sK = pxDriver.addKernelInstance<P2SKernel>(iBuilder);
     
    361359
    362360
     361
    363362// Kernel Pipeline
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.h

    r5874 r5921  
    3030    void generateExtractAndDepositOnlyPipeline(const std::string& outputFile);
    3131
    32 private:
    33     //// Private Method
     32protected:
     33    //// Protected Method
    3434    inline void generateMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3535    // Pipeline
    36     inline void generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    37     inline void generateExtractAndDepositMarkers(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    38     inline std::pair<parabix::StreamSetBuffer*, parabix::StreamSetBuffer*> generateSwizzleExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     36    virtual void generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     37    virtual void generateExtractAndDepositMarkers(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     38    virtual std::pair<parabix::StreamSetBuffer*, parabix::StreamSetBuffer*> generateSwizzleExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3939
    4040    // BufferSize related Helper Function
    41     inline int getInputBufferBlocks();
    42     inline int getDecompressedBufferBlocks();
     41    virtual int getInputBufferBlocks();
     42    virtual int getDecompressedBufferBlocks();
     43
    4344
    4445    //// Data Member
  • icGREP/icgrep-devel/icgrep/lz4d_ext_dep.cpp

    r5874 r5921  
    2626#include <iostream>
    2727#include <lz4/LZ4Generator.h>
     28#include <lz4/LZ4GeneratorNew.h>
    2829
    2930namespace re { class CC; }
     
    4243static cl::opt<bool> extractOnly("extract-only", cl::desc("Only extract literal data to output file"), cl::init(false), cl::cat(lz4dDebugFlags));
    4344static cl::opt<bool> extractAndDepositOnly("extract-and-deposit-only", cl::desc("Only extract and deposit literal data to output file"), cl::init(false), cl::cat(lz4dDebugFlags));
     45static cl::opt<bool> newApproach("new-approach", cl::desc("Use new approach"), cl::init(false), cl::cat(lz4dDebugFlags));
    4446
    4547
     
    7981    }
    8082
    81     LZ4Generator g;
     83    std::unique_ptr<LZ4Generator> g = newApproach? llvm::make_unique<LZ4GeneratorNew>(): llvm::make_unique<LZ4Generator>();
    8284
    8385    if (extractOnly) {
    84         g.generateExtractOnlyPipeline(outputFile);
     86        g->generateExtractOnlyPipeline(outputFile);
    8587    } else if (extractAndDepositOnly) {
    86         g.generateExtractAndDepositOnlyPipeline(outputFile);
     88        g->generateExtractAndDepositOnlyPipeline(outputFile);
    8789    } else {
    88         g.generatePipeline(outputFile);
     90        g->generatePipeline(outputFile);
    8991    }
    9092
    91     auto main = g.getMainFunc();
     93    auto main = g->getMainFunc();
    9294
    9395    main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
  • icGREP/icgrep-devel/icgrep/toolchain/pipeline.cpp

    r5883 r5921  
    736736            }
    737737
     738            if (input.isConstantStrideLengthOne()) {
     739                // TODO workaround here
     740                strideLength = b->getSize(1);
     741            }
     742
    738743            Value * const processed = b->getProcessedItemCount(name);
    739744//            if (input.getRate().isFixed()) {
     
    749754          //  b->CallPrintInt("< " + kernel->getName() + "_" + name + "_unprocessed", unprocessed);
    750755
    751             Value * const hasSufficientData = b->CreateOr(b->CreateICmpUGT(unprocessed, strideLength), isFinal);
     756            Value * const hasSufficientData = input.isConstantStrideLengthOne() ?
     757                                              b->CreateOr(b->CreateICmpUGE(unprocessed, strideLength), isFinal) :
     758                                              b->CreateOr(b->CreateICmpUGT(unprocessed, strideLength), isFinal);
     759//            Value * const hasSufficientData = b->CreateOr(b->CreateICmpUGT(unprocessed, strideLength), isFinal);
     760//            Value * const hasSufficientData = b->CreateOr(b->CreateICmpUGE(unprocessed, strideLength), isFinal);
    752761
    753762          //  b->CallPrintInt("* < " + kernel->getName() + "_" + name + "_sufficientData", hasSufficientData);
Note: See TracChangeset for help on using the changeset viewer.