Changeset 5591


Ignore:
Timestamp:
Aug 2, 2017, 9:01:23 PM (4 months ago)
Author:
lindanl
Message:

Editd: Bug fixed for buffer size (add assert). Some restructuring.

Location:
icGREP/icgrep-devel/icgrep
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/CMakeLists.txt

    r5579 r5591  
    107107add_executable(base64 base64.cpp kernels/radix64.cpp)
    108108add_executable(wc wc.cpp)
    109 add_executable(editd editd/editd.cpp editd/pattern_compiler.cpp editd/editdscan_kernel.cpp editd/editd_gpu_kernel.cpp editd/editd_cpu_kernel.cpp kernels/streams_merge.cpp)
     109add_executable(editd editd/editd.cpp editd/pattern_compiler.cpp editd/editdscan_kernel.cpp editd/editd_gpu_kernel.cpp editd/editd_cpu_kernel.cpp kernels/streams_merge.cpp kernels/cc_kernel.cpp)
    110110add_executable(array-test array-test.cpp kernels/alignedprint.cpp)
    111111add_executable(lz4d lz4d.cpp lz4FrameDecoder.cpp kernels/cc_kernel.cpp kernels/lz4_index_decoder.cpp kernels/lz4_bytestream_decoder.cpp)
  • icGREP/icgrep-devel/icgrep/editd/editd.cpp

    r5584 r5591  
    2121#include <kernels/source_kernel.h>
    2222#include <kernels/s2p_kernel.h>
     23#include <kernels/cc_kernel.h>
    2324#include <editd/editdscan_kernel.h>
    2425#include <kernels/streams_merge.h>
     
    240241
    241242    const unsigned segmentSize = codegen::SegmentSize;
    242     const unsigned bufferSegments = codegen::BufferSegments;
     243    const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
    243244
    244245    Function * const main = cast<Function>(m->getOrInsertFunction("Main", voidTy, inputType, sizeTy, nullptr));
     
    262263    auto editdScanK = pxDriver.addKernelInstance(make_unique<editdScanKernel>(idb, editDistance));
    263264    pxDriver.makeKernelCall(editdScanK, {MatchResults}, {});
    264 
    265     pxDriver.generatePipelineIR();
    266 
    267     idb->CreateRetVoid();
    268 
    269     pxDriver.finalizeObject();
    270 }
    271 
    272 
    273 void multiEditdPipeline(ParabixDriver & pxDriver) {
    274 
    275     auto & idb = pxDriver.getBuilder();
    276     Module * const m = idb->getModule();
    277     Type * const sizeTy = idb->getSizeTy();
    278     Type * const voidTy = idb->getVoidTy();
    279     Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(idb->getBitBlockType(), 8), 1), 0);
    280 
    281     idb->LinkFunction("wrapped_report_pos", &wrapped_report_pos);
    282 
    283     const unsigned segmentSize = codegen::SegmentSize;
    284     const unsigned bufferSegments = codegen::BufferSegments;
    285 
    286     Function * const main = cast<Function>(m->getOrInsertFunction("Main", voidTy, inputType, sizeTy, nullptr));
    287     main->setCallingConv(CallingConv::C);
    288     auto args = main->arg_begin();
    289     Value * const inputStream = &*(args++);
    290     inputStream->setName("input");
    291     Value * const fileSize = &*(args++);
    292     fileSize->setName("fileSize");
    293     idb->SetInsertPoint(BasicBlock::Create(m->getContext(), "entry", main,0));
    294 
    295     auto ChStream = pxDriver.addBuffer(make_unique<SourceBuffer>(idb, idb->getStreamSetTy(4)));
    296     auto mmapK = pxDriver.addKernelInstance(make_unique<MemorySourceKernel>(idb, inputType, segmentSize));
    297     mmapK->setInitialArguments({inputStream, fileSize});
    298     pxDriver.makeKernelCall(mmapK, {}, {ChStream});
    299 
    300     const auto n = pattGroups.size();
    301    
    302     std::vector<StreamSetBuffer *> MatchResultsBufs(n);
    303    
    304     for(unsigned i = 0; i < n; ++i){
    305         auto MatchResults = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(editDistance + 1), segmentSize * bufferSegments));
    306         auto editdk = pxDriver.addKernelInstance(make_unique<PatternKernel>(idb, pattGroups[i]));
    307         pxDriver.makeKernelCall(editdk, {ChStream}, {MatchResults});
    308         MatchResultsBufs[i] = MatchResults;
    309     }
    310     StreamSetBuffer * MergedResults = MatchResultsBufs[0];
    311     if (n > 1) {
    312         MergedResults = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(editDistance + 1), segmentSize * bufferSegments));
    313         kernel::Kernel * streamsMergeK = pxDriver.addKernelInstance(make_unique<kernel::StreamsMerge>(idb, editDistance + 1, n));
    314         pxDriver.makeKernelCall(streamsMergeK, MatchResultsBufs, {MergedResults});
    315     }
    316 
    317     auto editdScanK = pxDriver.addKernelInstance(make_unique<editdScanKernel>(idb, editDistance));
    318     pxDriver.makeKernelCall(editdScanK, {MergedResults}, {});
    319265
    320266    pxDriver.generatePipelineIR();
     
    363309
    364310    const unsigned segmentSize = codegen::SegmentSize;
    365     const unsigned bufferSegments = codegen::BufferSegments;
     311    const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
    366312
    367313    Function * const main = cast<Function>(m->getOrInsertFunction("Main", voidTy, int32Ty, outputType, nullptr));
     
    397343}
    398344
     345void multiEditdPipeline(ParabixDriver & pxDriver) {
     346    auto & idb = pxDriver.getBuilder();
     347    Module * const m = idb->getModule();
     348    Type * const voidTy = idb->getVoidTy();
     349    Type * const int32Ty = idb->getInt32Ty();
     350
     351    idb->LinkFunction("wrapped_report_pos", &wrapped_report_pos);
     352
     353    const unsigned segmentSize = codegen::SegmentSize;
     354    const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
     355
     356    Function * const main = cast<Function>(m->getOrInsertFunction("Main", voidTy, int32Ty, nullptr));
     357    main->setCallingConv(CallingConv::C);
     358    Function::arg_iterator args = main->arg_begin();
     359
     360    Value * const fileDescriptor = &*(args++);
     361    fileDescriptor->setName("fileDescriptor");
     362
     363    idb->SetInsertPoint(BasicBlock::Create(m->getContext(), "entry", main,0));
     364
     365    auto ByteStream = pxDriver.addBuffer(make_unique<SourceBuffer>(idb, idb->getStreamSetTy(1, 8)));
     366
     367    auto mmapK = pxDriver.addKernelInstance(make_unique<MMapSourceKernel>(idb, segmentSize));
     368    mmapK->setInitialArguments({fileDescriptor});
     369    pxDriver.makeKernelCall(mmapK, {}, {ByteStream});
     370
     371    auto ChStream = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(4), segmentSize * bufferSegments));
     372    auto ccck = pxDriver.addKernelInstance(make_unique<kernel::DirectCharacterClassKernelBuilder>(idb, "ccc",
     373        std::vector<re::CC *>{re::makeCC(re::makeCC(0x41), re::makeCC(0x61)),
     374                              re::makeCC(re::makeCC(0x43), re::makeCC(0x63)),
     375                              re::makeCC(re::makeCC(0x54), re::makeCC(0x74)),
     376                              re::makeCC(re::makeCC(0x47), re::makeCC(0x67))}, 1));
     377    pxDriver.makeKernelCall(ccck, {ByteStream}, {ChStream});
     378
     379    const auto n = pattGroups.size();
     380   
     381    std::vector<StreamSetBuffer *> MatchResultsBufs(n);
     382   
     383    for(unsigned i = 0; i < n; ++i){
     384        auto MatchResults = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(editDistance + 1), segmentSize * bufferSegments));
     385        auto editdk = pxDriver.addKernelInstance(make_unique<PatternKernel>(idb, pattGroups[i]));
     386        pxDriver.makeKernelCall(editdk, {ChStream}, {MatchResults});
     387        MatchResultsBufs[i] = MatchResults;
     388    }
     389    StreamSetBuffer * MergedResults = MatchResultsBufs[0];
     390    if (n > 1) {
     391        MergedResults = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(editDistance + 1), segmentSize * bufferSegments));
     392        kernel::Kernel * streamsMergeK = pxDriver.addKernelInstance(make_unique<kernel::StreamsMerge>(idb, editDistance + 1, n));
     393        pxDriver.makeKernelCall(streamsMergeK, MatchResultsBufs, {MergedResults});
     394    }
     395
     396    auto editdScanK = pxDriver.addKernelInstance(make_unique<editdScanKernel>(idb, editDistance));
     397    pxDriver.makeKernelCall(editdScanK, {MergedResults}, {});
     398
     399    pxDriver.generatePipelineIR();
     400
     401    idb->CreateRetVoid();
     402
     403    pxDriver.finalizeObject();
     404}
    399405
    400406typedef void (*preprocessFunctionType)(const int fd, char * output_data);
    401407
    402408typedef void (*editdFunctionType)(char * byte_data, size_t filesize);
     409
     410typedef void (*multiEditdFunctionType)(const int fd);
    403411
    404412static char * chStream;
     
    640648    get_editd_pattern(pattern_segs, total_len);
    641649
     650    if (MultiEditdKernels) {
     651        ParabixDriver pxDriver("editd");
     652        multiEditdPipeline(pxDriver);
     653        auto editd_ptr = reinterpret_cast<multiEditdFunctionType>(pxDriver.getMain());
     654
     655        std::string fileName = inputFiles[0];
     656        const int fd = open(inputFiles[0].c_str(), O_RDONLY);
     657        if (LLVM_UNLIKELY(fd == -1)) {
     658            std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
     659            exit(-1);
     660        }
     661        editd_ptr(fd);
     662        close(fd);
     663        run_second_filter(pattern_segs, total_len, 0.15);
     664        return 0;
     665    }
     666
    642667#ifdef CUDA_ENABLED
    643668    codegen::BlockSize = 64;
     
    682707    }
    683708    else{
    684         if (Threads == 1) {
    685             if (MultiEditdKernels) {
     709        if (Threads == 1) {     
     710            for(unsigned i=0; i<pattGroups.size(); i++){
     711
    686712                ParabixDriver pxDriver("editd");
    687                 multiEditdPipeline(pxDriver);
     713                editdPipeline(pxDriver, pattGroups[i]);
    688714                auto editd_ptr = reinterpret_cast<editdFunctionType>(pxDriver.getMain());
    689715                editd(editd_ptr, chStream, size);
    690             }
    691             else{               
    692                 for(unsigned i=0; i<pattGroups.size(); i++){
    693 
    694                     ParabixDriver pxDriver("editd");
    695                     editdPipeline(pxDriver, pattGroups[i]);
    696                     auto editd_ptr = reinterpret_cast<editdFunctionType>(pxDriver.getMain());
    697                     editd(editd_ptr, chStream, size);
    698                 }
    699716            }
    700717        }
  • icGREP/icgrep-devel/icgrep/toolchain/pipeline.cpp

    r5486 r5591  
    4545 ** ------------------------------------------------------------------------------------------------------------- */
    4646void generateSegmentParallelPipeline(const std::unique_ptr<KernelBuilder> & iBuilder, const std::vector<Kernel *> & kernels) {
     47
     48    assert (codegen::BufferSegments >= codegen::ThreadNum);
    4749
    4850    const unsigned n = kernels.size();
Note: See TracChangeset for help on using the changeset viewer.