Ignore:
Timestamp:
Apr 9, 2017, 3:59:17 PM (2 years ago)
Author:
nmedfort
Message:

Updated all projects to use ParabixDriver?. Deprecated original pipeline generation methods. Enabled LLVM optimizations, IR and ASM printing for Kernel modules. Enabled object cache by default. Begun work on moving consumed position information back to producing kernels.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep_engine.cpp

    r5400 r5401  
    6262static cl::alias ShowLineNumbersLong("line-number", cl::desc("Alias for -n"), cl::aliasopt(ShowLineNumbers));
    6363
    64 /// iNVESTIGATE: icgrep is reporting stdin is not empty even when nothing is being piped into it?
    65 static cl::opt<bool> UseStdIn("stdin", cl::desc("Read from standard input."), cl::cat(bGrepOutputOptions));
    66 
    67 bool isUTF_16 = false;
    68 std::string IRFilename = "icgrep.ll";
    69 std::string PTXFilename = "icgrep.ptx";
     64#ifdef CUDA_ENABLED
     65const auto IRFilename = "icgrep.ll";
     66const auto PTXFilename = "icgrep.ptx";
     67#endif
    7068
    7169static re::CC * parsedCodePointSet = nullptr;
     70
    7271static std::vector<std::string> parsedPropertyValues;
    7372
     
    288287template<typename CodeUnit>
    289288void wrapped_report_match(const size_t lineNum, size_t line_start, size_t line_end, const CodeUnit * const buffer, const size_t filesize, const size_t fileIdx) {
    290 
    291 //    errs() << lineNum << " : (" << line_start << ", " << line_end << ", " << filesize << ")\n";
    292 
    293289    assert (buffer);
    294290    assert (line_start <= line_end);
     
    390386}
    391387
    392 void GrepEngine::grepCodeGen(std::string moduleName, re::RE * re_ast, bool CountOnly, bool UTF_16, GrepType grepType, const bool usingStdIn) {
    393     isUTF_16 = UTF_16;
     388inline void linkGrepFunction(ParabixDriver & pxDriver, const GrepType grepType, const bool UTF_16, kernel::KernelBuilder & kernel) {
     389    switch (grepType) {
     390        case GrepType::Normal:
     391            if (UTF_16) {
     392                pxDriver.addExternalLink(kernel, "matcher", &wrapped_report_match<uint16_t>);
     393            } else {
     394                pxDriver.addExternalLink(kernel, "matcher", &wrapped_report_match<uint8_t>);
     395            }
     396            break;
     397        case GrepType::NameExpression:
     398            pxDriver.addExternalLink(kernel, "matcher", &insert_codepoints);
     399            break;
     400        case GrepType::PropertyValue:
     401            pxDriver.addExternalLink(kernel, "matcher", &insert_property_values);
     402            break;
     403    }
     404}
     405
     406void GrepEngine::grepCodeGen(std::string moduleName, re::RE * re_ast, const bool CountOnly, const bool UTF_16, const GrepType grepType, const bool usingStdIn) {
    394407    int addrSpace = 0;
    395408    bool CPU_Only = true;
     
    499512    pxDriver.addKernelCall(s2pk, {byteStream}, {&BasisBits});
    500513
    501     kernel::LineBreakKernelBuilder linebreakK(iBuilder, "lb", encodingBits);
     514    kernel::LineBreakKernelBuilder linebreakK(iBuilder, encodingBits);
    502515    CircularBuffer LineBreakStream(iBuilder, iBuilder->getStreamSetTy(1, 1), segmentSize * bufferSegments);
    503516    LineBreakStream.allocateBuffer();
     
    517530        iBuilder->CreateRet(icgrepK.createGetAccumulatorCall(icgrepK.getInstance(), "matchedLineCount"));
    518531
    519         pxDriver.JITcompileMain();
    520532        pxDriver.linkAndFinalize();
    521533
     
    550562            pxDriver.addKernelCall(scanMatchK, {&MatchResults, &LineBreakStream, byteStream}, {});
    551563
    552             switch (grepType) {
    553                 case GrepType::Normal:
    554                     if (UTF_16) {
    555                         pxDriver.addExternalLink(scanMatchK, "matcher", &wrapped_report_match<uint16_t>);
    556                     } else {
    557                         pxDriver.addExternalLink(scanMatchK, "matcher", &wrapped_report_match<uint8_t>);
    558                     }
    559                     break;
    560                 case GrepType::NameExpression:
    561                     pxDriver.addExternalLink(scanMatchK, "matcher", &insert_codepoints);
    562                     break;
    563                 case GrepType::PropertyValue:
    564                     pxDriver.addExternalLink(scanMatchK, "matcher", &insert_property_values);
    565                     break;
    566             }
     564            linkGrepFunction(pxDriver, grepType, UTF_16, scanMatchK);
    567565
    568566            pxDriver.generatePipelineIR();
     
    570568            iBuilder->CreateRetVoid();
    571569
    572             pxDriver.JITcompileMain();
    573570            pxDriver.linkAndFinalize();
    574571        }
     
    610607
    611608
    612 void GrepEngine::multiGrepCodeGen(std::string moduleName, std::vector<re::RE *> REs, bool CountOnly, bool UTF_16, GrepType grepType, const bool usingStdIn) {
    613 
    614     isUTF_16 = UTF_16;
     609
     610void GrepEngine::grepCodeGen(std::string moduleName, std::vector<re::RE *> REs, const bool CountOnly, const bool UTF_16, const GrepType grepType, const bool usingStdIn) {
     611
    615612    Module * M = new Module(moduleName + ":icgrep", getGlobalContext());;
    616613    IDISA::IDISA_Builder * iBuilder = IDISA::GetIDISA_Builder(M);;
     614    ParabixDriver pxDriver(iBuilder);
    617615
    618616    const unsigned segmentSize = codegen::SegmentSize;
     
    639637    kernel::KernelBuilder * sourceK = nullptr;
    640638    if (usingStdIn) {
     639        // TODO: use fstat(STDIN_FILENO) to see if we can mmap the stdin safely and avoid the calls to read
    641640        byteStream = new ExtensibleBuffer(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize);
    642         cast<ExtensibleBuffer>(byteStream)->allocateBuffer();
    643641        sourceK = new kernel::StdInKernel(iBuilder, segmentSize);
    644642    } else {
    645         byteStream = new ExternalFileBuffer(iBuilder, iBuilder->getStreamSetTy(1, 8));
    646         cast<ExternalFileBuffer>(byteStream)->setStreamSetBuffer(inputStream);
    647         sourceK = new kernel::MMapSourceKernel(iBuilder, segmentSize);
    648         sourceK->setInitialArguments({fileSize});
    649     }
    650     sourceK->generateKernel({}, {byteStream});
     643        byteStream = new SourceFileBuffer(iBuilder, iBuilder->getStreamSetTy(1, 8));
     644        sourceK = new kernel::FileSourceKernel(iBuilder, inputStream->getType(), segmentSize);
     645        sourceK->setInitialArguments({inputStream, fileSize});
     646    }
     647    byteStream->allocateBuffer();
     648    pxDriver.addKernelCall(*sourceK, {}, {byteStream});
    651649
    652650    CircularBuffer BasisBits(iBuilder, iBuilder->getStreamSetTy(8), segmentSize * bufferSegments);
    653651    BasisBits.allocateBuffer();
    654652
    655     kernel::S2PKernel  s2pk(iBuilder);
    656     s2pk.generateKernel({byteStream}, {&BasisBits});
     653    kernel::S2PKernel s2pk(iBuilder);
     654    pxDriver.addKernelCall(s2pk, {byteStream}, {&BasisBits});
     655
     656    kernel::LineBreakKernelBuilder linebreakK(iBuilder, encodingBits);
     657    CircularBuffer LineBreakStream(iBuilder, iBuilder->getStreamSetTy(1, 1), segmentSize * bufferSegments);
     658    LineBreakStream.allocateBuffer();
     659    pxDriver.addKernelCall(linebreakK, {&BasisBits}, {&LineBreakStream});
    657660
    658661    std::vector<pablo::PabloKernel *> icgrepKs;
    659662    std::vector<StreamSetBuffer *> MatchResultsBufs;
    660663
    661     for(unsigned i=0; i<REs.size(); i++){
    662         pablo::PabloKernel * icgrepK = new pablo::PabloKernel(iBuilder, "icgrep"+std::to_string(i), {Binding{iBuilder->getStreamSetTy(8), "basis"}, Binding{iBuilder->getStreamSetTy(1, 1), "linebreak"}});
     664    for(unsigned i = 0; i < REs.size(); ++i){
     665        pablo::PabloKernel * const icgrepK = new pablo::PabloKernel(iBuilder, "icgrep" + std::to_string(i), {Binding{iBuilder->getStreamSetTy(8), "basis"}, Binding{iBuilder->getStreamSetTy(1, 1), "linebreak"}});
    663666        re::re2pablo_compiler(icgrepK, re::regular_expression_passes(REs[i]), false);
    664667        pablo_function_passes(icgrepK);
     668        CircularBuffer * const matchResults = new CircularBuffer(iBuilder, iBuilder->getStreamSetTy(2, 1), segmentSize * bufferSegments);
     669        matchResults->allocateBuffer();
     670
     671        pxDriver.addKernelCall(*icgrepK, {&BasisBits, &LineBreakStream}, {matchResults});
    665672        icgrepKs.push_back(icgrepK);
    666         CircularBuffer * MatchResults = new CircularBuffer(iBuilder, iBuilder->getStreamSetTy(2, 1), segmentSize * bufferSegments);
    667         MatchResults->allocateBuffer();
    668         MatchResultsBufs.push_back(MatchResults);
    669     }
    670 
    671     std::vector<kernel::KernelBuilder *> KernelList;
    672     KernelList.push_back(sourceK);
    673     KernelList.push_back(&s2pk);
     673        MatchResultsBufs.push_back(matchResults);
     674    }
    674675
    675676    CircularBuffer mergedResults(iBuilder, iBuilder->getStreamSetTy(1, 1), segmentSize * bufferSegments);
     
    677678
    678679    kernel::StreamsMerge streamsMergeK(iBuilder, 1, REs.size());
    679     streamsMergeK.generateKernel(MatchResultsBufs, {&mergedResults});
    680 
    681     kernel::LineBreakKernelBuilder linebreakK(iBuilder, "lb", encodingBits);
    682     CircularBuffer LineBreakStream(iBuilder, iBuilder->getStreamSetTy(1, 1), segmentSize * bufferSegments);
    683     LineBreakStream.allocateBuffer();
    684     linebreakK.generateKernel({&BasisBits}, {&LineBreakStream});
    685 
    686     KernelList.push_back(&linebreakK);
    687     for(unsigned i=0; i<REs.size(); i++){
    688         icgrepKs[i]->generateKernel({&BasisBits, &LineBreakStream}, {MatchResultsBufs[i]});
    689         KernelList.push_back(icgrepKs[i]);
    690     }
    691     KernelList.push_back(&streamsMergeK);
     680    pxDriver.addKernelCall(streamsMergeK, MatchResultsBufs, {&mergedResults});
    692681
    693682    if (CountOnly) {
    694683        kernel::MatchCount matchCountK(iBuilder);
    695         matchCountK.generateKernel({&mergedResults}, {});
    696 
    697         KernelList.push_back(&matchCountK);
    698 
    699         generatePipeline(iBuilder, KernelList);
     684        pxDriver.addKernelCall(matchCountK, {&mergedResults}, {});
     685        pxDriver.generatePipelineIR();
    700686        iBuilder->CreateRet(matchCountK.getScalarField(matchCountK.getInstance(), "matchedLineCount"));
    701 
     687        pxDriver.linkAndFinalize();
    702688    } else {
    703689        kernel::ScanMatchKernel scanMatchK(iBuilder, grepType, encodingBits);
    704         scanMatchK.generateKernel({byteStream, &mergedResults, &LineBreakStream}, {});
    705690        scanMatchK.setInitialArguments({fileIdx});
    706 
    707         KernelList.push_back(&scanMatchK);
    708 
    709         generatePipeline(iBuilder, KernelList);
    710 
     691        pxDriver.addKernelCall(scanMatchK, {&mergedResults, &LineBreakStream, byteStream}, {});
     692        linkGrepFunction(pxDriver, grepType, UTF_16, scanMatchK);
     693        pxDriver.generatePipelineIR();
    711694        iBuilder->CreateRetVoid();
    712     }
    713 
    714     mEngine = JIT_to_ExecutionEngine(M);
    715     ApplyObjectCache(mEngine);
    716     icgrep_Linking(M, mEngine);
    717 
    718     mEngine->finalizeObject();
     695        pxDriver.linkAndFinalize();
     696    }
     697
    719698    delete iBuilder;
    720699    delete sourceK;
    721700    delete byteStream;
     701    for (StreamSetBuffer * buf : MatchResultsBufs) {
     702        delete buf;
     703    }
    722704
    723705    if (CountOnly) {
    724         mGrepFunction_CountOnly = reinterpret_cast<GrepFunctionType_CountOnly>(mEngine->getPointerToFunction(mainFn));
    725     } else {
    726         mGrepFunction = reinterpret_cast<GrepFunctionType>(mEngine->getPointerToFunction(mainFn));
    727     }
    728 
     706        mGrepFunction_CountOnly = reinterpret_cast<GrepFunctionType_CountOnly>(pxDriver.getPointerToMain());
     707    } else {
     708        mGrepFunction = reinterpret_cast<GrepFunctionType>(pxDriver.getPointerToMain());
     709    }
    729710}
    730711
     
    753734}
    754735
    755 void icgrep_Linking(Module * m, ExecutionEngine * e) {
    756     Module::FunctionListType & fns = m->getFunctionList();
    757     for (auto it = fns.begin(), it_end = fns.end(); it != it_end; ++it) {
    758         std::string fnName = it->getName().str();
    759         if (fnName == "s2p_block") continue;
    760         if (fnName == "process_block") continue;
    761         if (fnName == "process_block_initialize_carries") continue;
    762        
    763         if (fnName == "wrapped_report_match8") {
    764             e->addGlobalMapping(cast<GlobalValue>(it), (void *)&wrapped_report_match<uint8_t>);
    765         }
    766         if (fnName == "wrapped_report_match16") {
    767             e->addGlobalMapping(cast<GlobalValue>(it), (void *)&wrapped_report_match<uint16_t>);
    768         }
    769         if (fnName == "insert_codepoints") {
    770             e->addGlobalMapping(cast<GlobalValue>(it), (void *)&insert_codepoints);
    771         }
    772         if (fnName == "insert_property_values") {
    773             e->addGlobalMapping(cast<GlobalValue>(it), (void *)&insert_property_values);
    774         }
    775     }
    776 }
    777 
    778736GrepEngine::GrepEngine()
    779737: mGrepFunction(nullptr)
     
    782740, mGrepFunction_CPU(nullptr)
    783741#endif
    784 , mEngine(nullptr) {
    785 
    786 }
    787 
    788 GrepEngine::~GrepEngine() {
    789     delete mEngine;
    790 }
     742{
     743
     744}
Note: See TracChangeset for help on using the changeset viewer.