Changeset 5574


Ignore:
Timestamp:
Jul 19, 2017, 4:36:46 PM (5 months ago)
Author:
cameron
Message:

Refactor to make a common grep pipeline

Location:
icGREP/icgrep-devel/icgrep
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep_engine.cpp

    r5561 r5574  
    8383}
    8484
    85 bool GrepEngine::matchesNeedToBeMovedToEOL() const {
     85bool matchesNeedToBeMovedToEOL() {
    8686    if ((Mode == QuietMode) | (Mode == FilesWithMatch) | (Mode == FilesWithoutMatch)) {
    8787        return false;
     
    300300}
    301301
     302   
     303std::pair<StreamSetBuffer *, StreamSetBuffer *> grepPipeline(Driver * grepDriver, std::vector<re::RE *> REs, const GrepModeType grepMode, unsigned encodingBits, StreamSetBuffer * ByteStream) {
     304    auto & idb = grepDriver->getBuilder();
     305    const unsigned segmentSize = codegen::SegmentSize;
     306    const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
     307    size_t MatchLimit = ((grepMode == QuietMode) | (grepMode == FilesWithMatch) | (grepMode == FilesWithoutMatch)) ? 1 : MaxCountFlag;
     308   
     309
     310    StreamSetBuffer * BasisBits = grepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(encodingBits, 1), segmentSize * bufferSegments));
     311    kernel::Kernel * s2pk = grepDriver->addKernelInstance(make_unique<kernel::S2PKernel>(idb));
     312    grepDriver->makeKernelCall(s2pk, {ByteStream}, {BasisBits});
     313   
     314    StreamSetBuffer * LineBreakStream = grepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
     315    kernel::Kernel * linebreakK = grepDriver->addKernelInstance(make_unique<kernel::LineBreakKernelBuilder>(idb, encodingBits));
     316    grepDriver->makeKernelCall(linebreakK, {BasisBits}, {LineBreakStream});
     317   
     318    kernel::Kernel * requiredStreamsK = grepDriver->addKernelInstance(make_unique<kernel::RequiredStreams_UTF8>(idb));
     319    StreamSetBuffer * RequiredStreams = grepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(4, 1), segmentSize * bufferSegments));
     320    grepDriver->makeKernelCall(requiredStreamsK, {BasisBits}, {RequiredStreams});
     321   
     322    const auto n = REs.size();
     323   
     324    std::vector<StreamSetBuffer *> MatchResultsBufs(n);
     325   
     326    for(unsigned i = 0; i < n; ++i){
     327        StreamSetBuffer * MatchResults = grepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
     328        kernel::Kernel * icgrepK = grepDriver->addKernelInstance(make_unique<kernel::ICGrepKernel>(idb, REs[i]));
     329        grepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults});
     330        MatchResultsBufs[i] = MatchResults;
     331    }
     332    StreamSetBuffer * MergedResults = MatchResultsBufs[0];
     333    if (REs.size() > 1) {
     334        MergedResults = grepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
     335        kernel::Kernel * streamsMergeK = grepDriver->addKernelInstance(make_unique<kernel::StreamsMerge>(idb, 1, REs.size()));
     336        grepDriver->makeKernelCall(streamsMergeK, MatchResultsBufs, {MergedResults});
     337    }
     338    StreamSetBuffer * Matches = MergedResults;
     339   
     340    if (matchesNeedToBeMovedToEOL()) {
     341        StreamSetBuffer * OriginalMatches = Matches;
     342        kernel::Kernel * matchedLinesK = grepDriver->addKernelInstance(make_unique<kernel::MatchedLinesKernel>(idb));
     343        Matches = grepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
     344        grepDriver->makeKernelCall(matchedLinesK, {OriginalMatches, LineBreakStream}, {Matches});
     345    }
     346   
     347    if (InvertMatchFlag) {
     348        kernel::Kernel * invertK = grepDriver->addKernelInstance(make_unique<kernel::InvertMatchesKernel>(idb));
     349        StreamSetBuffer * OriginalMatches = Matches;
     350        Matches = grepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
     351        grepDriver->makeKernelCall(invertK, {OriginalMatches, LineBreakStream}, {Matches});
     352    }
     353    if (MatchLimit > 0) {
     354        kernel::Kernel * untilK = grepDriver->addKernelInstance(make_unique<kernel::UntilNkernel>(idb));
     355        untilK->setInitialArguments({idb->getSize(MatchLimit)});
     356        StreamSetBuffer * AllMatches = Matches;
     357        Matches = grepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
     358        grepDriver->makeKernelCall(untilK, {AllMatches}, {Matches});
     359    }
     360    return std::pair<StreamSetBuffer *, StreamSetBuffer *>(LineBreakStream, Matches);
     361}
     362
     363
     364   
    302365void GrepEngine::grepCodeGen_nvptx(std::vector<re::RE *> REs, const GrepModeType grepMode, const bool UTF_16) {
    303366
     
    309372
    310373    const unsigned segmentSize = codegen::SegmentSize;
    311     const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
    312374    const unsigned encodingBits = UTF_16 ? 16 : 8;
    313375
     
    319381    Type * const voidTy = idb->getVoidTy();
    320382   
    321     size_t MatchLimit = ((grepMode == QuietMode) | (grepMode == FilesWithMatch) | (grepMode == FilesWithoutMatch)) ? 1 : MaxCountFlag;
    322 
    323383    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", voidTy, int64tyPtr, sizeTyPtr, sizeTyPtr, int64tyPtr, nullptr));
    324384    mainFunc->setCallingConv(CallingConv::C);
     
    352412    mGrepDriver->makeKernelCall(sourceK, {}, {ByteStream});
    353413
    354     StreamSetBuffer * BasisBits = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(8, 1), segmentSize * bufferSegments));
    355     kernel::Kernel * s2pk = mGrepDriver->addKernelInstance(make_unique<kernel::S2PKernel>(idb));
    356     mGrepDriver->makeKernelCall(s2pk, {ByteStream}, {BasisBits});
    357  
    358     StreamSetBuffer * LineBreakStream = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    359     kernel::Kernel * linebreakK = mGrepDriver->addKernelInstance(make_unique<kernel::LineBreakKernelBuilder>(idb, encodingBits));
    360     mGrepDriver->makeKernelCall(linebreakK, {BasisBits}, {LineBreakStream});
    361    
    362     kernel::Kernel * requiredStreamsK = mGrepDriver->addKernelInstance(make_unique<kernel::RequiredStreams_UTF8>(idb));
    363     StreamSetBuffer * RequiredStreams = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(4, 1), segmentSize));
    364     mGrepDriver->makeKernelCall(requiredStreamsK, {BasisBits}, {RequiredStreams});
    365    
    366     const auto n = REs.size();
    367 
    368     std::vector<StreamSetBuffer *> MatchResultsBufs(n);
    369 
    370     for(unsigned i = 0; i < n; ++i){
    371         StreamSetBuffer * MatchResults = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    372         kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance(make_unique<kernel::ICGrepKernel>(idb, REs[i]));
    373         mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults});
    374         MatchResultsBufs[i] = MatchResults;
    375     }
    376     StreamSetBuffer * MergedResults = MatchResultsBufs[0];
    377     if (REs.size() > 1) {
    378         MergedResults = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    379         kernel::Kernel * streamsMergeK = mGrepDriver->addKernelInstance(make_unique<kernel::StreamsMerge>(idb, 1, REs.size()));
    380         mGrepDriver->makeKernelCall(streamsMergeK, MatchResultsBufs, {MergedResults});
    381     }
    382     StreamSetBuffer * Matches = MergedResults;
    383 
    384     if (matchesNeedToBeMovedToEOL()) {
    385         StreamSetBuffer * OriginalMatches = Matches;
    386         kernel::Kernel * matchedLinesK = mGrepDriver->addKernelInstance(make_unique<kernel::MatchedLinesKernel>(idb));
    387         Matches = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    388         mGrepDriver->makeKernelCall(matchedLinesK, {OriginalMatches, LineBreakStream}, {Matches});
    389     }
    390    
    391     if (InvertMatchFlag) {
    392         kernel::Kernel * invertK = mGrepDriver->addKernelInstance(make_unique<kernel::InvertMatchesKernel>(idb));
    393         StreamSetBuffer * OriginalMatches = Matches;
    394         Matches = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    395         mGrepDriver->makeKernelCall(invertK, {OriginalMatches, LineBreakStream}, {Matches});
    396     }
    397     if (MatchLimit > 0) {
    398         kernel::Kernel * untilK = mGrepDriver->addKernelInstance(make_unique<kernel::UntilNkernel>(idb));
    399         untilK->setInitialArguments({idb->getSize(MatchLimit)});
    400         StreamSetBuffer * AllMatches = Matches;
    401         Matches = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    402         mGrepDriver->makeKernelCall(untilK, {AllMatches}, {Matches});
    403     }
     414    StreamSetBuffer * Matches = std::get<1>(grepPipeline(mGrepDriver, REs, grepMode, encodingBits, ByteStream));
    404415   
    405416    kernel::Kernel * matchCountK = mGrepDriver->addKernelInstance(make_unique<kernel::PopcountKernel>(idb));
     
    426437
    427438    const unsigned segmentSize = codegen::SegmentSize;
    428     const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
    429439    const unsigned encodingBits = UTF_16 ? 16 : 8;
    430440
     
    434444    kernel::Kernel * sourceK = nullptr;
    435445   
    436     size_t MatchLimit = ((grepMode == QuietMode) | (grepMode == FilesWithMatch) | (grepMode == FilesWithoutMatch)) ? 1 : MaxCountFlag;
    437 
    438446    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", int64Ty, idb->getInt32Ty(), int32Ty, nullptr));
    439447    mainFunc->setCallingConv(CallingConv::C);
     
    456464
    457465    mGrepDriver->makeKernelCall(sourceK, {}, {ByteStream});
    458     StreamSetBuffer * BasisBits = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(8, 1), segmentSize * bufferSegments));
    459    
    460     kernel::Kernel * s2pk = mGrepDriver->addKernelInstance(make_unique<kernel::S2PKernel>(idb));
    461     mGrepDriver->makeKernelCall(s2pk, {ByteStream}, {BasisBits});
    462    
    463     kernel::Kernel * linebreakK = mGrepDriver->addKernelInstance(make_unique<kernel::LineBreakKernelBuilder>(idb, encodingBits));
    464     StreamSetBuffer * LineBreakStream = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    465     mGrepDriver->makeKernelCall(linebreakK, {BasisBits}, {LineBreakStream});
    466    
    467     kernel::Kernel * requiredStreamsK = mGrepDriver->addKernelInstance(make_unique<kernel::RequiredStreams_UTF8>(idb));
    468     StreamSetBuffer * RequiredStreams = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(4, 1), segmentSize));
    469     mGrepDriver->makeKernelCall(requiredStreamsK, {BasisBits}, {RequiredStreams});
    470    
    471     const auto n = REs.size();
    472 
    473     std::vector<StreamSetBuffer *> MatchResultsBufs(n);
    474 
    475     for(unsigned i = 0; i < n; ++i){
    476         StreamSetBuffer * MatchResults = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    477         kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance(make_unique<kernel::ICGrepKernel>(idb, REs[i]));
    478         mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults});
    479         MatchResultsBufs[i] = MatchResults;
    480     }
    481     StreamSetBuffer * MergedResults = MatchResultsBufs[0];
    482     if (REs.size() > 1) {
    483         MergedResults = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    484         kernel::Kernel * streamsMergeK = mGrepDriver->addKernelInstance(make_unique<kernel::StreamsMerge>(idb, 1, REs.size()));
    485         mGrepDriver->makeKernelCall(streamsMergeK, MatchResultsBufs, {MergedResults});
    486     }
    487     StreamSetBuffer * Matches = MergedResults;
    488    
    489     if (matchesNeedToBeMovedToEOL()) {
    490         StreamSetBuffer * OriginalMatches = Matches;
    491         kernel::Kernel * matchedLinesK = mGrepDriver->addKernelInstance(make_unique<kernel::MatchedLinesKernel>(idb));
    492         Matches = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    493         mGrepDriver->makeKernelCall(matchedLinesK, {OriginalMatches, LineBreakStream}, {Matches});
    494     }
    495    
    496     if (InvertMatchFlag) {
    497         kernel::Kernel * invertK = mGrepDriver->addKernelInstance(make_unique<kernel::InvertMatchesKernel>(idb));
    498         StreamSetBuffer * OriginalMatches = Matches;
    499         Matches = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    500         mGrepDriver->makeKernelCall(invertK, {OriginalMatches, LineBreakStream}, {Matches});
    501     }
    502     if (MatchLimit > 0) {
    503         kernel::Kernel * untilK = mGrepDriver->addKernelInstance(make_unique<kernel::UntilNkernel>(idb));
    504         untilK->setInitialArguments({idb->getSize(MatchLimit)});
    505         StreamSetBuffer * AllMatches = Matches;
    506         Matches = mGrepDriver->addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments));
    507         mGrepDriver->makeKernelCall(untilK, {AllMatches}, {Matches});
    508     }
     466   
     467    StreamSetBuffer * LineBreakStream;
     468    StreamSetBuffer * Matches;
     469    std::tie(LineBreakStream, Matches) = grepPipeline(mGrepDriver, REs, grepMode, encodingBits, ByteStream);
     470   
    509471    if (grepMode == NormalMode) {
    510472        kernel::Kernel * scanMatchK = mGrepDriver->addKernelInstance(make_unique<kernel::ScanMatchKernel>(idb, GrepType::Normal, encodingBits));
  • icGREP/icgrep-devel/icgrep/grep_engine.h

    r5550 r5574  
    3737    void doGrep(const std::string & fileName, std::string & PTXFilename) const;
    3838   
    39     bool matchesNeedToBeMovedToEOL() const;
    40 
    4139    uint64_t doGrep(const std::string & fileName, const uint32_t fileIdx) const;
    4240
Note: See TracChangeset for help on using the changeset viewer.