Ignore:
Timestamp:
Apr 8, 2018, 5:31:54 PM (17 months ago)
Author:
cameron
Message:

InternalSearchEngine?

Location:
icGREP/icgrep-devel/icgrep/grep
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.cpp

    r5953 r5954  
    119119}
    120120
    121    
    122 
    123    
    124 void grepBuffer(re::RE * pattern, const char * search_buffer, size_t bufferLength, MatchAccumulator * accum) {
    125     const unsigned segmentSize = codegen::BufferSegments * codegen::SegmentSize * codegen::ThreadNum;
    126     auto segParallelModeSave = codegen::SegmentPipelineParallel;
    127     codegen::SegmentPipelineParallel = false;
    128    
    129     pattern = resolveCaseInsensitiveMode(pattern, false);
    130     pattern = regular_expression_passes(pattern);
    131     pattern = re::exclude_CC(pattern, re::makeByte(0x0A));
    132     pattern = resolveAnchors(pattern, re::makeByte(0x0A));
    133 
    134     ParabixDriver pxDriver("codepointEngine");
    135     auto & idb = pxDriver.getBuilder();
    136     Module * M = idb->getModule();
    137    
    138     Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getVoidTy(), idb->getInt8PtrTy(), idb->getSizeTy(), nullptr));
    139     mainFunc->setCallingConv(CallingConv::C);
    140     auto args = mainFunc->arg_begin();
    141     Value * const buffer = &*(args++);
    142     buffer->setName("buffer");
    143     Value * length = &*(args++);
    144     length->setName("length");
    145    
    146     idb->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", mainFunc, 0));
    147     StreamSetBuffer * ByteStream = pxDriver.addBuffer<SourceBuffer>(idb, idb->getStreamSetTy(1, 8));
    148     kernel::Kernel * sourceK = pxDriver.addKernelInstance<kernel::MemorySourceKernel>(idb, idb->getInt8PtrTy());
    149     sourceK->setInitialArguments({buffer, length});
    150     pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
    151    
    152    
    153     StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(8, 1), segmentSize);
    154     kernel::Kernel * s2pk = pxDriver.addKernelInstance<kernel::S2PKernel>(idb);
    155     pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
    156    
    157     StreamSetBuffer * LineFeedStream = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    158     kernel::Kernel * linefeedK = pxDriver.addKernelInstance<kernel::LineFeedKernelBuilder>(idb, Binding{idb->getStreamSetTy(8), "basis", FixedRate(), Principal()});
    159     pxDriver.makeKernelCall(linefeedK, {BasisBits}, {LineFeedStream});
    160    
    161     StreamSetBuffer * LineBreakStream = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    162    
    163     kernel::Kernel * requiredStreamsK = pxDriver.addKernelInstance<kernel::RequiredStreams_UTF8>(idb);
    164     StreamSetBuffer * RequiredStreams = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    165     pxDriver.makeKernelCall(requiredStreamsK, {BasisBits, LineFeedStream}, {RequiredStreams, LineBreakStream});
    166    
    167     StreamSetBuffer * MatchResults = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    168     kernel::Kernel * icgrepK = pxDriver.addKernelInstance<kernel::ICGrepKernel>(idb, pattern, std::vector<std::string>{"UTF8_LB", "UTF8_nonfinal"});
    169     pxDriver.makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults});
    170    
    171     StreamSetBuffer * MatchedLines = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    172     kernel::Kernel * matchedLinesK = pxDriver.addKernelInstance<kernel::MatchedLinesKernel>(idb);
    173     pxDriver.makeKernelCall(matchedLinesK, {MatchResults, LineBreakStream}, {MatchedLines});
    174    
    175     kernel::Kernel * scanMatchK = pxDriver.addKernelInstance<kernel::ScanMatchKernel>(idb);
    176     scanMatchK->setInitialArguments({ConstantInt::get(idb->getIntAddrTy(), reinterpret_cast<intptr_t>(accum))});
    177     pxDriver.makeKernelCall(scanMatchK, {MatchedLines, LineBreakStream, ByteStream}, {});
    178     pxDriver.LinkFunction(*scanMatchK, "accumulate_match_wrapper", &accumulate_match_wrapper);
    179     pxDriver.LinkFunction(*scanMatchK, "finalize_match_wrapper", &finalize_match_wrapper);
    180    
    181     pxDriver.generatePipelineIR();
    182     pxDriver.deallocateBuffers();
    183     idb->CreateRetVoid();
    184     pxDriver.finalizeObject();
    185    
    186     typedef void (*GrepFunctionType)(const char * buffer, const size_t length);
    187     auto f = reinterpret_cast<GrepFunctionType>(pxDriver.getMain());
    188     f(search_buffer, bufferLength);
    189     codegen::SegmentPipelineParallel = segParallelModeSave;
    190 }
    191 
    192121
    193122
     
    818747    const unsigned encodingBits = 8;
    819748    const unsigned segmentSize = codegen::BufferSegments * codegen::SegmentSize * codegen::ThreadNum;
    820     auto segParallelModeSave = codegen::SegmentPipelineParallel;
    821     codegen::SegmentPipelineParallel = false;
    822749   
    823750    re::CC * breakCC = nullptr;
    824751    if (mGrepRecordBreak == GrepRecordBreakKind::Null) {
    825         breakCC = re::makeByte(0);
     752        breakCC = re::makeByte(0xA);
    826753    } else {// if (mGrepRecordBreak == GrepRecordBreakKind::LF)
    827754        breakCC = re::makeByte(0x0A);
    828755    }
    829    
    830     if (matchingRE) {
     756    if (matchingRE != nullptr) {
    831757        matchingRE = resolveCaseInsensitiveMode(matchingRE, mCaseInsensitive);
    832758        matchingRE = regular_expression_passes(matchingRE);
    833         matchingRE = re::exclude_CC(excludedRE, breakCC);
    834         matchingRE = resolveAnchors(excludedRE, breakCC);
    835     }
    836    
    837     if (excludedRE) {
    838         excludedRE = resolveCaseInsensitiveMode(matchingRE, mCaseInsensitive);
    839         excludedRE = regular_expression_passes(matchingRE);
     759        matchingRE = re::exclude_CC(matchingRE, breakCC);
     760        matchingRE = resolveAnchors(matchingRE, breakCC);
     761    }
     762    if (excludedRE != nullptr) {
     763        excludedRE = resolveCaseInsensitiveMode(excludedRE, mCaseInsensitive);
     764        excludedRE = regular_expression_passes(excludedRE);
    840765        excludedRE = re::exclude_CC(excludedRE, breakCC);
    841766        excludedRE = resolveAnchors(excludedRE, breakCC);
    842767    }
    843    
    844768    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getVoidTy(), idb->getInt8PtrTy(), idb->getSizeTy(), nullptr));
    845769    mainFunc->setCallingConv(CallingConv::C);
     
    855779    sourceK->setInitialArguments({buffer, length});
    856780    mGrepDriver->makeKernelCall(sourceK, {}, {ByteStream});
    857    
    858781    StreamSetBuffer * BasisBits = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(8, 1), segmentSize);
    859782    kernel::Kernel * s2pk = mGrepDriver->addKernelInstance<kernel::S2PKernel>(idb);
     
    868791   
    869792    std::vector<std::string> externalStreamNames;
    870     if (matchingRE) {
     793    if (matchingRE != nullptr) {
     794        StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    871795        kernel::Kernel * includeK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, matchingRE, externalStreamNames);
    872         mGrepDriver->makeKernelCall(includeK, {BasisBits}, {MatchingRecords});
    873     }
    874    
    875     if (excludedRE) {
     796        mGrepDriver->makeKernelCall(includeK, {BasisBits}, {MatchResults});
     797        kernel::Kernel * matchedLinesK = mGrepDriver->addKernelInstance<kernel::MatchedLinesKernel>(idb);
     798        mGrepDriver->makeKernelCall(matchedLinesK, {MatchResults, RecordBreakStream}, {MatchingRecords});
     799    }
     800   
     801    if (excludedRE != nullptr) {
     802        StreamSetBuffer * ExcludedResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
     803        kernel::Kernel * excludeK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, excludedRE, externalStreamNames);
     804        mGrepDriver->makeKernelCall(excludeK, {BasisBits}, {ExcludedResults});
    876805        StreamSetBuffer * ExcludedRecords = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    877         kernel::Kernel * excludeK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, excludedRE, externalStreamNames);
    878         mGrepDriver->makeKernelCall(excludeK, {BasisBits}, {ExcludedRecords});
    879        
     806        kernel::Kernel * matchedLinesK = mGrepDriver->addKernelInstance<kernel::MatchedLinesKernel>(idb);
     807        mGrepDriver->makeKernelCall(matchedLinesK, {ExcludedResults, RecordBreakStream}, {ExcludedRecords});
     808
    880809        kernel::Kernel * invertK = mGrepDriver->addKernelInstance<kernel::InvertMatchesKernel>(idb);
    881         if (matchingRE) {
     810        if (matchingRE != nullptr) {
    882811            StreamSetBuffer * nonExcluded = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    883812            mGrepDriver->makeKernelCall(invertK, {ExcludedRecords, RecordBreakStream}, {nonExcluded});
     
    890819        }
    891820    }
    892    
     821
    893822    kernel::Kernel * scanMatchK = mGrepDriver->addKernelInstance<kernel::ScanMatchKernel>(idb);
    894823    scanMatchK->setInitialArguments({ConstantInt::get(idb->getIntAddrTy(), reinterpret_cast<intptr_t>(accum))});
     
    896825    mGrepDriver->LinkFunction(*scanMatchK, "accumulate_match_wrapper", &accumulate_match_wrapper);
    897826    mGrepDriver->LinkFunction(*scanMatchK, "finalize_match_wrapper", &finalize_match_wrapper);
    898    
    899827    mGrepDriver->generatePipelineIR();
    900828    mGrepDriver->deallocateBuffers();
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.h

    r5953 r5954  
    3535extern "C" void finalize_match_wrapper(intptr_t accum_addr, char * buffer_end);
    3636
    37    
    38 #define MAX_SIMD_WIDTH_SUPPORTED 512
    39 #define INITIAL_CAPACITY 1024
    40    
    41 class SearchableBuffer  {
    42     SearchableBuffer();
    43     void addSearchCandidate(char * string_ptr, size_t length);
    44     size_t getCandidateCount() {return mEntries;}
    45     ~SearchableBuffer();
    46 private:
    47     static const unsigned BUFFER_ALIGNMENT = MAX_SIMD_WIDTH_SUPPORTED/8;
    48     size_t mAllocated_capacity;
    49     char * mBuffer_base;
    50     alignas(BUFFER_ALIGNMENT) char mInitial_buffer[INITIAL_CAPACITY];
    51     size_t mSpace_used;
    52     size_t mEntries;
    53 };
    54 
    55 void grepBuffer(re::RE * pattern, const char * buffer, size_t bufferLength, MatchAccumulator * accum);
    5637
    5738class GrepEngine {
     
    197178};
    198179   
     180   
     181#define MAX_SIMD_WIDTH_SUPPORTED 512
     182#define INITIAL_CAPACITY 1024
     183   
     184    class SearchableBuffer  {
     185        SearchableBuffer();
     186        void addSearchCandidate(char * string_ptr, size_t length);
     187        size_t getCandidateCount() {return mEntries;}
     188        char * getBufferBase() {return mBuffer_base;}
     189        size_t getBufferSize() {return mSpace_used;}
     190        ~SearchableBuffer();
     191    private:
     192        static const unsigned BUFFER_ALIGNMENT = MAX_SIMD_WIDTH_SUPPORTED/8;
     193        size_t mAllocated_capacity;
     194        char * mBuffer_base;
     195        alignas(BUFFER_ALIGNMENT) char mInitial_buffer[INITIAL_CAPACITY];
     196        size_t mSpace_used;
     197        size_t mEntries;
     198    };
    199199
    200200}
Note: See TracChangeset for help on using the changeset viewer.