Changeset 5678 for icGREP/icgrep-devel


Ignore:
Timestamp:
Oct 8, 2017, 9:20:51 AM (20 months ago)
Author:
cameron
Message:

MatchAccumulator? modularization step

Location:
icGREP/icgrep-devel/icgrep
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep_engine.cpp

    r5677 r5678  
    523523}
    524524
    525 
    526    
    527 static re::CC * parsedCodePointSet = nullptr;
    528 
    529 void insert_codepoints(const size_t lineNum, const size_t line_start, const size_t line_end, const char * const buffer) {
    530     assert (buffer);
    531     assert (line_start <= line_end);
    532     re::codepoint_t c = 0;
    533     size_t line_pos = line_start;
    534     while (isxdigit(buffer[line_pos])) {
    535         assert (line_pos < line_end);
    536         if (isdigit(buffer[line_pos])) {
    537             c = (c << 4) | (buffer[line_pos] - '0');
    538         }
    539         else {
    540             c = (c << 4) | (tolower(buffer[line_pos]) - 'a' + 10);
    541         }
    542         line_pos++;
    543     }
    544     assert(((line_pos - line_start) >= 4) && ((line_pos - line_start) <= 6)); // UCD format 4 to 6 hex digits.
    545     parsedCodePointSet->insert(c);
    546 }
    547 
    548 class CodepointAccumulator : public kernel::MatchAccumulator {
     525   
     526void accumulate_match_wrapper(intptr_t accum_addr, const size_t lineNum, size_t line_start, size_t line_end) {
     527    reinterpret_cast<MatchAccumulator *>(accum_addr)->accumulate_match(lineNum, line_start, line_end);
     528}
     529
     530   
     531
     532
     533void grepBuffer(re::RE * pattern, char * UnicodeDataBuffer, size_t bufferLength, MatchAccumulator * accum) {
     534    const unsigned segmentSize = 8;
     535
     536    ParabixDriver pxDriver("codepointEngine");
     537    auto & idb = pxDriver.getBuilder();
     538    Module * M = idb->getModule();
     539   
     540    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getVoidTy(), idb->getInt8PtrTy(), idb->getSizeTy(), nullptr));
     541    mainFunc->setCallingConv(CallingConv::C);
     542    auto args = mainFunc->arg_begin();
     543    Value * const buffer = &*(args++);
     544    buffer->setName("buffer");
     545    Value * length = &*(args++);
     546    length->setName("length");
     547   
     548    idb->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", mainFunc, 0));
     549   
     550    StreamSetBuffer * ByteStream = pxDriver.addBuffer(make_unique<SourceBuffer>(idb, idb->getStreamSetTy(1, 8)));
     551    kernel::Kernel * sourceK = pxDriver.addKernelInstance(make_unique<kernel::MemorySourceKernel>(idb, idb->getInt8PtrTy(), segmentSize));
     552    sourceK->setInitialArguments({buffer, length});
     553    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
     554   
     555    StreamSetBuffer * BasisBits = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(8, 1), segmentSize));
     556   
     557    kernel::Kernel * s2pk = pxDriver.addKernelInstance(make_unique<kernel::S2PKernel>(idb));
     558    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
     559   
     560    kernel::Kernel * linebreakK = pxDriver.addKernelInstance(make_unique<kernel::LineBreakKernelBuilder>(idb, 8));
     561    StreamSetBuffer * LineBreakStream = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize));
     562    pxDriver.makeKernelCall(linebreakK, {BasisBits}, {LineBreakStream});
     563   
     564    kernel::Kernel * requiredStreamsK = pxDriver.addKernelInstance(make_unique<kernel::RequiredStreams_UTF8>(idb));
     565    StreamSetBuffer * RequiredStreams = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(4, 1), segmentSize));
     566    pxDriver.makeKernelCall(requiredStreamsK, {BasisBits}, {RequiredStreams});
     567   
     568    StreamSetBuffer * MatchResults = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize));
     569    kernel::Kernel * icgrepK = pxDriver.addKernelInstance(make_unique<kernel::ICGrepKernel>(idb, pattern));
     570    pxDriver.makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults});
     571   
     572    StreamSetBuffer * MatchedLines = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize));
     573    kernel::Kernel * matchedLinesK = pxDriver.addKernelInstance(make_unique<kernel::MatchedLinesKernel>(idb));
     574    pxDriver.makeKernelCall(matchedLinesK, {MatchResults, LineBreakStream}, {MatchedLines});
     575   
     576    kernel::Kernel * scanMatchK = pxDriver.addKernelInstance(make_unique<kernel::ScanMatchKernel>(idb, GrepType::CallBack, 8));
     577    scanMatchK->setInitialArguments({ConstantInt::get(idb->getIntAddrTy(), reinterpret_cast<intptr_t>(accum))});
     578    pxDriver.makeKernelCall(scanMatchK, {MatchedLines, LineBreakStream, ByteStream}, {});
     579    pxDriver.LinkFunction(*scanMatchK, "accumulate_match_wrapper", &accumulate_match_wrapper);
     580    pxDriver.generatePipelineIR();
     581    pxDriver.deallocateBuffers();
     582    idb->CreateRetVoid();
     583    pxDriver.finalizeObject();
     584   
     585    typedef void (*GrepFunctionType)(const char * buffer, const size_t length);
     586    auto f = reinterpret_cast<GrepFunctionType>(pxDriver.getMain());
     587    f(UnicodeDataBuffer, bufferLength);
     588}
     589
     590class CodepointAccumulator : public MatchAccumulator {
    549591public:
    550592   
     
    557599    re::CC * mParsedCodePointSet;
    558600};
    559    
     601
    560602void CodepointAccumulator::accumulate_match(const size_t lineNum, size_t line_start, size_t line_end) {
    561603    assert (line_start <= line_end);
     
    573615    }
    574616    assert(((line_pos - line_start) >= 4) && ((line_pos - line_start) <= 6)); // UCD format 4 to 6 hex digits.
    575     mParsedCodePointSet->insert(c);   
    576 }
    577 
    578 
    579    
     617    mParsedCodePointSet->insert(c);
     618}
    580619re::CC * grepCodepoints(re::RE * pattern, char * UnicodeDataBuffer, size_t bufferLength) {
    581     //parsedCodePointSet = re::makeCC();       
    582     const unsigned segmentSize = 8;
    583 
    584     ParabixDriver pxDriver("codepointEngine");
    585     auto & idb = pxDriver.getBuilder();
    586     Module * M = idb->getModule();
    587    
    588     Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getVoidTy(), idb->getInt8PtrTy(), idb->getSizeTy(), nullptr));
    589     mainFunc->setCallingConv(CallingConv::C);
    590     auto args = mainFunc->arg_begin();
    591     Value * const buffer = &*(args++);
    592     buffer->setName("buffer");
    593     Value * length = &*(args++);
    594     length->setName("length");
    595    
    596     idb->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", mainFunc, 0));
    597    
    598     StreamSetBuffer * ByteStream = pxDriver.addBuffer(make_unique<SourceBuffer>(idb, idb->getStreamSetTy(1, 8)));
    599     kernel::Kernel * sourceK = pxDriver.addKernelInstance(make_unique<kernel::MemorySourceKernel>(idb, idb->getInt8PtrTy(), segmentSize));
    600     sourceK->setInitialArguments({buffer, length});
    601     pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
    602    
    603     StreamSetBuffer * BasisBits = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(8, 1), segmentSize));
    604    
    605     kernel::Kernel * s2pk = pxDriver.addKernelInstance(make_unique<kernel::S2PKernel>(idb));
    606     pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
    607    
    608     kernel::Kernel * linebreakK = pxDriver.addKernelInstance(make_unique<kernel::LineBreakKernelBuilder>(idb, 8));
    609     StreamSetBuffer * LineBreakStream = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize));
    610     pxDriver.makeKernelCall(linebreakK, {BasisBits}, {LineBreakStream});
    611    
    612     kernel::Kernel * requiredStreamsK = pxDriver.addKernelInstance(make_unique<kernel::RequiredStreams_UTF8>(idb));
    613     StreamSetBuffer * RequiredStreams = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(4, 1), segmentSize));
    614     pxDriver.makeKernelCall(requiredStreamsK, {BasisBits}, {RequiredStreams});
    615    
    616     StreamSetBuffer * MatchResults = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize));
    617     kernel::Kernel * icgrepK = pxDriver.addKernelInstance(make_unique<kernel::ICGrepKernel>(idb, pattern));
    618     pxDriver.makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults});
    619    
    620     StreamSetBuffer * MatchedLines = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize));
    621     kernel::Kernel * matchedLinesK = pxDriver.addKernelInstance(make_unique<kernel::MatchedLinesKernel>(idb));
    622     pxDriver.makeKernelCall(matchedLinesK, {MatchResults, LineBreakStream}, {MatchedLines});
    623    
    624     kernel::Kernel * scanMatchK = pxDriver.addKernelInstance(make_unique<kernel::ScanMatchKernel>(idb, GrepType::CallBack, 8));
     620   
    625621    CodepointAccumulator accum(UnicodeDataBuffer);
    626     intptr_t accum_addr = (intptr_t) &accum;
    627     scanMatchK->setInitialArguments({ConstantInt::get(idb->getIntAddrTy(), accum_addr)});
    628     pxDriver.makeKernelCall(scanMatchK, {MatchedLines, LineBreakStream, ByteStream}, {});
    629     pxDriver.generatePipelineIR();
    630     pxDriver.deallocateBuffers();
    631     idb->CreateRetVoid();
    632     pxDriver.finalizeObject();
    633    
    634     typedef void (*GrepFunctionType)(const char * buffer, const size_t length);
    635     auto f = reinterpret_cast<GrepFunctionType>(pxDriver.getMain());
    636     f(UnicodeDataBuffer, bufferLength);
    637    
    638     //return parsedCodePointSet;
     622   
     623    grepBuffer(pattern, UnicodeDataBuffer, bufferLength, & accum);
    639624    return accum.getCodePoints();
    640625}
    641626
    642    
    643 static std::vector<std::string> parsedPropertyValues;
    644 
    645 void insert_property_values(size_t lineNum, size_t line_start, size_t line_end, const char * buffer) {
     627
     628class PropertyValueAccumulator : public MatchAccumulator {
     629public:
     630   
     631    PropertyValueAccumulator(const char * searchBuffer, std::vector<std::string> & accumulatedPropertyValues)
     632       : mSearchBuffer(searchBuffer), mParsedPropertyValueSet(accumulatedPropertyValues) {}
     633   
     634    void accumulate_match(const size_t lineNum, size_t line_start, size_t line_end) override;
     635private:
     636    const char * mSearchBuffer;
     637    std::vector<std::string> & mParsedPropertyValueSet;
     638};
     639void PropertyValueAccumulator::accumulate_match(const size_t lineNum, size_t line_start, size_t line_end) {
    646640    assert (line_start <= line_end);
    647     parsedPropertyValues.emplace_back(buffer + line_start, buffer + line_end);
    648 }
    649 
    650 const std::vector<std::string> & grepPropertyValues(const std::string& propertyName, re::RE * propertyValuePattern) {
     641    mParsedPropertyValueSet.emplace_back(mSearchBuffer + line_start, mSearchBuffer + line_end);
     642}
     643
     644
     645const std::vector<std::string> grepPropertyValues(const std::string& propertyName, re::RE * propertyValuePattern) {
    651646    ParabixDriver pxDriver("propertyValueEngine");
    652647    AlignedAllocator<char, 32> alloc;
    653 
    654     parsedPropertyValues.clear();
     648    std::vector<std::string> accumulatedValues;
    655649
    656650    const std::string & str = UCD::getPropertyValueGrepString(propertyName);
     
    667661    std::memset(aligned + n, 0, m);
    668662
    669     Module * M = idb->getModule();
    670    
    671     Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getVoidTy(), idb->getInt8PtrTy(), idb->getSizeTy(), nullptr));
    672     mainFunc->setCallingConv(CallingConv::C);
    673     auto args = mainFunc->arg_begin();
    674     Value * const buffer = &*(args++);
    675     buffer->setName("buffer");
    676     Value * length = &*(args++);
    677     length->setName("length");
    678    
    679     idb->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", mainFunc, 0));
    680    
    681     StreamSetBuffer * ByteStream = pxDriver.addBuffer(make_unique<SourceBuffer>(idb, idb->getStreamSetTy(1, 8)));
    682     kernel::Kernel * sourceK = pxDriver.addKernelInstance(make_unique<kernel::MemorySourceKernel>(idb, idb->getInt8PtrTy(), segmentSize));
    683     sourceK->setInitialArguments({buffer, length});
    684     pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
    685    
    686     StreamSetBuffer * BasisBits = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(8, 1), segmentSize));
    687    
    688     kernel::Kernel * s2pk = pxDriver.addKernelInstance(make_unique<kernel::S2PKernel>(idb));
    689     pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
    690    
    691     kernel::Kernel * linebreakK = pxDriver.addKernelInstance(make_unique<kernel::LineBreakKernelBuilder>(idb, 8));
    692     StreamSetBuffer * LineBreakStream = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize));
    693     pxDriver.makeKernelCall(linebreakK, {BasisBits}, {LineBreakStream});
    694    
    695     kernel::Kernel * requiredStreamsK = pxDriver.addKernelInstance(make_unique<kernel::RequiredStreams_UTF8>(idb));
    696     StreamSetBuffer * RequiredStreams = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(4, 1), segmentSize));
    697     pxDriver.makeKernelCall(requiredStreamsK, {BasisBits}, {RequiredStreams});
    698    
    699     StreamSetBuffer * MatchResults = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize));
    700     kernel::Kernel * icgrepK = pxDriver.addKernelInstance(make_unique<kernel::ICGrepKernel>(idb, propertyValuePattern));
    701     pxDriver.makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults});
    702 
    703     StreamSetBuffer * MatchedLines = pxDriver.addBuffer(make_unique<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize));
    704     kernel::Kernel * matchedLinesK = pxDriver.addKernelInstance(make_unique<kernel::MatchedLinesKernel>(idb));
    705     pxDriver.makeKernelCall(matchedLinesK, {MatchResults, LineBreakStream}, {MatchedLines});
    706 
    707     kernel::Kernel * scanMatchK = pxDriver.addKernelInstance(make_unique<kernel::ScanMatchKernel>(idb, GrepType::PropertyValue, 8));
    708     scanMatchK->setInitialArguments({idb->getInt32(0)});
    709     pxDriver.makeKernelCall(scanMatchK, {MatchedLines, LineBreakStream, ByteStream}, {});
    710     pxDriver.LinkFunction(*scanMatchK, "matcher", &insert_property_values);
    711     pxDriver.generatePipelineIR();
    712     pxDriver.deallocateBuffers();
    713     idb->CreateRetVoid();
    714     pxDriver.finalizeObject();
    715 
    716     typedef void (*GrepFunctionType)(const char * buffer, const size_t length);
    717     auto f = reinterpret_cast<GrepFunctionType>(pxDriver.getMain());
    718     f(aligned, n);
    719    
     663    PropertyValueAccumulator accum(aligned, accumulatedValues);
     664    grepBuffer(propertyValuePattern, aligned, n, & accum);
    720665    alloc.deallocate(aligned, 0);
    721     return parsedPropertyValues;
    722 }
    723 
    724    
    725 }
     666    return accumulatedValues;
     667}
     668}
  • icGREP/icgrep-devel/icgrep/grep_engine.h

    r5620 r5678  
    2424void *DoGrepThreadFunction(void *args);
    2525   
     26   
     27class MatchAccumulator {
     28public:
     29    MatchAccumulator() {};
     30    virtual void accumulate_match(const size_t lineNum, size_t line_start, size_t line_end) = 0;
     31};
     32
     33void accumulate_match_wrapper(intptr_t accum_addr, const size_t lineNum, size_t line_start, size_t line_end);
     34   
     35
    2636class GrepEngine {
    2737public:
     
    5161void PrintResults();
    5262   
     63//void grepBuffer(re::RE * pattern, const char * buffer, size_t bufferLength, MatchAccumulator * accum);
    5364
    5465re::CC * grepCodepoints(re::RE * pattern, char * UnicodeDataBuffer, size_t bufferLength);
    5566   
    56 const std::vector<std::string> & grepPropertyValues(const std::string& propertyName, re::RE * propertyValuePattern);
     67const std::vector<std::string> grepPropertyValues(const std::string& propertyName, re::RE * propertyValuePattern);
    5768
    5869}
  • icGREP/icgrep-devel/icgrep/kernels/scanmatchgen.cpp

    r5676 r5678  
    1010#include <IR_Gen/FunctionTypeBuilder.h>
    1111#include <llvm/Support/raw_ostream.h>
     12#include <grep_engine.h>
    1213
    1314using namespace llvm;
     
    3536}
    3637   
    37 void accumulate_match_wrapper(intptr_t accum_addr, const size_t lineNum, size_t line_start, size_t line_end) {
    38     reinterpret_cast<MatchAccumulator *>(accum_addr)->accumulate_match(lineNum, line_start, line_end);
    39 }
    40 
    41 
    4238void ScanMatchKernel::generateMultiBlockLogic(const std::unique_ptr<KernelBuilder> &iBuilder) {
    4339
     
    158154            Value * matchRecordEnd = iBuilder->CreateAdd(phiScanwordPos, iBuilder->CreateCountForwardZeroes(phiMatchWord));
    159155            if (mGrepType == GrepType::CallBack) {
    160                 Function * dispatcher = iBuilder->LinkFunction<void (intptr_t, size_t, size_t, size_t)>("accumulate_match_wrapper", & accumulate_match_wrapper);
     156                Function * dispatcher = m->getFunction("accumulate_match_wrapper"); assert (dispatcher);
     157                //Function * dispatcher = iBuilder->LinkFunction<void (intptr_t, size_t, size_t, size_t)>("accumulate_match_wrapper", & grep::accumulate_match_wrapper);
    161158                Value * accumulator = iBuilder->getScalarField("accumulator_address");
    162159                iBuilder->CreateCall(dispatcher, {accumulator, matchRecordNum, matchRecordStart, matchRecordEnd});
  • icGREP/icgrep-devel/icgrep/kernels/scanmatchgen.h

    r5676 r5678  
    1515   
    1616
    17 class MatchAccumulator {
    18 public:
    19     MatchAccumulator() {};
    20     virtual void accumulate_match(const size_t lineNum, size_t line_start, size_t line_end) = 0;
    21 };
    22 
    23    
    2417class ScanMatchKernel : public MultiBlockKernel {
    2518public:
Note: See TracChangeset for help on using the changeset viewer.