Ignore:
Timestamp:
Jan 14, 2018, 3:30:04 PM (16 months ago)
Author:
nmedfort
Message:

Potential bug fix for 32-bit

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep_engine.cpp

    r5824 r5831  
    9797//
    9898// All engines share a common pipeline to compute a stream of Matches from a given input Bytestream.
    99 //#define USE_DIRECT_LF_BUILDER 1
     99
     100unsigned LLVM_READNONE calculateMaxCountRate(const std::unique_ptr<kernel::KernelBuilder> & b) {
     101    const unsigned packSize = b->getSizeTy()->getBitWidth();
     102    return (packSize * packSize) / b->getBitBlockWidth();
     103}
    100104
    101105std::pair<StreamSetBuffer *, StreamSetBuffer *> GrepEngine::grepPipeline(std::vector<re::RE *> & REs, StreamSetBuffer * ByteStream) {
     
    103107    const unsigned segmentSize = codegen::SegmentSize;
    104108    const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
     109    // TODO: until we automate stream buffer sizing, use this calculation to determine how large our matches buffer needs to be.
     110    const unsigned baseBufferSize = segmentSize * (MaxCountFlag > 0 ? (std::max(bufferSegments, calculateMaxCountRate(idb))) : bufferSegments);
    105111    const unsigned encodingBits = 8;
    106112
    107     StreamSetBuffer * BasisBits = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(encodingBits, 1), segmentSize * bufferSegments);
     113    StreamSetBuffer * LineFeedStream = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
     114
     115    #ifdef USE_DIRECT_LF_BUILDER
     116    kernel::Kernel * linefeedK = mGrepDriver->addKernelInstance<kernel::LineFeedKernelBuilder>(idb, encodingBits);
     117    mGrepDriver->makeKernelCall(linefeedK, {ByteStream}, {LineFeedStream});
     118    #endif
     119
     120    StreamSetBuffer * BasisBits = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(encodingBits, 1), baseBufferSize);
    108121    kernel::Kernel * s2pk = mGrepDriver->addKernelInstance<kernel::S2PKernel>(idb);
    109122    mGrepDriver->makeKernelCall(s2pk, {ByteStream}, {BasisBits});
    110123
    111     StreamSetBuffer * LineFeedStream = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
    112 #ifdef USE_DIRECT_LF_BUILDER
    113     kernel::Kernel * linefeedK = mGrepDriver->addKernelInstance<kernel::DirectLineFeedBuilder>(idb);
    114     mGrepDriver->makeKernelCall(linefeedK, {ByteStream}, {LineFeedStream});
    115 #else
     124    #ifndef USE_DIRECT_LF_BUILDER
    116125    kernel::Kernel * linefeedK = mGrepDriver->addKernelInstance<kernel::LineFeedKernelBuilder>(idb, encodingBits);
    117126    mGrepDriver->makeKernelCall(linefeedK, {BasisBits}, {LineFeedStream});
    118 #endif
    119 
    120     StreamSetBuffer * LineBreakStream = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
    121     StreamSetBuffer * CRLFStream = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
     127    #endif
     128
     129    StreamSetBuffer * LineBreakStream = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
     130    StreamSetBuffer * CRLFStream = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    122131    kernel::Kernel * linebreakK = mGrepDriver->addKernelInstance<kernel::LineBreakKernelBuilder>(idb, encodingBits);
    123132    mGrepDriver->makeKernelCall(linebreakK, {BasisBits, LineFeedStream}, {LineBreakStream, CRLFStream});
    124133
    125134    kernel::Kernel * requiredStreamsK = mGrepDriver->addKernelInstance<kernel::RequiredStreams_UTF8>(idb);
    126     StreamSetBuffer * RequiredStreams = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(3, 1), segmentSize * bufferSegments);
     135    StreamSetBuffer * RequiredStreams = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(3, 1), baseBufferSize);
    127136    mGrepDriver->makeKernelCall(requiredStreamsK, {BasisBits}, {RequiredStreams});
    128137
    129138    const auto n = REs.size();
    130139    std::vector<StreamSetBuffer *> MatchResultsBufs(n);
    131 
    132140    for(unsigned i = 0; i < n; ++i) {
    133141        REs[i] = resolveModesAndExternalSymbols(REs[i]);
     
    137145#ifdef USE_MULTIPLEX_CC
    138146        const std::vector<const re::CC *> UnicodeSets = re::collectUnicodeSets(REs[i]);
     147
     148        StreamSetBuffer * const MatchResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    139149        if (UnicodeSets.size() <= 1) {
    140             StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
    141150            kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, REs[i]);
    142151            mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, CRLFStream, RequiredStreams}, {MatchResults});
    143152            MatchResultsBufs[i] = MatchResults;
    144         }
    145         else {
     153        } else {
    146154            mpx = make_unique<MultiplexedAlphabet>("mpx", UnicodeSets);
    147155            REs[i] = transformCCs(mpx.get(), REs[i]);
    148156            std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
    149157            auto numOfCharacterClasses = mpx_basis.size();
    150             StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), segmentSize * bufferSegments);
     158            StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize);
    151159            kernel::Kernel * ccK = mGrepDriver->addKernelInstance<kernel::CharClassesKernel>(idb, std::move(mpx_basis));
    152             mGrepDriver->makeKernelCall(ccK, {BasisBits}, {CharClasses});
    153             StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
     160            mGrepDriver->makeKernelCall(ccK, {BasisBits}, {CharClasses});           
    154161            kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, REs[i], std::vector<cc::Alphabet *>{mpx.get()});
    155162            mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, CRLFStream, RequiredStreams, CharClasses}, {MatchResults});
     
    157164        }
    158165#else
    159         StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
     166        StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    160167        kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, REs[i]);
    161168        mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, CRLFStream, RequiredStreams}, {MatchResults});
     
    165172    StreamSetBuffer * MergedResults = MatchResultsBufs[0];
    166173    if (REs.size() > 1) {
    167         MergedResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
     174        MergedResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    168175        kernel::Kernel * streamsMergeK = mGrepDriver->addKernelInstance<kernel::StreamsMerge>(idb, 1, REs.size());
    169176        mGrepDriver->makeKernelCall(streamsMergeK, MatchResultsBufs, {MergedResults});
     
    174181        StreamSetBuffer * OriginalMatches = Matches;
    175182        kernel::Kernel * matchedLinesK = mGrepDriver->addKernelInstance<kernel::MatchedLinesKernel>(idb);
    176         Matches = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
     183        Matches = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    177184        mGrepDriver->makeKernelCall(matchedLinesK, {OriginalMatches, LineBreakStream}, {Matches});
    178185    }
     
    181188        kernel::Kernel * invertK = mGrepDriver->addKernelInstance<kernel::InvertMatchesKernel>(idb);
    182189        StreamSetBuffer * OriginalMatches = Matches;
    183         Matches = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
     190        Matches = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    184191        mGrepDriver->makeKernelCall(invertK, {OriginalMatches, LineBreakStream}, {Matches});
    185192    }
     
    187194        kernel::Kernel * untilK = mGrepDriver->addKernelInstance<kernel::UntilNkernel>(idb);
    188195        untilK->setInitialArguments({idb->getSize(MaxCountFlag)});
    189         StreamSetBuffer * AllMatches = Matches;
    190         Matches = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize * bufferSegments);
     196        StreamSetBuffer * const AllMatches = Matches;
     197        Matches = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);
    191198        mGrepDriver->makeKernelCall(untilK, {AllMatches}, {Matches});
    192199    }
Note: See TracChangeset for help on using the changeset viewer.