- Timestamp:
- Feb 8, 2018, 9:32:11 PM (13 months ago)
- Location:
- icGREP/icgrep-devel/icgrep
- Files:
-
- 5 edited
Legend:
- Unmodified
- Added
- Removed
-
icGREP/icgrep-devel/icgrep/grep_engine.cpp
r5861 r5867 137 137 138 138 StreamSetBuffer * LineBreakStream = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize); 139 StreamSetBuffer * CRLFStream = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize);140 kernel::Kernel * linebreakK = mGrepDriver->addKernelInstance<kernel::LineBreakKernelBuilder>(idb, encodingBits);141 mGrepDriver->makeKernelCall(linebreakK, {BasisBits, LineFeedStream}, {LineBreakStream, CRLFStream});142 139 143 140 kernel::Kernel * requiredStreamsK = mGrepDriver->addKernelInstance<kernel::RequiredStreams_UTF8>(idb); 144 StreamSetBuffer * RequiredStreams = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy( 3, 1), baseBufferSize);145 mGrepDriver->makeKernelCall(requiredStreamsK, {BasisBits }, {RequiredStreams});141 StreamSetBuffer * RequiredStreams = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize); 142 mGrepDriver->makeKernelCall(requiredStreamsK, {BasisBits, LineFeedStream}, {RequiredStreams, LineBreakStream}); 146 143 147 144 const auto n = REs.size(); … … 156 153 if (UnicodeSets.size() <= 1) { 157 154 kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, REs[i]); 158 mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, CRLFStream,RequiredStreams}, {MatchResults});155 mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults}); 159 156 MatchResultsBufs[i] = MatchResults; 160 157 } else { … … 169 166 // mGrepDriver->makeKernelCall(ccK, {ByteStream}, {CharClasses}); 170 167 kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, REs[i], std::vector<cc::Alphabet *>{mpx.get()}); 171 mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, CRLFStream,RequiredStreams, CharClasses}, {MatchResults});168 mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams, CharClasses}, {MatchResults}); 172 169 MatchResultsBufs[i] = MatchResults; 173 170 } … … 175 172 StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize); 176 173 kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, REs[i]); 177 mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, CRLFStream,RequiredStreams}, {MatchResults});174 mGrepDriver->makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults}); 178 175 MatchResultsBufs[i] = MatchResults; 179 176 } -
icGREP/icgrep-devel/icgrep/kernels/grep_kernel.cpp
r5863 r5867 43 43 PabloBuilder pb(getEntryScope()); 44 44 cc::Parabix_CC_Compiler ccc(this, getInputStreamSet("basis")); 45 46 PabloAST * const LF = pb.createExtract(getInput(1), pb.getInteger(0), "LF"); 47 PabloAST * const CR = ccc.compileCC(makeByte(0x0D)); 48 PabloAST * const LF_VT_FF_CR = ccc.compileCC("LF,VT,FF,CR", makeByte(0x0A, 0x0D), pb); 49 Var * const LineBreak = pb.createVar("LineBreak", LF_VT_FF_CR); 50 51 // Remove the CR of any CR+LF 52 Var * const CRLF = pb.createVar("CRLF", pb.createZeroes()); 53 auto crb = pb.createScope(); 54 pb.createIf(CR, crb); 55 PabloAST * const lookaheadLF = crb.createLookahead(LF, 1, "lookaheadLF"); 56 PabloAST * const crlf = crb.createAnd(CR, lookaheadLF); 57 crb.createAssign(CRLF, crlf); 58 PabloAST * removedCRLF = crb.createAnd(LineBreak, crb.createNot(CRLF)); 59 crb.createAssign(LineBreak, removedCRLF); 60 61 45 62 Zeroes * const ZEROES = pb.createZeroes(); 46 63 PabloAST * const u8pfx = ccc.compileCC(makeByte(0xC0, 0xFF)); … … 64 81 it.createIf(u8pfx2, it2); 65 82 it2.createAssign(anyscope, it2.createAdvance(u8pfx2, 1)); 83 PabloAST * NEL = it2.createAnd(it2.createAdvance(ccc.compileCC(makeByte(0xC2), it2), 1), ccc.compileCC(makeByte(0x85), it2), "NEL"); 84 it2.createAssign(LineBreak, it2.createOr(LineBreak, NEL)); 85 66 86 67 87 // … … 79 99 PabloAST * const EX_invalid = it3.createOr(E0_invalid, ED_invalid); 80 100 it3.createAssign(EF_invalid, EX_invalid); 81 101 PabloAST * E2_80 = it3.createAnd(it3.createAdvance(ccc.compileCC(makeByte(0xE2), it3), 1), ccc.compileCC(makeByte(0x80), it3)); 102 PabloAST * LS_PS = it3.createAnd(it3.createAdvance(E2_80, 1), ccc.compileCC(makeByte(0xA8,0xA9), it3), "LS_PS"); 103 it3.createAssign(LineBreak, it3.createOr(LineBreak, LS_PS)); 82 104 83 105 // … … 110 132 // 111 133 it.createAssign(nonFinal, it.createAnd(nonFinal, u8valid)); 112 113 Var * const required = getOutputStreamVar("required"); 134 pb.createAssign(nonFinal, pb.createOr(nonFinal, CRLF)); 135 PabloAST * unterminatedLineAtEOF = pb.createAtEOF(pb.createAdvance(pb.createNot(LineBreak), 1), "unterminatedLineAtEOF"); 136 137 Var * const required = getOutputStreamVar("nonFinal"); 114 138 pb.createAssign(pb.createExtract(required, pb.getInteger(0)), nonFinal); 139 pb.createAssign(pb.createExtract(getOutputStreamVar("linebreak"), pb.getInteger(0)), pb.createOr(LineBreak, unterminatedLineAtEOF, "EOL")); 115 140 } 116 141 … … 118 143 : PabloKernel(kb, "RequiredStreams_UTF8", 119 144 // input 120 {Binding{kb->getStreamSetTy(8), "basis"}}, 121 // output 122 {Binding{kb->getStreamSetTy(1), "required", FixedRate()}}) { 145 {Binding{kb->getStreamSetTy(8), "basis"}, Binding{kb->getStreamSetTy(1), "lf", FixedRate(), LookAhead(1)}}, 146 // output 147 {Binding{kb->getStreamSetTy(1), "nonFinal", FixedRate()}, 148 Binding{kb->getStreamSetTy(1), "linebreak", FixedRate(), Add1()}}) { 123 149 124 150 } … … 172 198 Binding{b->getStreamSetTy(8), "basis"}, 173 199 Binding{b->getStreamSetTy(1, 1), "linebreak"}, 174 Binding{b->getStreamSetTy(1, 1), "cr+lf"},175 200 Binding{b->getStreamSetTy(1, 1), "required"} 176 201 }; -
icGREP/icgrep-devel/icgrep/re/re_compiler.cpp
r5863 r5867 21 21 #include <re/re_intersect.h> // for Intersect 22 22 #include <re/re_name.h> // for Name, Name::Type, Name::Type::Zer... 23 #include <re/re_name_resolve.h> // for resolveNames24 #include <re/re_name_gather.h> // for gatherNames25 23 #include <re/re_rep.h> // for Rep, Rep::::UNBOUNDED_REP 26 24 #include <re/re_seq.h> // for Seq … … 556 554 inline MarkerType RE_Compiler::compileEnd(MarkerType marker, pablo::PabloBuilder & pb) { 557 555 PabloAST * const nextPos = markerVar(AdvanceMarker(marker, FinalPostPositionUnit, pb)); 558 PabloAST * const atEOL = pb.createOr(pb.createAnd(mLineBreak, nextPos), pb.createAdvance(pb.createAnd(nextPos, mCRLF), 1), "eol"); 556 PabloAST * const atEOL = pb.createAnd(mLineBreak, nextPos, "eol"); 557 //PabloAST * const atEOL = pb.createOr(pb.createAnd(mLineBreak, nextPos), pb.createAdvance(pb.createAnd(nextPos, mCRLF), 1), "eol"); 559 558 return makeMarker(FinalPostPositionUnit, atEOL); 560 559 } … … 590 589 , mCCCompiler(ccCompiler) 591 590 , mLineBreak(nullptr) 592 , mCRLF(nullptr)593 591 , mNonFinal(nullptr) 594 592 , mFinal(nullptr) … … 599 597 Var * const linebreak = mKernel->getInputStreamVar("linebreak"); 600 598 mLineBreak = mPB.createExtract(linebreak, 0); 601 Var * const crlf = mKernel->getInputStreamVar("cr+lf");602 mCRLF = mPB.createExtract(crlf, 0);603 599 Var * const required = mKernel->getInputStreamVar("required"); 604 600 mNonFinal = mPB.createExtract(required, 0); -
icGREP/icgrep-devel/icgrep/re/re_compiler.h
r5843 r5867 137 137 cc::CC_Compiler & mCCCompiler; 138 138 pablo::PabloAST * mLineBreak; 139 pablo::PabloAST * mCRLF;140 pablo::PabloAST * mInitial;141 139 pablo::PabloAST * mNonFinal; 142 140 pablo::PabloAST * mFinal; -
icGREP/icgrep-devel/icgrep/toolchain/grep_pipeline.cpp
r5865 r5867 63 63 pxDriver.makeKernelCall(sourceK, {}, {ByteStream}); 64 64 65 StreamSetBuffer * LineFeedStream = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);66 #ifdef USE_DIRECT_LF_BUILDER67 kernel::Kernel * linefeedK = pxDriver.addKernelInstance<kernel::LineFeedKernelBuilder>(idb, Binding{idb->getStreamSetTy(1, 8), "byteStream", FixedRate(), Principal()});68 pxDriver.makeKernelCall(linefeedK, {ByteStream}, {LineFeedStream});69 #endif70 65 71 66 StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(8, 1), segmentSize); … … 73 68 pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits}); 74 69 75 #ifndef USE_DIRECT_LF_BUILDER70 StreamSetBuffer * LineFeedStream = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize); 76 71 kernel::Kernel * linefeedK = pxDriver.addKernelInstance<kernel::LineFeedKernelBuilder>(idb, Binding{idb->getStreamSetTy(8), "basis", FixedRate(), Principal()}); 77 72 pxDriver.makeKernelCall(linefeedK, {BasisBits}, {LineFeedStream}); 78 #endif79 73 80 kernel::Kernel * linebreakK = pxDriver.addKernelInstance<kernel::LineBreakKernelBuilder>(idb, 8);81 74 StreamSetBuffer * LineBreakStream = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize); 82 StreamSetBuffer * CRLFStream = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);83 pxDriver.makeKernelCall(linebreakK, {BasisBits, LineFeedStream}, {LineBreakStream, CRLFStream});84 75 85 76 kernel::Kernel * requiredStreamsK = pxDriver.addKernelInstance<kernel::RequiredStreams_UTF8>(idb); 86 StreamSetBuffer * RequiredStreams = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy( 3, 1), segmentSize);87 pxDriver.makeKernelCall(requiredStreamsK, {BasisBits }, {RequiredStreams});77 StreamSetBuffer * RequiredStreams = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(2, 1), segmentSize); 78 pxDriver.makeKernelCall(requiredStreamsK, {BasisBits, LineFeedStream}, {RequiredStreams, LineBreakStream}); 88 79 89 80 StreamSetBuffer * MatchResults = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize); 90 81 kernel::Kernel * icgrepK = pxDriver.addKernelInstance<kernel::ICGrepKernel>(idb, pattern); 91 pxDriver.makeKernelCall(icgrepK, {BasisBits, LineBreakStream, CRLFStream,RequiredStreams}, {MatchResults});82 pxDriver.makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults}); 92 83 93 84 StreamSetBuffer * MatchedLines = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
Note: See TracChangeset
for help on using the changeset viewer.