Changeset 6218 for icGREP

Timestamp:

Dec 6, 2018, 7:26:06 AM (2 months ago)

Author:

cameron

Message:

Grep Kernel Options

Location:

icGREP/icgrep-devel/icgrep

Files:

• grep/grep_engine.cpp (modified) (14 diffs)
• kernels/grep_kernel.cpp (modified) (4 diffs)
• kernels/grep_kernel.h (modified) (1 diff)

icGREP/icgrep-devel/icgrep/grep/grep_engine.cpp

@@ -81 +81 @@
 namespace grep {
 
-using Alphabets = ICGrepKernel::Alphabets;
-
 void GrepCallBackObject::handle_signal(unsigned s) {
     if (static_cast<GrepSignal>(s) == GrepSignal::BinaryFile) {
@@ -288 +286 @@
     Component internalComponents = Component::NoComponents;
 
+    
 
     if (isSimple) {
@@ -295 +294 @@
             setComponent(internalComponents, Component::MoveMatchesToEOL);
         }
+        std::unique_ptr<GrepKernelOptions> options = make_unique<GrepKernelOptions>();
         const auto isWithinByteTestLimit = byteTestsWithinLimit(mREs[0], ByteCClimit);
         const auto hasTriCC = hasTriCCwithinLimit(mREs[0], ByteCClimit, prefixRE, suffixRE);
-        ICGrepKernel::Externals externals;
         if (isWithinByteTestLimit || hasTriCC) {
             if (MultithreadedSimpleRE && hasTriCC) {
@@ -307 +306 @@
                     StreamSet * const ccStream = P->CreateStreamSet(1, 1);
                     P->CreateKernelCall<DirectCharacterClassKernelBuilder>(ccNameStr, std::vector<re::CC *>{cc}, ByteStream, ccStream);
-                    externals.emplace_back(ccNameStr, ccStream);
+                    options->addExternal(ccNameStr, ccStream);
                 }
             }
@@ -313 +312 @@
             MatchResultsBufs[0] = MatchResults;
             if (isWithinByteTestLimit) {
-                P->CreateKernelCall<ICGrepKernel>(mREs[0], ByteStream, MatchResults, externals);
+                options->setRE(mREs[0]);
+                options->setSource(ByteStream);
+                options->setResults(MatchResults);
+                P->CreateKernelCall<ICGrepKernel>(std::move(options));
             } else {
-                P->CreateKernelCall<ByteBitGrepKernel>(prefixRE, suffixRE, ByteStream, MatchResults, externals);
+                //P->CreateKernelCall<ByteBitGrepKernel>(prefixRE, suffixRE, ByteStream, MatchResults, externals);
+                options->setPrefixRE(prefixRE);
+                options->setRE(suffixRE);
+                options->setSource(ByteStream);
+                options->setResults(MatchResults);
+                P->CreateKernelCall<ICGrepKernel>(std::move(options));
             }
             Kernel * LB_nullK = P->CreateKernelCall<DirectCharacterClassKernelBuilder>( "breakCC", std::vector<re::CC *>{mBreakCC}, ByteStream, LineBreakStream, callbackObject);
@@ -329 +336 @@
             P->CreateKernelCall<S2P_PabloKernel>(ByteStream, BasisBits);
         } else {
+            //P->CreateKernelCall<S2PKernel>(ByteStream, BasisBits);
             Kernel * s2pK = P->CreateKernelCall<S2PKernel>(ByteStream, BasisBits, cc::BitNumbering::LittleEndian, callbackObject);
             mGrepDriver.LinkFunction(s2pK, "signal_dispatcher", kernel::signal_dispatcher);
@@ -366 +374 @@
 
         for(unsigned i = 0; i < numOfREs; ++i) {
-            ICGrepKernel::Externals externals;
+            std::unique_ptr<GrepKernelOptions> options = make_unique<GrepKernelOptions>();
             if (mGrepRecordBreak == GrepRecordBreakKind::Unicode) {
-                externals.emplace_back("UTF8_LB", LineBreakStream);
-                externals.emplace_back("UTF8_nonfinal", RequiredStreams);
+                options->addExternal("UTF8_LB", LineBreakStream);
+                options->addExternal("UTF8_nonfinal", RequiredStreams);
             }
             std::set<re::Name *> UnicodeProperties;
@@ -380 +388 @@
                         report_fatal_error(name + " not found");
                     }
-                    externals.emplace_back(name, f->second);
+                    options->addExternal(name, f->second);
                 }
             }
             if (hasGCB[i]) { assert (GCB_stream);
-                externals.emplace_back("\\b{g}", GCB_stream);
+                options->addExternal("\\b{g}", GCB_stream);
             }
 
@@ -393 +401 @@
                 const auto UnicodeSets = re::collectCCs(mREs[i], cc::Unicode, std::set<re::Name *>{re::makeZeroWidth("\\b{g}")});
                 if (UnicodeSets.size() <= 1) {
-                    P->CreateKernelCall<ICGrepKernel>(mREs[i], BasisBits, MatchResults, externals);
+                    options->setRE(mREs[i]);
+                    options->setSource(BasisBits);
+                    options->setResults(MatchResults);
                 } else {
                     auto mpx = std::make_shared<MultiplexedAlphabet>("mpx", UnicodeSets);
@@ -404 +414 @@
                     // Multiplexing Grep Kernel is not Cachable, since for now it use string representation of RE AST as cache key,
                     // whileit is possible that two multiplexed REs with the same name "mpx_1" have different alphabets
-
-                    Alphabets alphabets;
-                    alphabets.emplace_back(mpx, CharClasses);
-                    P->CreateKernelCall<ICGrepKernel>(mREs[i], BasisBits, MatchResults, externals, alphabets, cc::BitNumbering::LittleEndian, false);
+                    options->setRE(mREs[i]);
+                    options->setSource(BasisBits);
+                    options->setResults(MatchResults);
+                    options->addAlphabet(mpx, CharClasses);
+                    P->CreateKernelCall<ICGrepKernel>(std::move(options));
                 }
             } else {
-                P->CreateKernelCall<ICGrepKernel>(mREs[i], BasisBits, MatchResults, externals);
+                options->setRE(mREs[i]);
+                options->setSource(BasisBits);
+                options->setResults(MatchResults);
+                P->CreateKernelCall<ICGrepKernel>(std::move(options));
             }
         }
@@ -568 +582 @@
     close(fileDescriptor);
     if (handler.binaryFileSignalled()) {
+        llvm::errs() << "Binary file " << fileName << "\n";
         return 0;
     }
@@ -612 +627 @@
     if (accum.binaryFileSignalled()) {
         accum.mResultStr.clear();
-        accum.mResultStr << "Binary file " << fileName << " skipped.\n";
+        if (!mSuppressFileMessages) {
+            accum.mResultStr << "Binary file " << fileName << " skipped.\n";
+        }
     }
     if (accum.mLineCount > 0) grepMatchFound = true;
@@ -793 +810 @@
     } else {
         StreamSet * MatchResults = E->CreateStreamSet();
-        E->CreateKernelCall<ICGrepKernel>(matchingRE, BasisBits, MatchResults);
+        std::unique_ptr<GrepKernelOptions> options = make_unique<GrepKernelOptions>();
+        options->setRE(matchingRE);
+        options->setSource(BasisBits);
+        options->setResults(MatchResults);
+        E->CreateKernelCall<ICGrepKernel>(std::move(options));
         MatchingRecords = E->CreateStreamSet();
         E->CreateKernelCall<MatchedLinesKernel>(MatchResults, RecordBreakStream, MatchingRecords);
@@ -799 +820 @@
     if (!excludeNothing) {
         StreamSet * ExcludedResults = E->CreateStreamSet();
-        E->CreateKernelCall<ICGrepKernel>(excludedRE, BasisBits, ExcludedResults);
+        std::unique_ptr<GrepKernelOptions> options = make_unique<GrepKernelOptions>();
+        options->setRE(excludedRE);
+        options->setSource(BasisBits);
+        options->setResults(ExcludedResults);
+        E->CreateKernelCall<ICGrepKernel>(std::move(options));
         StreamSet * ExcludedRecords = E->CreateStreamSet();
         E->CreateKernelCall<MatchedLinesKernel>(ExcludedResults, RecordBreakStream, ExcludedRecords);

icGREP/icgrep-devel/icgrep/kernels/grep_kernel.cpp

@@ -5 +5 @@
 
 #include "grep_kernel.h"
+#include <cc/alphabet.h>
 #include <re/printer_re.h>
 #include <re/re_cc.h>
@@ -214 +215 @@
 }
 
-ICGrepSignature::ICGrepSignature(re::RE * const re_ast)
-: mRE(re_ast)
-, mSignature(Printer_RE::PrintRE(mRE)) {
-
-}
-// Helper to compute stream set inputs to pass into PabloKernel constructor.
-Bindings ICGrepKernel::makeInputBindings(StreamSet * const basis, const Externals & externals, const Alphabets & alphabets) {
+void GrepKernelOptions::setNumbering(cc::BitNumbering numbering) {mBasisSetNumbering = numbering;}
+void GrepKernelOptions::setIndexingAlphabet(cc::Alphabet * a) {mIndexingAlphabet = a;}
+void GrepKernelOptions::setRE(RE * e) {mRE = e;}
+void GrepKernelOptions::setPrefixRE(RE * e) {mPrefixRE = e;}
+void GrepKernelOptions::setSource(StreamSet * s) {mSource = s;}
+void GrepKernelOptions::setResults(StreamSet * r) {mResults = r;}
+void GrepKernelOptions::addExternal(std::string name, StreamSet * strm) {
+    mExternals.emplace_back(name, strm);
+}
+void GrepKernelOptions::addAlphabet(std::shared_ptr<cc::Alphabet> a, StreamSet * basis) {
+    mAlphabets.emplace_back(a, basis);
+}
+
+Bindings GrepKernelOptions::streamSetInputBindings() {
     Bindings inputs;
-    inputs.emplace_back("basis", basis);
-    for (const auto & e : externals) {
+    inputs.emplace_back("basis", mSource);
+    for (const auto & e : mExternals) {
         inputs.emplace_back(e.first, e.second);
     }
-    for (const auto & a : alphabets) {
+    for (const auto & a : mAlphabets) {
         inputs.emplace_back(a.first->getName() + "_basis", a.second);
     }
@@ -232 +240 @@
 }
 
-ICGrepKernel::ICGrepKernel(const std::unique_ptr<kernel::KernelBuilder> & b,
-                           RE * const re,
-                           StreamSet * const BasisBits,
-                           StreamSet * const matches,
-                           const Externals externals,
-                           const Alphabets alphabets,
-                           const cc::BitNumbering basisSetNumbering,
-                           const bool cachable)
-: ICGrepSignature(re)
-, PabloKernel(b, "ic" + getStringHash(mSignature),
-// inputs
-makeInputBindings(BasisBits, externals, alphabets),
-// output
-{Binding{"matches", matches, FixedRate(), Add1()}})
-, mExternals(std::move(externals))
-, mAlphabets(std::move(alphabets))
-, mBasisSetNumbering(basisSetNumbering)
-, mIsCachable(cachable) {
+Bindings GrepKernelOptions::streamSetOutputBindings() {
+    return {Binding{"matches", mResults, FixedRate(), Add1()}};
+}
+
+Bindings GrepKernelOptions::scalarInputBindings() {
+    return {};
+}
+
+Bindings GrepKernelOptions::scalarOutputBindings() {
+    return {};
+}
+
+std::string GrepKernelOptions::getSignature() {
+    if (mSignature == "") {
+        mSignature = std::to_string(mSource->getNumElements()) + "x" + std::to_string(mSource->getFieldWidth());
+        mSignature += "/" + mIndexingAlphabet->getName();
+        for (auto e: mExternals) {
+            mSignature += "_" + e.first;
+        }
+        for (auto a: mAlphabets) {
+            mSignature += "_" + a.first->getName();
+        }
+        if (mPrefixRE) {
+            mSignature += ":" + Printer_RE::PrintRE(mPrefixRE);
+        }
+        mSignature += ":" + Printer_RE::PrintRE(mRE);
+    }
+    return mSignature;
+}
+
+ICGrepKernel::ICGrepKernel(const std::unique_ptr<kernel::KernelBuilder> & b, std::unique_ptr<GrepKernelOptions> options)
+: PabloKernel(b, "ic" + getStringHash(options->getSignature()),
+    options->streamSetInputBindings(),
+    options->streamSetOutputBindings(),
+    options->scalarInputBindings(),
+              options->scalarOutputBindings()), mOptions(std::move(options)) {
 }
 
 std::string ICGrepKernel::makeSignature(const std::unique_ptr<kernel::KernelBuilder> &) {
-    return mSignature;
+    return mOptions->getSignature();
 }
 
@@ -263 +289 @@
         ccc = make_unique<cc::Direct_CC_Compiler>(getEntryScope(), pb.createExtract(getInput(0), pb.getInteger(0)));
     } else {
-        ccc = make_unique<cc::Parabix_CC_Compiler>(getEntryScope(), getInputStreamSet("basis"), mBasisSetNumbering);
-    }
-    //cc::Parabix_CC_Compiler ccc(getEntryScope(), getInputStreamSet("basis"), mBasisSetNumbering);
-    RE_Compiler re_compiler(getEntryScope(), *ccc.get(), mBasisSetNumbering);
-    for (const auto & e : mExternals) {
+        ccc = make_unique<cc::Parabix_CC_Compiler>(getEntryScope(), getInputStreamSet("basis"), mOptions->mBasisSetNumbering);
+    }
+    //cc::Parabix_CC_Compiler ccc(getEntryScope(), getInputStreamSet("basis"), mOptions->mBasisSetNumbering);
+    RE_Compiler re_compiler(getEntryScope(), *ccc.get(), mOptions->mBasisSetNumbering);
+    for (const auto & e : mOptions->mExternals) {
         re_compiler.addPrecompiled(e.first, pb.createExtract(getInputStreamVar(e.first), pb.getInteger(0)));
     }
-    for (const auto & a : mAlphabets) {
+    for (const auto & a : mOptions->mAlphabets) {
         auto & alpha = a.first;
         auto mpx_basis = getInputStreamSet(alpha->getName() + "_basis");
         re_compiler.addAlphabet(alpha, mpx_basis);
     }
-    PabloAST * const matches = re_compiler.compile(mRE);
-    Var * const output = getOutputStreamVar("matches");
-    pb.createAssign(pb.createExtract(output, pb.getInteger(0)), matches);
+    if (mOptions->mPrefixRE) {
+        PabloAST * const prefixMatches = re_compiler.compile(mOptions->mPrefixRE);
+        Var * const final_matches = pb.createVar("final_matches", pb.createZeroes());
+        PabloBlock * scope1 = getEntryScope()->createScope();
+        pb.createIf(prefixMatches, scope1);
+        
+        PabloAST * u8bytes = pb.createExtract(getInput(0), pb.getInteger(0));
+        PabloAST * nybbles[2];
+        nybbles[0] = scope1->createPackL(scope1->getInteger(8), u8bytes);
+        nybbles[1] = scope1->createPackH(scope1->getInteger(8), u8bytes);
+        
+        PabloAST * bitpairs[4];
+        for (unsigned i = 0; i < 2; i++) {
+            bitpairs[2*i] = scope1->createPackL(scope1->getInteger(4), nybbles[i]);
+            bitpairs[2*i + 1] = scope1->createPackH(scope1->getInteger(4), nybbles[i]);
+        }
+        
+        std::vector<PabloAST *> basis(8);
+        for (unsigned i = 0; i < 4; i++) {
+            basis[2*i] = scope1->createPackL(scope1->getInteger(2), bitpairs[i]);
+            basis[2*i + 1] = scope1->createPackH(scope1->getInteger(2), bitpairs[i]);
+        }
+        
+        cc::Parabix_CC_Compiler ccc(scope1, basis);
+        RE_Compiler re_compiler(scope1, ccc);
+        scope1->createAssign(final_matches, re_compiler.compile(mOptions->mRE, prefixMatches));
+        Var * const output = getOutputStreamVar("matches");
+        pb.createAssign(pb.createExtract(output, pb.getInteger(0)), final_matches);
+    } else {
+        PabloAST * const matches = re_compiler.compile(mOptions->mRE);
+        Var * const output = getOutputStreamVar("matches");
+        pb.createAssign(pb.createExtract(output, pb.getInteger(0)), matches);
+    }
 }
 

icGREP/icgrep-devel/icgrep/kernels/grep_kernel.h

@@ -52 +52 @@
 
 
-struct ICGrepSignature {
-    ICGrepSignature(re::RE * re_ast);
+class GrepKernelOptions {
+    friend class ICGrepKernel;
+public:
+    using Externals = std::vector<std::pair<std::string, StreamSet *>>;
+    using Alphabets = std::vector<std::pair<std::shared_ptr<cc::Alphabet>, StreamSet *>>;
+    GrepKernelOptions() :
+        mBasisSetNumbering(cc::BitNumbering::LittleEndian),
+        mIndexingAlphabet(&cc::Byte),
+        mPrefixRE(nullptr) {}
+    void setNumbering(cc::BitNumbering numbering);
+    void setIndexingAlphabet(cc::Alphabet * a);
+    void setSource(StreamSet * s);
+    void setResults(StreamSet * r);
+    void addExternal(std::string name, StreamSet * strm);
+    void addAlphabet(std::shared_ptr<cc::Alphabet> a, StreamSet * basis);
+    void setRE(re::RE * re);
+    void setPrefixRE(re::RE * re);
+
 protected:
-    re::RE * const  mRE;
+    Bindings streamSetInputBindings();
+    Bindings streamSetOutputBindings();
+    Bindings scalarInputBindings();
+    Bindings scalarOutputBindings();
+    std::string getSignature();
+
+private:
+    cc::BitNumbering mBasisSetNumbering;
+    const cc::Alphabet * mIndexingAlphabet;
+    StreamSet * mSource;
+    StreamSet * mResults;
+    Externals mExternals;
+    Alphabets mAlphabets;
+    re::RE * mRE;
+    re::RE * mPrefixRE;
     std::string     mSignature;
 };
 
-    
-class ICGrepKernel : public ICGrepSignature, public pablo::PabloKernel {
+
+class ICGrepKernel : public pablo::PabloKernel {
 public:
-
-    using Externals = std::vector<std::pair<std::string, StreamSet *>>;
-    using Alphabets = std::vector<std::pair<std::shared_ptr<cc::Alphabet>, StreamSet *>>;
-
     ICGrepKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder,
-                 re::RE * const re_ast, StreamSet * const BasisBits, StreamSet * const MatchResults,
-                 const Externals externals = {}, const Alphabets alphabets = {},
-                 const cc::BitNumbering basisSetNumbering = cc::BitNumbering::LittleEndian,
-                 const bool cachable = true);
+                 std::unique_ptr<GrepKernelOptions> options);
     std::string makeSignature(const std::unique_ptr<kernel::KernelBuilder> &) override;
-    bool isCachable() const override { return mIsCachable; }
+    bool isCachable() const override { return true; }
     bool hasFamilyName() const override { return true; }
 protected:
     void generatePabloMethod() override;
-private:
-    static Bindings makeInputBindings(StreamSet * const basis, const Externals & externals, const Alphabets & alphabets);
-protected:
-    const Externals mExternals;
-    const Alphabets mAlphabets;
-    const cc::BitNumbering mBasisSetNumbering;
-    const bool mIsCachable;
+    std::unique_ptr<GrepKernelOptions> mOptions;
 };