Changeset 5630

Timestamp:

Sep 7, 2017, 4:56:56 PM (10 months ago)

Author:

nmedfort

Message:

Partial check-in for avoidance of compiling Pablo/LLVM code to determine the Kernel struct type when using a cached object. Inactive RE alternation minimization check in.

Location:

icGREP/icgrep-devel/icgrep

Files:

• CMakeLists.txt (modified) (1 diff)
• IR_Gen/CBuilder.cpp (modified) (8 diffs)
• IR_Gen/CBuilder.h (modified) (1 diff)
• IR_Gen/idisa_nvptx_builder.cpp (modified) (1 diff)
• cc/multiplex_CCs.cpp (modified) (1 diff)
• cc/multiplex_CCs.h (modified) (1 diff)
• editd/editd.cpp (modified) (1 diff)
• icgrep-devel.files (modified) (1 diff)
• icgrep-devel.includes (modified) (1 diff)
• kernels/kernel.cpp (modified) (22 diffs)
• kernels/kernel.h (modified) (3 diffs)
• kernels/lz4_bytestream_decoder.cpp (modified) (1 diff)
• kernels/scanmatchgen.cpp (modified) (1 diff)
• pablo/carry_data.h (modified) (3 diffs)
• pablo/carrypack_manager.cpp (modified) (1 diff)
• pablo/pablo_kernel.cpp (modified) (2 diffs)
• pablo/pablo_kernel.h (modified) (4 diffs)
• re/re_memoizer.cpp (added)
• re/re_memoizer.hpp (modified) (3 diffs)
• re/re_minimizer.cpp (added)
• re/re_minimizer.h (added)
• re/re_name.h (modified) (2 diffs)
• re/re_parser.cpp (modified) (3 diffs)
• re/re_re.h (modified) (2 diffs)
• re/re_reverse.cpp (modified) (1 diff)
• re/re_simplifier.cpp (modified) (3 diffs)
• re/re_toolchain.cpp (modified) (4 diffs)
• toolchain/NVPTXDriver.cpp (modified) (1 diff)
• toolchain/cpudriver.cpp (modified) (6 diffs)
• toolchain/cpudriver.h (modified) (1 diff)
• toolchain/object_cache.cpp (modified) (10 diffs)
• toolchain/object_cache.h (modified) (3 diffs)

icGREP/icgrep-devel/icgrep/CMakeLists.txt

@@ -87 +87 @@
 add_library(PabloADT ${PABLO_SRC})
 add_library(RegExpADT re/re_re.cpp re/re_cc.cpp re/re_rep.cpp re/re_diff.cpp re/re_intersect.cpp re/printer_re.cpp)
-add_library(RegExpCompiler re/re_parser.cpp re/re_nullable.cpp re/re_simplifier.cpp re/re_star_normal.cpp re/re_local.cpp re/re_compiler.cpp re/re_analysis.cpp re/re_toolchain.cpp re/re_name_resolve.cpp re/re_name_gather.cpp re/re_collect_unicodesets.cpp re/re_multiplex.cpp re/re_parser_pcre.cpp re/re_parser_ere.cpp re/re_parser_bre.cpp re/re_parser_prosite.cpp re/re_utility.cpp)
+add_library(RegExpCompiler re/re_parser.cpp re/re_memoizer.cpp re/re_nullable.cpp re/re_simplifier.cpp re/re_star_normal.cpp re/re_minimizer.cpp re/re_local.cpp re/re_compiler.cpp re/re_analysis.cpp re/re_toolchain.cpp re/re_name_resolve.cpp re/re_name_gather.cpp re/re_collect_unicodesets.cpp re/re_multiplex.cpp re/re_parser_pcre.cpp re/re_parser_ere.cpp re/re_parser_bre.cpp re/re_parser_prosite.cpp re/re_utility.cpp)
 add_library(CCADT cc/cc_compiler.cpp utf8_encoder.cpp utf16_encoder.cpp UCD/CaseFolding_txt.cpp cc/alphabet.cpp cc/multiplex_CCs.cpp)
 add_library(UCDlib UCD/unicode_set.cpp UCD/ucd_compiler.cpp UCD/PropertyObjects.cpp UCD/resolve_properties.cpp UCD/UnicodeNameData.cpp)

icGREP/icgrep-devel/icgrep/IR_Gen/CBuilder.cpp

@@ -201 +201 @@
     if (LLVM_UNLIKELY(printRegister == nullptr)) {
         FunctionType *FT = FunctionType::get(getVoidTy(), { getInt8PtrTy(), int64Ty }, false);
-        Function * function = Function::Create(FT, Function::InternalLinkage, "PrintInt", m);
+        Function * function = Function::Create(FT, Function::ExternalLinkage, "PrintInt", m);
         auto arg = function->arg_begin();
         std::string out = "%-40s = %" PRIx64 "\n";
@@ -561 +561 @@
 
 PointerType * CBuilder::getVoidPtrTy() const {
-    return TypeBuilder<void *, true>::get(getContext());
+    return TypeBuilder<void *, false>::get(getContext());
 }
 
@@ -1050 +1050 @@
 }
 
-#define CONCAT(a__, b__) a__##b__
-#define STRINGIFY(a__) #a__
-
 #ifdef HAS_ADDRESS_SANITIZER
 #define CHECK_ADDRESS_SANITIZER(Ptr, Name) \
@@ -1070 +1067 @@
     Value * check = CreateCall(isPoisoned, { addr, size }); \
     check = CreateICmpEQ(check, ConstantPointerNull::get(cast<PointerType>(isPoisoned->getReturnType()))); \
-    CreateAssert(check, STRINGIFY(CONCAT(Name, ": invalid memory address"))); \
+    CreateAssert(check, Name ": invalid memory address"); \
 }
 #else
@@ -1078 +1075 @@
 #define CHECK_ADDRESS(Ptr, Name) \
     if (codegen::EnableAsserts) { \
-        CreateAssert(Ptr, STRINGIFY(CONCAT(Name, ": null pointer address"))); \
+        CreateAssert(Ptr, Name ": null pointer address"); \
         CHECK_ADDRESS_SANITIZER(Ptr, Name) \
     }
@@ -1161 +1158 @@
 }
 
-CallInst * CBuilder::CreateMemMove(Value * Dst, Value * Src, Value *Size, unsigned Align, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
+CallInst * CBuilder::CreateMemMove(Value * Dst, Value * Src, Value *Size, unsigned Align, bool isVolatile,
+                                   MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
     if (codegen::EnableAsserts) {
         DataLayout DL(getModule());
@@ -1167 +1165 @@
         Value * intDst = CreatePtrToInt(Dst, intPtrTy);
         Value * intSrc = CreatePtrToInt(Src, intPtrTy);
+        // If the call to this intrinisic has an alignment value that is not 0 or 1, then the caller
+        // guarantees that both the source and destination pointers are aligned to that boundary.
         if (Align > 1) {
             ConstantInt * align = ConstantInt::get(intPtrTy, Align);
@@ -1174 +1174 @@
     }
     return IRBuilder<>::CreateMemMove(Dst, Src, Size, Align, isVolatile, TBAATag, ScopeTag, NoAliasTag);
+}
+
+llvm::CallInst * CBuilder::CreateMemCpy(llvm::Value *Dst, llvm::Value *Src, llvm::Value *Size, unsigned Align, bool isVolatile,
+                                        llvm::MDNode *TBAATag, llvm::MDNode *TBAAStructTag, llvm::MDNode *ScopeTag, llvm::MDNode *NoAliasTag) {
+    if (codegen::EnableAsserts) {
+        DataLayout DL(getModule());
+        IntegerType * const intPtrTy = DL.getIntPtrType(getContext());
+        Value * intDst = CreatePtrToInt(Dst, intPtrTy);
+        Value * intSrc = CreatePtrToInt(Src, intPtrTy);
+        // If the call to this intrinisic has an alignment value that is not 0 or 1, then the caller
+        // guarantees that both the source and destination pointers are aligned to that boundary.
+        if (Align > 1) {
+            ConstantInt * align = ConstantInt::get(intPtrTy, Align);
+            CreateAssertZero(CreateURem(intDst, align), "CreateMemCpy: Dst pointer is misaligned");
+            CreateAssertZero(CreateURem(intSrc, align), "CreateMemCpy: Src pointer is misaligned");
+        }
+        Value * intSize = CreateZExtOrTrunc(Size, intSrc->getType());
+        Value * nonOverlapping = CreateOr(CreateICmpULT(CreateAdd(intSrc, intSize), intDst),
+                                          CreateICmpULT(CreateAdd(intDst, intSize), intSrc));
+        CreateAssert(nonOverlapping, "CreateMemCpy: overlapping ranges is undefined");
+    }
+    return IRBuilder<>::CreateMemCpy(Dst, Src, Size, Align, isVolatile, TBAATag, TBAAStructTag, ScopeTag, NoAliasTag);
 }
 

icGREP/icgrep-devel/icgrep/IR_Gen/CBuilder.h

@@ -258 +258 @@
     llvm::StoreInst * CreateAlignedStore(llvm::Value * Val, llvm::Value * Ptr, unsigned Align, bool isVolatile = false);
 
+    llvm::CallInst * CreateMemMove(llvm::Value *Dst, llvm::Value *Src, uint64_t Size, unsigned Align,
+                            bool isVolatile = false, llvm::MDNode *TBAATag = nullptr,
+                            llvm::MDNode *ScopeTag = nullptr,
+                            llvm::MDNode *NoAliasTag = nullptr) {
+        return CreateMemMove(Dst, Src, getInt64(Size), Align, isVolatile, TBAATag, ScopeTag, NoAliasTag);
+    }
+
     llvm::CallInst * CreateMemMove(llvm::Value *Dst, llvm::Value *Src, llvm::Value *Size, unsigned Align,
                             bool isVolatile = false, llvm::MDNode *TBAATag = nullptr,
                             llvm::MDNode *ScopeTag = nullptr,
                             llvm::MDNode *NoAliasTag = nullptr);
+
+    llvm::CallInst * CreateMemCpy(llvm::Value *Dst, llvm::Value *Src, uint64_t Size, unsigned Align,
+                           bool isVolatile = false, llvm::MDNode *TBAATag = nullptr,
+                           llvm::MDNode *TBAAStructTag = nullptr,
+                           llvm::MDNode *ScopeTag = nullptr,
+                           llvm::MDNode *NoAliasTag = nullptr) {
+        return CreateMemCpy(Dst, Src, getInt64(Size), Align, isVolatile, TBAATag, TBAAStructTag, ScopeTag, NoAliasTag);
+    }
+
+    llvm::CallInst * CreateMemCpy(llvm::Value *Dst, llvm::Value *Src, llvm::Value *Size, unsigned Align,
+                           bool isVolatile = false, llvm::MDNode *TBAATag = nullptr,
+                           llvm::MDNode *TBAAStructTag = nullptr,
+                           llvm::MDNode *ScopeTag = nullptr,
+                           llvm::MDNode *NoAliasTag = nullptr);
+
 
     void setDriver(Driver * const driver) {

icGREP/icgrep-devel/icgrep/IR_Gen/idisa_nvptx_builder.cpp

@@ -210 +210 @@
   CreateCall(barrierFunc);
 
-  Value * carryOffsetPtr = nullptr;
   Value * carryVal = carryInitVal;
-  Value * bubbleOffsetPtr = nullptr;
   Value * bubbleVal = bubbleInitVal;
 
   for (unsigned offset = groupThreads/2; offset>0; offset=offset>>1){
-    carryOffsetPtr = CreateGEP(carry, {getInt32(0), CreateXor(id, getInt32(offset))});
+    Value * carryOffsetPtr = CreateGEP(carry, {getInt32(0), CreateXor(id, getInt32(offset))});
     carryVal = CreateOr(carryVal, CreateLoad(carryOffsetPtr));
     CreateStore(carryVal, carryPtr);
-    bubbleOffsetPtr = CreateGEP(bubble, {getInt32(0), CreateXor(id, getInt32(offset))});
+    Value * bubbleOffsetPtr = CreateGEP(bubble, {getInt32(0), CreateXor(id, getInt32(offset))});
     bubbleVal = CreateOr(bubbleVal, CreateLoad(bubbleOffsetPtr));
     CreateStore(bubbleVal, bubblePtr);

icGREP/icgrep-devel/icgrep/cc/multiplex_CCs.cpp

@@ -44 +44 @@
 
 
-void doMultiplexCCs(std::vector<UCD::UnicodeSet> CCs,
+void doMultiplexCCs(const std::vector<UCD::UnicodeSet> & CCs,
                     std::vector<std::vector<unsigned>> & exclusiveSetIDs,
                     std::vector<UCD::UnicodeSet> & multiplexedCCs) {

icGREP/icgrep-devel/icgrep/cc/multiplex_CCs.h

@@ -10 +10 @@
 
 
-void doMultiplexCCs(std::vector<UCD::UnicodeSet> CCs,
+void doMultiplexCCs(const std::vector<UCD::UnicodeSet> & CCs,
                     std::vector<std::vector<unsigned>> & exclusiveSetIDs,
                     std::vector<UCD::UnicodeSet> & multiplexedCCs);

icGREP/icgrep-devel/icgrep/editd/editd.cpp

@@ -685 +685 @@
     auto editdScanK = pxDriver.addKernelInstance(make_unique<editdScanKernel>(iBuilder, editDistance));
     pxDriver.makeKernelCall(editdScanK, {MatchResults}, {});
-        
+    pxDriver.LinkFunction(*editdScanK, "wrapped_report_pos", &wrapped_report_pos);
     pxDriver.generatePipelineIR();
     pxDriver.deallocateBuffers();
     iBuilder->CreateRetVoid();
 
-    pxDriver.LinkFunction(*editdScanK, "wrapped_report_pos", &wrapped_report_pos);
     pxDriver.finalizeObject();
 

icGREP/icgrep-devel/icgrep/icgrep-devel.files

@@ -291 +291 @@
 wc.cpp
 CMakeLists.txt
+segtok.cpp
+re/re_minimizer.h
+re/re_minimizer.cpp
+re/re_memoizer.cpp

icGREP/icgrep-devel/icgrep/icgrep-devel.includes

@@ -2 +2 @@
 ../boost/include/
 ../libllvm/include/
+re

icGREP/icgrep-devel/icgrep/kernels/kernel.cpp

@@ -111 +111 @@
 }
 
-Module * Kernel::makeModule(const std::unique_ptr<KernelBuilder> & idb) {
-    assert (mModule == nullptr);
-    std::stringstream cacheName;   
+std::string Kernel::getCacheName(const std::unique_ptr<KernelBuilder> & idb) const {
+    std::stringstream cacheName;
     cacheName << getName() << '_' << idb->getBuilderUniqueName();
     for (const StreamSetBuffer * b: mStreamSetInputBuffers) {
@@ -121 +120 @@
         cacheName <<  ':' <<  b->getUniqueID();
     }
-    mModule = new Module(cacheName.str(), idb->getContext());
-    prepareKernel(idb);
+    return cacheName.str();
+}
+
+Module * Kernel::setModule(Module * const module) {
+    assert (mModule == nullptr || mModule == module);
+    assert (module != nullptr);
+    mModule = module;
     return mModule;
 }
 
-Module * Kernel::setModule(const std::unique_ptr<KernelBuilder> & idb, llvm::Module * const module) {
-    assert (mModule == nullptr);
-    mModule = module;
-    prepareKernel(idb);
-    return mModule;
+Module * Kernel::makeModule(const std::unique_ptr<kernel::KernelBuilder> & idb) {
+    return setModule(new Module(getCacheName(idb), idb->getContext()));
 }
 
@@ -145 +146 @@
     const auto requiredBlocks = codegen::SegmentSize + ((blockSize + mLookAheadPositions - 1) / blockSize);
 
+    IntegerType * const sizeTy = idb->getSizeTy();
+
     for (unsigned i = 0; i < mStreamSetInputs.size(); i++) {
         if ((mStreamSetInputBuffers[i]->getBufferBlocks() != 0) && (mStreamSetInputBuffers[i]->getBufferBlocks() < requiredBlocks)) {
@@ -151 +154 @@
         mScalarInputs.emplace_back(mStreamSetInputBuffers[i]->getStreamSetHandle()->getType(), mStreamSetInputs[i].name + BUFFER_PTR_SUFFIX);
         if ((i == 0) || !mStreamSetInputs[i].rate.isExact()) {
-            addScalar(idb->getSizeTy(), mStreamSetInputs[i].name + PROCESSED_ITEM_COUNT_SUFFIX);
-        }
-    }
-
-    IntegerType * const sizeTy = idb->getSizeTy();
+            addScalar(sizeTy, mStreamSetInputs[i].name + PROCESSED_ITEM_COUNT_SUFFIX);
+        }
+    }
+
     for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
         mScalarInputs.emplace_back(mStreamSetOutputBuffers[i]->getStreamSetHandle()->getType(), mStreamSetOutputs[i].name + BUFFER_PTR_SUFFIX);
@@ -191 +193 @@
     // will be able to add instrumentation to cached modules without recompilation.
     addScalar(idb->getInt64Ty(), CYCLECOUNT_SCALAR);
+    addInternalKernelProperties(idb);
     // NOTE: StructType::create always creates a new type even if an identical one exists.
-    mKernelStateType = getModule()->getTypeByName(getName());
+    if (LLVM_UNLIKELY(mModule == nullptr)) {
+        setModule(new Module(getCacheName(idb), idb->getContext()));
+    }
+    mKernelStateType = mModule->getTypeByName(getName());
     if (LLVM_LIKELY(mKernelStateType == nullptr)) {
         mKernelStateType = StructType::create(idb->getContext(), mKernelFields, getName());
@@ -198 +204 @@
     processingRateAnalysis();
 }
-    
+
+void Kernel::prepareCachedKernel(const std::unique_ptr<KernelBuilder> & idb) {
+
+    assert ("KernelBuilder does not have a valid IDISA Builder" && idb);
+    if (LLVM_UNLIKELY(mKernelStateType != nullptr)) {
+        report_fatal_error("Cannot prepare kernel after kernel state finalized");
+    }
+    assert (getModule());
+    const auto blockSize = idb->getBitBlockWidth();
+    if (mStride == 0) {
+        // Set the default kernel stride.
+        mStride = blockSize;
+    }
+    const auto requiredBlocks = codegen::SegmentSize + ((blockSize + mLookAheadPositions - 1) / blockSize);
+
+    IntegerType * const sizeTy = idb->getSizeTy();
+    for (unsigned i = 0; i < mStreamSetInputs.size(); i++) {
+        if ((mStreamSetInputBuffers[i]->getBufferBlocks() != 0) && (mStreamSetInputBuffers[i]->getBufferBlocks() < requiredBlocks)) {
+            //report_fatal_error(getName() + ": " + mStreamSetInputs[i].name + " requires buffer size " + std::to_string(requiredBlocks));
+        }
+        mScalarInputs.emplace_back(mStreamSetInputBuffers[i]->getStreamSetHandle()->getType(), mStreamSetInputs[i].name + BUFFER_PTR_SUFFIX);
+        if ((i == 0) || !mStreamSetInputs[i].rate.isExact()) {
+            addScalar(sizeTy, mStreamSetInputs[i].name + PROCESSED_ITEM_COUNT_SUFFIX);
+        }
+    }
+
+    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
+        mScalarInputs.emplace_back(mStreamSetOutputBuffers[i]->getStreamSetHandle()->getType(), mStreamSetOutputs[i].name + BUFFER_PTR_SUFFIX);
+        if ((mStreamSetInputs.empty() && (i == 0)) || !mStreamSetOutputs[i].rate.isExact()) {
+            addScalar(sizeTy, mStreamSetOutputs[i].name + PRODUCED_ITEM_COUNT_SUFFIX);
+        }
+    }
+    for (const auto & binding : mScalarInputs) {
+        addScalar(binding.type, binding.name);
+    }
+    for (const auto & binding : mScalarOutputs) {
+        addScalar(binding.type, binding.name);
+    }
+    if (mStreamMap.empty()) {
+        prepareStreamSetNameMap();
+    }
+    for (const auto & binding : mInternalScalars) {
+        addScalar(binding.type, binding.name);
+    }
+
+    Type * const consumerSetTy = StructType::get(sizeTy, sizeTy->getPointerTo()->getPointerTo(), nullptr)->getPointerTo();
+    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
+        addScalar(consumerSetTy, mStreamSetOutputs[i].name + CONSUMER_SUFFIX);
+    }
+
+    addScalar(sizeTy, LOGICAL_SEGMENT_NO_SCALAR);
+    addScalar(idb->getInt1Ty(), TERMINATION_SIGNAL);
+
+    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
+        addScalar(sizeTy, mStreamSetOutputs[i].name + CONSUMED_ITEM_COUNT_SUFFIX);
+    }
+
+    // We compile in a 64-bit CPU cycle counter into every kernel.   It will remain unused
+    // in normal execution, but when codegen::EnableCycleCounter is specified, pipelines
+    // will be able to add instrumentation to cached modules without recompilation.
+    addScalar(idb->getInt64Ty(), CYCLECOUNT_SCALAR);
+    mKernelStateType = getModule()->getTypeByName(getName());
+    assert (mKernelStateType);
+    processingRateAnalysis();
+}
     
 void Kernel::processingRateAnalysis() {
@@ -290 +360 @@
         const auto m = idb->getModule();
         const auto ip = idb->saveIP();
-        const auto saveInstance = getInstance();
+        // const auto saveInstance = getInstance();
         idb->setModule(mModule);
         addKernelDeclarations(idb);
@@ -296 +366 @@
         callGenerateDoSegmentMethod(idb);
         callGenerateFinalizeMethod(idb);
-        setInstance(saveInstance);
+        // setInstance(saveInstance);
         idb->setModule(m);
         idb->restoreIP(ip);
@@ -812 +882 @@
 
     Value * blockBaseMask = kb->CreateNot(kb->getSize(kb->getBitBlockWidth() - 1));
-    //
-    // Define and allocate the temporary buffer area.
-    //
-    Type * tempBuffers[totalSetCount];
-    for (unsigned i = 0; i < totalSetCount; i++) {
-        unsigned blocks = maxBlocksToCopy[i];
-        Type * bufType = i < inputSetCount ? mStreamSetInputBuffers[i]->getStreamSetBlockType() : mStreamSetOutputBuffers[i -inputSetCount]->getStreamSetBlockType();
-        if (blocks > 1) {
-            tempBuffers[i] = ArrayType::get(bufType, blocks);
-        }
-        else {
-            tempBuffers[i] = bufType;
-        }
-    }
-    Type * tempParameterStructType = StructType::create(kb->getContext(), ArrayRef<Type *>(tempBuffers, totalSetCount), "tempBuf");
-    Value * tempParameterArea = kb->CreateCacheAlignedAlloca(tempParameterStructType);
     ConstantInt * blockSize = kb->getSize(kb->getBitBlockWidth());
     ConstantInt * strideSize = kb->getSize(mStride);
@@ -866 +920 @@
     // buffer block containing the next item, and the number of linearly available items.
 
-    std::vector<Value *> processedItemCount;
-    std::vector<Value *> inputBlockPtr;
+    Value * processedItemCount[inputSetCount];
+    Value * inputBlockPtr[inputSetCount];
     std::vector<Value *> producedItemCount;
     std::vector<Value *> outputBlockPtr;
@@ -873 +927 @@
     //  Now determine the linearly available blocks, based on blocks remaining reduced
     //  by limitations of linearly available input buffer space.
-
     Value * linearlyAvailStrides = stridesRemaining;
     for (unsigned i = 0; i < inputSetCount; i++) {
@@ -879 +932 @@
         Value * blkNo = kb->CreateUDiv(p, blockSize);
         Value * b = kb->getInputStreamBlockPtr(mStreamSetInputs[i].name, kb->getInt32(0));
-        processedItemCount.push_back(p);
-        inputBlockPtr.push_back(b);
+        // processedItemCount.push_back(p);
+        processedItemCount[i] = p;
+        // inputBlockPtr.push_back(b);
+        inputBlockPtr[i] = b;
         auto & rate = mStreamSetInputs[i].rate;
         if (rate.isUnknownRate()) continue;  // No calculation possible for unknown rates.
@@ -894 +949 @@
         linearlyAvailStrides = kb->CreateSelect(kb->CreateICmpULT(maxStrides, linearlyAvailStrides), maxStrides, linearlyAvailStrides);
     }
+
     //  Now determine the linearly writeable blocks, based on available blocks reduced
     //  by limitations of output buffer space.
@@ -915 +971 @@
         linearlyWritableStrides = kb->CreateSelect(kb->CreateICmpULT(maxStrides, linearlyWritableStrides), maxStrides, linearlyWritableStrides);
     }
-    Value * haveStrides = kb->CreateICmpUGT(linearlyWritableStrides, kb->getSize(0));
-    kb->CreateCondBr(haveStrides, doMultiBlockCall, tempBlockCheck);
+    Value * const haveFullStrides = kb->CreateICmpUGT(linearlyWritableStrides, kb->getSize(0));
+    kb->CreateCondBr(haveFullStrides, doMultiBlockCall, tempBlockCheck);
 
     //  At this point we have verified the availability of one or more blocks of input data and output buffer space for all stream sets.
@@ -944 +1000 @@
 
     kb->CreateCall(multiBlockFunction, doMultiBlockArgs);
+
     // Do copybacks if necessary.
     unsigned priorIdx = 0;
-    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
-        Value * log2BlockSize = kb->getSize(std::log2(kb->getBitBlockWidth()));
+    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {        
         if (auto cb = dyn_cast<SwizzledCopybackBuffer>(mStreamSetOutputBuffers[i]))  {
+            Value * log2BlockSize = kb->getSize(std::log2(kb->getBitBlockWidth()));
             BasicBlock * copyBack = kb->CreateBasicBlock(mStreamSetOutputs[i].name + "_copyBack");
             BasicBlock * done = kb->CreateBasicBlock(mStreamSetOutputs[i].name + "_copyBackDone");
@@ -986 +1043 @@
     kb->setProcessedItemCount(mStreamSetInputs[0].name, nowProcessed);
     Value * reducedStridesToDo = kb->CreateSub(stridesRemaining, linearlyWritableStrides);
-    BasicBlock * multiBlockFinal = kb->GetInsertBlock();
-    stridesRemaining->addIncoming(reducedStridesToDo, multiBlockFinal);
+    stridesRemaining->addIncoming(reducedStridesToDo, kb->GetInsertBlock());
     kb->CreateBr(doSegmentOuterLoop);
+
+
     //
     // We use temporary buffers in 3 different cases that preclude full block processing.
@@ -999 +1057 @@
 
     kb->SetInsertPoint(tempBlockCheck);
-    haveStrides = kb->CreateICmpUGT(stridesRemaining, kb->getSize(0));
+    Value * const haveStrides = kb->CreateICmpUGT(stridesRemaining, kb->getSize(0));
     kb->CreateCondBr(kb->CreateOr(mIsFinal, haveStrides), doTempBufferBlock, segmentDone);
 
@@ -1017 +1075 @@
         }
     }
+    //
+    // Define and allocate the temporary buffer area.
+    //
+    Type * tempBuffers[totalSetCount];
+    for (unsigned i = 0; i < inputSetCount; ++i) {
+        Type * bufType = mStreamSetInputBuffers[i]->getStreamSetBlockType();
+        tempBuffers[i] = ArrayType::get(bufType, maxBlocksToCopy[i]);
+    }
+    for (unsigned i = 0; i < outputSetCount; i++) {
+        Type * bufType = mStreamSetOutputBuffers[i]->getStreamSetBlockType();
+        tempBuffers[i + inputSetCount] = ArrayType::get(bufType, maxBlocksToCopy[i + inputSetCount]);
+    }
+    Type * tempParameterStructType = StructType::create(kb->getContext(), ArrayRef<Type *>(tempBuffers, totalSetCount), "tempBuf");
     // Prepare the temporary buffer area.
-    //
-    // First zero it out.
-    Constant * const tempAreaSize = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(tempParameterStructType), kb->getSizeTy(), false);
-    kb->CreateMemZero(tempParameterArea, tempAreaSize);
-    // For each input and output buffer, copy over necessary data starting from the last
-    // block boundary.
+    Value * tempParameterArea = kb->CreateCacheAlignedAlloca(tempParameterStructType);
+    kb->CreateMemZero(tempParameterArea, ConstantExpr::getSizeOf(tempParameterStructType));
+    // For each input and output buffer, copy over necessary data starting from the last block boundary.
     Value * itemCountNeeded[inputSetCount];
     itemCountNeeded[0] = tempBlockItems;
     Value * finalItemCountNeeded[inputSetCount];
 
-    for (unsigned i = 0; i < mStreamSetInputBuffers.size(); i++) {
+    for (unsigned i = 0; i < inputSetCount; i++) {
         Type * bufPtrType = mStreamSetInputBuffers[i]->getPointerType();
         if (mItemsPerStride[i] != 0) {
@@ -1035 +1103 @@
             ConstantInt * strideItems = kb->getSize(mItemsPerStride[i]);
             Value * strideBasePos = kb->CreateSub(processedItemCount[i], kb->CreateURem(processedItemCount[i], strideItems));
-            Value * blockBasePos = (mItemsPerStride[i] % bitBlockWidth == 0) ? strideBasePos : kb->CreateAnd(strideBasePos, blockBaseMask);
+            Value * blockBasePos = strideBasePos;
+            if (mItemsPerStride[i] & (bitBlockWidth - 1)) {
+                blockBasePos = kb->CreateAnd(strideBasePos, blockBaseMask);
+            }
 
             // The number of items to copy is determined by the processing rate requirements.
@@ -1077 +1148 @@
             }
             tempArgs.push_back(tempBufPtr);
-        }
-        else {
+        } else {
             Value * bufPtr = kb->getInputStreamBlockPtr(mStreamSetInputs[i].name, kb->getInt32(0));
             bufPtr = kb->CreatePointerCast(bufPtr, mStreamSetInputBuffers[i]->getPointerType());
@@ -1085 +1155 @@
     }
     Value * outputBasePos[outputSetCount];
-    for (unsigned i = 0; i < mStreamSetOutputBuffers.size(); i++) {
-        Value * tempBufPtr = kb->CreateGEP(tempParameterArea,  {kb->getInt32(0), kb->getInt32(mStreamSetInputs.size() + i)});
+    for (unsigned i = 0; i < outputSetCount; i++) {
+        Value * tempBufPtr = kb->CreateGEP(tempParameterArea,  {kb->getInt32(0), kb->getInt32(inputSetCount + i)});
         Type * bufPtrType = mStreamSetOutputBuffers[i]->getPointerType();
         tempBufPtr = kb->CreatePointerCast(tempBufPtr, bufPtrType);
@@ -1134 +1204 @@
     }
 
-
     //  We've dealt with the partial block processing and copied information back into the
     //  actual buffers.  If this isn't the final block, loop back for more multiblock processing.

icGREP/icgrep-devel/icgrep/kernels/kernel.h

@@ -88 +88 @@
 
     StreamPort getStreamPort(const std::string & name) const;
-    
-    llvm::Module * makeModule(const std::unique_ptr<KernelBuilder> & idb);
-
-    llvm::Module * setModule(const std::unique_ptr<KernelBuilder> & idb, llvm::Module * const module);
+
+    llvm::Module * setModule(llvm::Module * const module);
+
+    llvm::Module * makeModule(const std::unique_ptr<kernel::KernelBuilder> & idb);
 
     llvm::Module * getModule() const {
@@ -137 +137 @@
     virtual ~Kernel() = 0;
 
-protected:
+    void prepareKernel(const std::unique_ptr<KernelBuilder> & idb);
+
+    void prepareCachedKernel(const std::unique_ptr<KernelBuilder> & idb);
+
+    std::string getCacheName(const std::unique_ptr<KernelBuilder> & idb) const;
+
+protected:
+
+    virtual void addInternalKernelProperties(const std::unique_ptr<KernelBuilder> & idb) { }
 
     // Constructor
@@ -168 +176 @@
 
     void linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> &) override { }
-
-    virtual void prepareKernel(const std::unique_ptr<KernelBuilder> & idb);
 
     virtual void generateInitializeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) { }

icGREP/icgrep-devel/icgrep/kernels/lz4_bytestream_decoder.cpp

@@ -78 +78 @@
             outputBufferBasePtr,
             iBuilder->CreateGEP(inputBufferBasePtr, iBuilder->CreateAdd(literalStart, copyLength1)),
-            iBuilder->CreateSub(literalLength, copyLength1), 8);        // Buffer start is aligned.
+            iBuilder->CreateSub(literalLength, copyLength1), 1); // Buffer start is aligned.
+    // NOTE: Test case reported non-8-byte alignment
     outputItems = iBuilder->CreateAdd(outputItems, literalLength);
 

icGREP/icgrep-devel/icgrep/kernels/scanmatchgen.cpp

@@ -55 +55 @@
     /* self = */ args++;
     Value * itemsToDo = &*(args++);
-    Value * inputStreamAvail = &*(args++);
+    /* inputStreamAvail = */ args++;
     Value * match_result = &*(args++);
     Value * line_break = &*(args++);
-    Value * input_stream = &*(args);
+    /* input_stream = */ args++;
 
     Value * blocksToDo = iBuilder->CreateUDiv(iBuilder->CreateAdd(itemsToDo, blockSizeLess1), blockSize);

icGREP/icgrep-devel/icgrep/pablo/carry_data.h

@@ -11 +11 @@
 
 class CarryData {
-    friend class CarryManager;
 public:
 
-    enum SummaryType : int {
-        NoSummary
-        , ImplicitSummary
-        , BorrowedSummary
-        , ExplicitSummary
+    enum SummaryType : unsigned {
+        NoSummary = 0
+        , ImplicitSummary = 1
+        , BorrowedSummary = 2
+        , ExplicitSummary = 3
+        , NonCarryCollapsingMode = 4
     };
 
     CarryData()
-    : mSummaryType(NoSummary)
-    , mInNonCollapsingCarryMode(false) {
+    : mSummaryType(NoSummary) {
 
     }
              
     bool hasSummary() const {
-        return (mSummaryType != NoSummary);
+        return (mSummaryType & (ImplicitSummary | BorrowedSummary | ExplicitSummary)) != NoSummary;
     }
     
     bool hasImplicitSummary() const {
-        return (mSummaryType == ImplicitSummary);
+        return (mSummaryType & (ImplicitSummary | BorrowedSummary | ExplicitSummary)) == ImplicitSummary;
     }
 
     bool hasBorrowedSummary() const {
-        return (mSummaryType == BorrowedSummary);
+        return (mSummaryType & (ImplicitSummary | BorrowedSummary | ExplicitSummary)) == BorrowedSummary;
     }
 
     bool hasExplicitSummary() const {
-        return (mSummaryType == ExplicitSummary);
+        return (mSummaryType & (ImplicitSummary | BorrowedSummary | ExplicitSummary)) == ExplicitSummary;
     }
 
     bool nonCarryCollapsingMode() const {
-        return mInNonCollapsingCarryMode;
+        return (mSummaryType & (NonCarryCollapsingMode)) != 0;
     }
 
@@ -52 +51 @@
 
     void setNonCollapsingCarryMode(const bool value = true) {
-        mInNonCollapsingCarryMode = value;
+        if (value) {
+            mSummaryType = (SummaryType)(mSummaryType | NonCarryCollapsingMode);
+        } else {
+            mSummaryType = (SummaryType)(mSummaryType & ~NonCarryCollapsingMode);
+        }
     }
     
@@ -58 +61 @@
 
     SummaryType     mSummaryType;
-    bool            mInNonCollapsingCarryMode;
 
 };

icGREP/icgrep-devel/icgrep/pablo/carrypack_manager.cpp

@@ -127 +127 @@
     mCarryGroup.resize(assignDefaultCarryGroups(kernel->getEntryBlock()));
 
-    Type * const carryStateTy = analyse(iBuilder, kernel->getEntryBlock());
-
-    kernel->addScalar(carryStateTy, "carries");
-
-//    iBuilder->CallPrintInt("carry state size:", ConstantExpr::getSizeOf(carryStateTy));
+    kernel->setCarryDataTy(analyse(iBuilder, mCurrentScope));
+
+    kernel->addScalar(kernel->getCarryDataTy(), "carries");
 
     if (mHasLoop) {

icGREP/icgrep-devel/icgrep/pablo/pablo_kernel.cpp

@@ -125 +125 @@
 }
 
-void PabloKernel::prepareKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
+void PabloKernel::addInternalKernelProperties(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     mSizeTy = iBuilder->getSizeTy();
     mStreamTy = iBuilder->getStreamTy();
@@ -132 +132 @@
     mPabloCompiler->initializeKernelData(iBuilder);
     mSizeTy = nullptr;
-    mStreamTy = nullptr;
-    BlockOrientedKernel::prepareKernel(iBuilder);
+    mStreamTy = nullptr;   
 }
 

icGREP/icgrep-devel/icgrep/pablo/pablo_kernel.h

@@ -31 +31 @@
     friend class CarryManager;
     friend class CarryPackManager;
+    friend class ParabixObjectCache;
 
 public:
@@ -124 +125 @@
     Integer * getInteger(const int64_t value) const;
 
+    llvm::StructType * getCarryDataTy() const {
+        return mCarryDataTy;
+    }
+
 protected:
 
@@ -145 +150 @@
     llvm::IntegerType * getInt1Ty() const;
 
-private:
+    void setCarryDataTy(llvm::StructType * const carryDataTy) {
+        mCarryDataTy = carryDataTy;
+    }
 
     // A custom method for preparing kernel declarations is needed,
     // so that the carry data requirements may be accommodated before
     // finalizing the KernelStateType.
-    void prepareKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) final;
+    void addInternalKernelProperties(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) final;
+
+private:
 
     void generateDoBlockMethod(const std::unique_ptr<KernelBuilder> & iBuilder) final;
@@ -169 +178 @@
     llvm::IntegerType *             mSizeTy;
     llvm::VectorType *              mStreamTy;
+    llvm::StructType *              mCarryDataTy;
     std::vector<Var *>              mInputs;
     std::vector<Var *>              mOutputs;

icGREP/icgrep-devel/icgrep/re/re_memoizer.hpp

@@ -1 @@
-#ifndef RE_NAMEDICTIONARY_H
-#define RE_NAMEDICTIONARY_H
+#ifndef RE_MEMOIZER_H
+#define RE_MEMOIZER_H
 
 #include <re/re_name.h>
@@ -8 +8 @@
 
 struct MemoizerComparator {
-    inline bool operator() (const RE * lh, const RE * rh) const{
-        if (LLVM_LIKELY(llvm::isa<Name>(lh) && llvm::isa<Name>(rh))) {
-            return *llvm::cast<Name>(lh) < *llvm::cast<Name>(rh);
-        } else if (llvm::isa<Name>(lh)) {
-            return *llvm::cast<Name>(lh) < *llvm::cast<CC>(rh);
-        }
-        return *llvm::cast<Name>(rh) > *llvm::cast<CC>(lh);
-    }
+    bool operator() (const RE * lh, const RE * rh) const;
 };
 
 struct Memoizer : public std::set<RE *, MemoizerComparator> {
 
-    inline Name * memoize(CC * cc) {
+    RE * memoize(RE * const re) {
+        return *(insert(re).first);
+    }
+
+    Name * memoize(Name * const name) {
+        return llvm::cast<Name>(memoize(llvm::cast<RE>(name)));
+    }
+
+    Name * memoize(CC * const cc) {
         auto f = find(cc);
         if (f != end()) {
             return llvm::cast<Name>(*f);
         } else {
-            Name * name = makeName(cc);
-            insert(name);
-            return name;
+            return memoize(makeName(cc));
         }
-    }
-
-    inline Name * memoize(Name * name) {
-        return llvm::cast<Name>(*insert(name).first);
     }
 };
@@ -38 +33 @@
 }
 
-#endif // RE_NAMEDICTIONARY_H
+#endif // RE_MEMOIZER_H

icGREP/icgrep-devel/icgrep/re/re_name.h

@@ -127 +127 @@
         return *llvm::cast<CC>(mDefinition) < other;
     }
-    return false;
+    return RE::ClassTypeId::Name < RE::ClassTypeId::CC;
 }
 
@@ -134 +134 @@
         return other < *llvm::cast<CC>(mDefinition);
     }
-    return true;
+    return RE::ClassTypeId::CC < RE::ClassTypeId::Name;
 }
 

icGREP/icgrep-devel/icgrep/re/re_parser.cpp

@@ -149 +149 @@
     RE * re = nullptr;
     if (fModeFlagSet & IGNORE_SPACE_MODE_FLAG) {
-        while (*mCursor == ' ') mCursor++;
+        while (mCursor.more() && *mCursor == ' ') {
+            ++mCursor;
+        }
     }
     if (mCursor.more()) {
@@ -190 +192 @@
                     re = makeSeq({re, makeZeroWidth("GCB")});
                 }
-                return re;
+                break;
             case '.': // the 'any' metacharacter
                 mCursor++;
@@ -202 +204 @@
                     fGraphemeBoundaryPending = true;
                 }
-                return re;
-        }
-    }
-    return nullptr;
+        }
+    }
+    return re;
 }
 

icGREP/icgrep-devel/icgrep/re/re_re.h

@@ -26 +26 @@
         , Intersect
         , Name
-        , Permute
+        // , Permute
         , Rep
         , Seq
         , Start
-        , SymDiff
-        , Union
+        // , SymDiff
+        // , Union
     };
     inline ClassTypeId getClassTypeId() const {
@@ -52 +52 @@
 class Vector : public RE, public std::vector<RE*, RE::VectorAllocator> {
 public:
+    static inline bool classof(const RE * re) {
+        const auto typeId = re->getClassTypeId();
+        return typeId == ClassTypeId::Alt || typeId == ClassTypeId::Seq;
+    }
+    static inline bool classof(const void *) {
+        return false;
+    }
     virtual ~Vector() {}
 protected:

icGREP/icgrep-devel/icgrep/re/re_reverse.cpp

@@ -56 +56 @@
     } else if (Name * n = dyn_cast<Name>(re)) {
         switch (n->getType()) {
-            case Name::Type::Byte:  return makeName(cast<CC>(n->getDefinition()));
-            case Name::Type::Unicode:  return makeName(cast<CC>(n->getDefinition()));
+            case Name::Type::Byte:
+            case Name::Type::Unicode:
+                return makeName(cast<CC>(n->getDefinition()));
             case Name::Type::UnicodeProperty:
                 return makeName(n->getNamespace(), n->getName(), Name::Type::UnicodeProperty);

icGREP/icgrep-devel/icgrep/re/re_simplifier.cpp

@@ -1 +1 @@
 #include "re_simplifier.h"
 #include <re/re_name.h>
-#include <re/re_any.h>
-#include <re/re_start.h>
-#include <re/re_end.h>
 #include <re/re_alt.h>
-#include <re/re_cc.h>
 #include <re/re_seq.h>
 #include <re/re_rep.h>
@@ -11 +7 @@
 #include <re/re_intersect.h>
 #include <re/re_assertion.h>
-#include <re/re_analysis.h>
-#include <algorithm>
-#include <memory>
-#include <queue>
+#include <re/re_memoizer.hpp>
+#include <boost/container/flat_set.hpp>
 
 using namespace llvm;
@@ -20 +14 @@
 namespace re {
 
+struct PassContainer {
+    RE * simplify(RE * re) {
+        if (Alt * alt = dyn_cast<Alt>(re)) {
+            boost::container::flat_set<RE *> list;
+            list.reserve(alt->size());
+            for (RE * item : *alt) {
+                item = simplify(item);
+                if (LLVM_UNLIKELY(isa<Vector>(item) && cast<Vector>(item)->empty())) {
+                    continue;
+                }
+                list.insert(item);
+            }
+            re = makeAlt(list.begin(), list.end());
+        } else if (Seq * seq = dyn_cast<Seq>(re)) {
+            std::vector<RE *> list;
+            list.reserve(seq->size());
+            for (RE * item : *seq) {
+                item = simplify(item);
+                if (LLVM_UNLIKELY(isa<Vector>(item) && cast<Vector>(item)->empty())) {
+                    continue;
+                }
+                list.push_back(item);
+            }
+            re = makeSeq(list.begin(), list.end());
+        } else if (Assertion * a = dyn_cast<Assertion>(re)) {
+            re = makeAssertion(simplify(a->getAsserted()), a->getKind(), a->getSense());
+        } else if (Rep * rep = dyn_cast<Rep>(re)) {
+            RE * expr = simplify(rep->getRE());
+            re = makeRep(expr, rep->getLB(), rep->getUB());
+        } else if (Diff * diff = dyn_cast<Diff>(re)) {
+            re = makeDiff(simplify(diff->getLH()), simplify(diff->getRH()));
+        } else if (Intersect * e = dyn_cast<Intersect>(re)) {
+            re = makeIntersect(simplify(e->getLH()), simplify(e->getRH()));
+        }
+        return mMemoizer.memoize(re);
+    }
+private:
+    Memoizer mMemoizer;
+};
+
 RE * RE_Simplifier::simplify(RE * re) {
-    if (Alt * alt = dyn_cast<Alt>(re)) {
-        std::vector<RE *> list;
-        list.reserve(alt->size());
-        for (RE * re : *alt) {
-            list.push_back(simplify(re));
-        }
-        re = makeAlt(list.begin(), list.end());
-    } else if (Seq * seq = dyn_cast<Seq>(re)) {
-        std::vector<RE *> list;
-        list.reserve(seq->size());
-        for (RE * re : *seq) {
-            list.push_back(simplify(re));
-        }
-        re = makeSeq(list.begin(), list.end());
-    } else if (Assertion * a = dyn_cast<Assertion>(re)) {
-        re = makeAssertion(simplify(a->getAsserted()), a->getKind(), a->getSense());
-    } else if (Rep * rep = dyn_cast<Rep>(re)) {
-        RE * expr = simplify(rep->getRE());
-        re = makeRep(expr, rep->getLB(), rep->getUB());
-    } else if (Diff * diff = dyn_cast<Diff>(re)) {
-        re = makeDiff(simplify(diff->getLH()), simplify(diff->getRH()));
-    } else if (Intersect * e = dyn_cast<Intersect>(re)) {
-        re = makeIntersect(simplify(e->getLH()), simplify(e->getRH()));
-    } 
-    return re;
+    PassContainer pc;
+    return pc.simplify(re);
 }
 

icGREP/icgrep-devel/icgrep/re/re_toolchain.cpp

@@ -12 +12 @@
 #include <re/re_star_normal.h>         // for RE_Star_Normal
 #include <re/re_simplifier.h>          // for RE_Simplifier
+#include <re/re_minimizer.h>
 #include <re/re_local.h>
 #include <re/printer_re.h>
@@ -63 +64 @@
 
     //Optimization passes to simplify the AST.
-    re_ast = re::RE_Nullable::removeNullablePrefix(re_ast);
+    re_ast = RE_Nullable::removeNullablePrefix(re_ast);
     if (PrintOptions.isSet(ShowAllREs) || PrintOptions.isSet(ShowStrippedREs)) {
         std::cerr << "RemoveNullablePrefix:" << std::endl << Printer_RE::PrintRE(re_ast) << std::endl;
     }
-    re_ast = re::RE_Nullable::removeNullableSuffix(re_ast);
+    re_ast = RE_Nullable::removeNullableSuffix(re_ast);
     if (PrintOptions.isSet(ShowAllREs) || PrintOptions.isSet(ShowStrippedREs)) {
         std::cerr << "RemoveNullableSuffix:" << std::endl << Printer_RE::PrintRE(re_ast) << std::endl;
     }
-    re_ast = re::RE_Nullable::removeNullableAssertion(re_ast);
+    re_ast = RE_Nullable::removeNullableAssertion(re_ast);
     if (PrintOptions.isSet(ShowAllREs) || PrintOptions.isSet(ShowStrippedREs)) {
         std::cerr << "RemoveNullableAssertion:" << std::endl << Printer_RE::PrintRE(re_ast) << std::endl;
     }
-    //re_ast = re::RE_Nullable::removeNullableAfterAssertion(re_ast);
+    //re_ast = RE_Nullable::removeNullableAfterAssertion(re_ast);
     //if (PrintOptions.isSet(ShowAllREs) || PrintOptions.isSet(ShowStrippedREs)) {
     //    std::cerr << "RemoveNullableAfterAssertion" << std::endl << Printer_RE::PrintRE(re_ast) << std::endl;
     //}
     
-    re_ast = re::RE_Simplifier::simplify(re_ast);
+    // re_ast = RE_Minimizer::minimize(re_ast);
+
+    re_ast = RE_Simplifier::simplify(re_ast);
+
     if (PrintOptions.isSet(ShowAllREs) || PrintOptions.isSet(ShowSimplifiedREs)) {
         //Print to the terminal the AST that was generated by the simplifier.
@@ -86 +90 @@
     }
 
-    re_ast = re::RE_Star_Normal::star_normal(re_ast);
+    re_ast = RE_Star_Normal::star_normal(re_ast);
+
     if (PrintOptions.isSet(ShowAllREs) || PrintOptions.isSet(ShowSimplifiedREs)) {
         //Print to the terminal the AST that was transformed to the star normal form.
         std::cerr << "Star_Normal_Form:" << std::endl << Printer_RE::PrintRE(re_ast) << std::endl;
     }    
+
     return re_ast;
 }
@@ -98 +104 @@
     bool local = RE_Local::isLocalLanguage(re_ast) && isTypeForLocal(re_ast);
     cc::CC_Compiler cc_compiler(kernel, basis);
-    re::RE_Compiler re_compiler(kernel, cc_compiler, local);
+    RE_Compiler re_compiler(kernel, cc_compiler, local);
     re_compiler.compileUnicodeNames(re_ast);
     re_compiler.compile(re_ast);

icGREP/icgrep-devel/icgrep/toolchain/NVPTXDriver.cpp

@@ -54 +54 @@
     mPipeline.emplace_back(kb);
     kb->bindPorts(inputs, outputs);
-    kb->setModule(iBuilder, mMainModule);
+    kb->setModule(mMainModule);
 }
 

icGREP/icgrep-devel/icgrep/toolchain/cpudriver.cpp

@@ -70 +70 @@
         throw std::runtime_error("Could not create ExecutionEngine: " + errMessage);
     }
-    mTarget = builder.selectTarget();
+    mTarget = builder.selectTarget();    
     if (LLVM_LIKELY(codegen::EnableObjectCache)) {
         if (codegen::ObjectCacheDir) {
@@ -80 +80 @@
     }
     mMainModule->setTargetTriple(mTarget->getTargetTriple().getTriple());
-
     iBuilder.reset(IDISA::GetIDISA_Builder(*mContext));
     iBuilder->setDriver(this);
@@ -86 +85 @@
 }
 
-void ParabixDriver::makeKernelCall(Kernel * kb, const std::vector<StreamSetBuffer *> & inputs, const std::vector<StreamSetBuffer *> & outputs) {
-    assert ("addKernelCall or makeKernelCall was already run on this kernel." && (kb->getModule() == nullptr));
-    mPipeline.emplace_back(kb);
-    kb->bindPorts(inputs, outputs);
-    kb->makeModule(iBuilder);
+void ParabixDriver::makeKernelCall(Kernel * kernel, const std::vector<StreamSetBuffer *> & inputs, const std::vector<StreamSetBuffer *> & outputs) {
+    assert ("addKernelCall or makeKernelCall was already run on this kernel." && (kernel->getModule() == nullptr));
+    mPipeline.emplace_back(kernel);
+    kernel->bindPorts(inputs, outputs);
+    if (!mCache || !mCache->loadCachedObjectFile(iBuilder, kernel)) {
+        mUncachedKernel.push_back(kernel);
+    }
+    if (kernel->getModule() == nullptr) {
+        kernel->makeModule(iBuilder);
+    }
+    assert (kernel->getModule());
 }
 
 void ParabixDriver::generatePipelineIR() {
-    #ifndef NDEBUG
+
     if (LLVM_UNLIKELY(mPipeline.empty())) {
         report_fatal_error("Pipeline cannot be empty");
@@ -101 +106 @@
             for (auto j = i; ++j != mPipeline.end(); ) {
                 if (LLVM_UNLIKELY(*i == *j)) {
-                    report_fatal_error("Kernel instances cannot occur twice in the pipeline");
+                    report_fatal_error("Kernel " + (*i)->getName() + " occurs twice in the pipeline");
                 }
             }
         }
     }
-    #endif
+
+    for (Kernel * const kernel : mUncachedKernel) {
+        kernel->prepareKernel(iBuilder);
+    }
+
     // note: instantiation of all kernels must occur prior to initialization
     for (const auto & k : mPipeline) {
@@ -130 +139 @@
 
 Function * ParabixDriver::addLinkFunction(Module * mod, llvm::StringRef name, FunctionType * type, void * functionPtr) const {
-    assert ("addKernelCall or makeKernelCall must be called before LinkFunction" && (mod != nullptr));
+    if (LLVM_UNLIKELY(mod == nullptr)) {
+        report_fatal_error("addLinkFunction(" + name + ") cannot be called until after addKernelCall or makeKernelCall");
+    }
     Function * f = mod->getFunction(name);
     if (LLVM_UNLIKELY(f == nullptr)) {
         f = Function::Create(type, Function::ExternalLinkage, name, mod);
-        mEngine->addGlobalMapping(f, functionPtr);
+        mEngine->updateGlobalMapping(f, functionPtr);
     } else if (LLVM_UNLIKELY(f->getType() != type->getPointerTo())) {
         report_fatal_error("Cannot link " + name + ": a function with a different signature already exists with that name in " + mod->getName());
@@ -193 +204 @@
 
     Module * module = nullptr;
-
     try {
-
+        for (Kernel * const kernel : mUncachedKernel) {
+            iBuilder->setKernel(kernel);
+            kernel->generateKernel(iBuilder);
+            module = kernel->getModule(); assert (module);
+            module->setTargetTriple(mMainModule->getTargetTriple());
+            PM.run(*module);
+        }
+        module = mMainModule;
+        iBuilder->setKernel(nullptr);
+        PM.run(*mMainModule);
         for (Kernel * const kernel : mPipeline) {
-            iBuilder->setKernel(kernel);
-            module = kernel->getModule();
-            assert (module != mMainModule);
-            bool uncachedObject = true;
-            if (mCache && mCache->loadCachedObjectFile(iBuilder, kernel)) {
-                uncachedObject = false;
-            }
-            if (uncachedObject) {
-                module->setTargetTriple(mMainModule->getTargetTriple());
-                kernel->generateKernel(iBuilder);
-                PM.run(*module);
-            }
-            mEngine->addModule(std::unique_ptr<Module>(module));
-        }
-
-        iBuilder->setKernel(nullptr);
-        module = mMainModule;
-        PM.run(*mMainModule);
+            if (LLVM_UNLIKELY(kernel->getModule() == nullptr)) {
+                report_fatal_error(kernel->getName() + " was neither loaded from cache nor generated prior to finalizeObject");
+            }
+            mEngine->addModule(std::unique_ptr<Module>(kernel->getModule()));
+        }
         mEngine->finalizeObject();
-
         if (mCache) mCache->cleanUpObjectCacheFiles();
-
     } catch (const std::exception & e) {
-        report_fatal_error(e.what());
+        report_fatal_error(module->getName() + ": " + e.what());
     }
 

icGREP/icgrep-devel/icgrep/toolchain/cpudriver.h

@@ -34 +34 @@
     llvm::ExecutionEngine *                                 mEngine;
     ParabixObjectCache *                                    mCache;
+    std::vector<kernel::Kernel *>                           mUncachedKernel;
     // NOTE: when printing the IR/ASM, we cannot assume they're completely finished after finalizeObject is executed. Instead we store a
     // pointer and delete them once the driver (and any processing) is complete. This prevents us from reclaiming the memory early but

icGREP/icgrep-devel/icgrep/toolchain/object_cache.cpp

@@ -13 +13 @@
 #include <boost/filesystem.hpp>
 #include <boost/range/iterator_range.hpp>
+#include <boost/container/flat_set.hpp>
+#include <llvm/Bitcode/ReaderWriter.h>
+#include <llvm/IR/Verifier.h>
 #include <ctime>
 
@@ -59 +62 @@
 const static auto SIGNATURE = "signature";
 
-const static boost::uintmax_t CACHE_SIZE_LIMIT = 5 * 1024 * 1024;
+const static boost::uintmax_t CACHE_SIZE_LIMIT = 50 * 1024 * 1024;
 
 const MDString * getSignature(const llvm::Module * const M) {
@@ -73 +76 @@
 bool ParabixObjectCache::loadCachedObjectFile(const std::unique_ptr<kernel::KernelBuilder> & idb, kernel::Kernel * const kernel) {
     if (LLVM_LIKELY(kernel->isCachable())) {
-        Module * const module = kernel->getModule();
-        assert ("kernel module cannot be null!" && module);
-        const auto moduleId = module->getModuleIdentifier();
+        assert (kernel->getModule() == nullptr);
+        const auto moduleId = kernel->getCacheName(idb);
+
         // Have we already seen this module before?
-        if (LLVM_UNLIKELY(mCachedObject.count(moduleId) != 0)) {
+        const auto f = mCachedObject.find(moduleId);
+        if (LLVM_UNLIKELY(f != mCachedObject.end())) {
+            Module * const m = f->second.first; assert (m);
+            kernel->setModule(m);
             return true;
         }
@@ -94 +100 @@
                 if (signatureBuffer) {
                     const StringRef loadedSig = signatureBuffer.get()->getBuffer();
-                    if (!loadedSig.equals(kernel->makeSignature(idb))) {
-                        return false;
+                    if (LLVM_UNLIKELY(!loadedSig.equals(kernel->makeSignature(idb)))) {
+                        goto invalid;
                     }
                 } else {
                     report_fatal_error("signature file expected but not found: " + moduleId);
-                    return false;
+                }                
+            }
+            sys::path::replace_extension(objectName, ".kernel");
+            auto kernelBuffer = MemoryBuffer::getFile(objectName.c_str(), -1, false);
+            if (*kernelBuffer) {
+                //MemoryBuffer * kb = kernelBuffer.get().release();
+                //auto loadedFile = parseBitcodeFile(kb->getMemBufferRef(), mContext);
+                auto loadedFile = getLazyBitcodeModule(std::move(kernelBuffer.get()), idb->getContext());
+                if (*loadedFile) {
+                    Module * const m = loadedFile.get().release(); assert (m);
+                    // defaults to <path>/<moduleId>.kernel
+                    m->setModuleIdentifier(moduleId);
+                    kernel->setModule(m);
+                    kernel->prepareCachedKernel(idb);                    
+                    mCachedObject.emplace(moduleId, std::make_pair(m, std::move(objectBuffer.get())));
+                    // update the modified time of the object file
+                    sys::path::replace_extension(objectName, ".o");
+                    boost::filesystem::last_write_time(objectName.c_str(), time(0));
+                    return true;
                 }
             }
-            // update the modified time of the file then add it to our cache
-            boost::filesystem::last_write_time(objectName.c_str(), time(0));
-            mCachedObject.emplace(moduleId, std::move(objectBuffer.get()));
-            return true;
-        } else {
-            // mark this module as cachable
-            module->getOrInsertNamedMetadata(CACHEABLE);
-            // if this module has a signature, add it to the metadata
-            if (kernel->hasSignature()) {
-                NamedMDNode * const md = module->getOrInsertNamedMetadata(SIGNATURE);
-                assert (md->getNumOperands() == 0);
-                MDString * const sig = MDString::get(module->getContext(), kernel->makeSignature(idb));               
-                md->addOperand(MDNode::get(module->getContext(), {sig}));
-            }
+        }
+
+invalid:
+
+        Module * const module = kernel->setModule(new Module(moduleId, idb->getContext()));
+        // mark this module as cachable
+        module->getOrInsertNamedMetadata(CACHEABLE);
+        // if this module has a signature, add it to the metadata
+        if (kernel->hasSignature()) {
+            NamedMDNode * const md = module->getOrInsertNamedMetadata(SIGNATURE);
+            assert (md->getNumOperands() == 0);
+            MDString * const sig = MDString::get(module->getContext(), kernel->makeSignature(idb));
+            md->addOperand(MDNode::get(module->getContext(), {sig}));
         }
     }
@@ -124 +147 @@
 // exists, write it out.
 void ParabixObjectCache::notifyObjectCompiled(const Module * M, MemoryBufferRef Obj) {
-    if (M->getNamedMetadata(CACHEABLE)) {
+    if (LLVM_LIKELY(M->getNamedMetadata(CACHEABLE))) {
         const auto moduleId = M->getModuleIdentifier();
         Path objectName(mCachePath);
@@ -135 +158 @@
         }
 
+        // Write the object code
         std::error_code EC;
-        raw_fd_ostream outfile(objectName, EC, sys::fs::F_None);
-        outfile.write(Obj.getBufferStart(), Obj.getBufferSize());
-        outfile.close();
-
-        // If this module has a signature, write it.
+        raw_fd_ostream objFile(objectName, EC, sys::fs::F_None);
+        objFile.write(Obj.getBufferStart(), Obj.getBufferSize());
+        objFile.close();
+
+        // then the signature (if one exists)
         const MDString * const sig = getSignature(M);
         if (sig) {
@@ -148 +172 @@
             sigfile.close();
         }
+
+        // and finally kernel prototype header.
+        std::unique_ptr<Module> header(new Module(M->getModuleIdentifier(), M->getContext()));
+        for (const Function & f : M->getFunctionList()) {
+            if (f.hasExternalLinkage() && !f.empty()) {
+                Function::Create(f.getFunctionType(), Function::ExternalLinkage, f.getName(), header.get());
+            }
+        }
+
+        sys::path::replace_extension(objectName, ".kernel");
+        raw_fd_ostream kernelFile(objectName.str(), EC, sys::fs::F_None);
+        WriteBitcodeToFile(header.get(), kernelFile, false, false);
+        kernelFile.close();
     }
 }
@@ -178 +215 @@
                     objectPath.replace_extension("sig");
                     remove(objectPath);
+                    objectPath.replace_extension("kernel");
+                    remove(objectPath);
                 }
             }
@@ -185 +224 @@
 
 std::unique_ptr<MemoryBuffer> ParabixObjectCache::getObject(const Module * module) {
-    const auto moduleId = module->getModuleIdentifier();
-    const auto f = mCachedObject.find(moduleId);
+    const auto f = mCachedObject.find(module->getModuleIdentifier());
     if (f == mCachedObject.end()) {
         return nullptr;
     }
     // Return a copy of the buffer, for MCJIT to modify, if necessary.
-    return MemoryBuffer::getMemBufferCopy(f->second.get()->getBuffer());
+    return MemoryBuffer::getMemBufferCopy(f->second.second.get()->getBuffer());
 }
 
@@ -211 +249 @@
 }
 
-ParabixObjectCache::ParabixObjectCache(const std::string & dir)
+ParabixObjectCache::ParabixObjectCache(const std::string dir)
 : mCachePath(dir) {
 

icGREP/icgrep-devel/icgrep/toolchain/object_cache.h

@@ -18 +18 @@
 namespace llvm { class MemoryBuffer; }
 namespace llvm { class MemoryBufferRef; }
+namespace llvm { class LLVMContext; }
 namespace kernel { class Kernel; }
 namespace kernel { class KernelBuilder; }
@@ -37 +38 @@
     template <typename K, typename V>
     using Map = boost::container::flat_map<K, V>;
-    using ModuleCache = Map<std::string, std::unique_ptr<llvm::MemoryBuffer>>;
+    using ModuleCache = Map<std::string, std::pair<llvm::Module *, std::unique_ptr<llvm::MemoryBuffer>>>;
 public:
     ParabixObjectCache();
-    ParabixObjectCache(const std::string & dir);
+    ParabixObjectCache(const std::string dir);
     bool loadCachedObjectFile(const std::unique_ptr<kernel::KernelBuilder> & idb, kernel::Kernel * const kernel);
-    void notifyObjectCompiled(const llvm::Module *M, llvm::MemoryBufferRef Obj) override;
+    void notifyObjectCompiled(const llvm::Module * M, llvm::MemoryBufferRef Obj) override;
     void cleanUpObjectCacheFiles();
     std::unique_ptr<llvm::MemoryBuffer> getObject(const llvm::Module * M) override;
@@ -48 +49 @@
     static Path getDefaultPath();
 private:
-    ModuleCache     mCachedObject;
-    const Path      mCachePath;
+    ModuleCache         mCachedObject;
+    const Path          mCachePath;
 };