Changeset 5771

Timestamp:

Dec 9, 2017, 5:05:16 PM (15 months ago)

Author:

nmedfort

Message:

Minor changes and hopefully a fix for bug exposed by base64 test

Location:

icGREP/icgrep-devel/icgrep

Files:

• IR_Gen/CBuilder.cpp (modified) (3 diffs)
• IR_Gen/CBuilder.h (modified) (3 diffs)
• base64.cpp (modified) (1 diff)
• grep_engine.cpp (modified) (15 diffs)
• grep_engine.h (modified) (3 diffs)
• kernels/kernel.cpp (modified) (16 diffs)
• kernels/p2s_kernel.cpp (modified) (2 diffs)
• kernels/source_kernel.cpp (modified) (1 diff)
• pablo/carry_manager.cpp (modified) (1 diff)
• pablo/carrypack_manager.cpp (modified) (1 diff)
• toolchain/object_cache.cpp (modified) (3 diffs)
• toolchain/object_cache.h (modified) (2 diffs)
• toolchain/toolchain.cpp (modified) (1 diff)
• toolchain/toolchain.h (modified) (2 diffs)

icGREP/icgrep-devel/icgrep/IR_Gen/CBuilder.cpp

@@ -93 +93 @@
     DataLayout DL(b->getModule());
     IntegerType * const intPtrTy = cast<IntegerType>(DL.getIntPtrType(Ptr->getType()));
-    Value * sz = ConstantExpr::getTrunc(ConstantExpr::getSizeOf(Base->getAllocatedType()), intPtrTy);
+    Value * sz = ConstantExpr::getBitCast(ConstantExpr::getSizeOf(Base->getAllocatedType()), intPtrTy);
     if (dyn_cast_or_null<Constant>(Base->getArraySize()) && !cast<Constant>(Base->getArraySize())->isNullValue()) {
         sz = b->CreateMul(sz, b->CreateZExtOrTrunc(Base->getArraySize(), intPtrTy));
@@ -966 +966 @@
 #endif
 
-void CBuilder::__CreateAssert(Value * const assertion, const Twine failureMessage) {
+void CBuilder::__CreateAssert(Value * const assertion, const Twine & failureMessage) {
     if (LLVM_UNLIKELY(isa<Constant>(assertion))) {
         if (LLVM_UNLIKELY(cast<Constant>(assertion)->isNullValue())) {
@@ -1082 +1082 @@
         IRBuilder<>::CreateCall(function, {assertion, GetString(failureMessage.toStringRef(tmp)), trace, depth});
     } else { // if assertions are not enabled, make it a compiler assumption.
-        IRBuilder<>::CreateAssumption(assertion);
+
+        // INVESTIGATE: while interesting, this does not seem to produce faster code and only provides a trivial reduction
+        // of compiled code size in LLVM 3.8 but nearly doubles compilation time. This may have been improved with later
+        // versions of LLVM but it's likely that assumptions ought to be hand placed once they're prove to improve performance.
+
+        // IRBuilder<>::CreateAssumption(assertion);
     }
 }

icGREP/icgrep-devel/icgrep/IR_Gen/CBuilder.h

@@ -236 +236 @@
     virtual llvm::StoreInst *  CreateAtomicStoreRelease(llvm::Value * val, llvm::Value * ptr);
 
-    void CreateAssert(llvm::Value * assertion, const llvm::Twine failureMessage) {
+    void CreateAssert(llvm::Value * assertion, const llvm::Twine & failureMessage) {
         if (LLVM_UNLIKELY(assertion->getType()->isVectorTy())) {
             assertion = CreateBitCast(assertion, getIntNTy(assertion->getType()->getPrimitiveSizeInBits()));
@@ -243 +243 @@
     }
 
-    void CreateAssertZero(llvm::Value * assertion, const llvm::Twine failureMessage) {
+    void CreateAssertZero(llvm::Value * assertion, const llvm::Twine & failureMessage) {
         if (LLVM_UNLIKELY(assertion->getType()->isVectorTy())) {
             assertion = CreateBitCast(assertion, getIntNTy(assertion->getType()->getPrimitiveSizeInBits()));
@@ -355 +355 @@
     bool hasAddressSanitizer() const;
 
-    void __CreateAssert(llvm::Value * assertion, const llvm::Twine failureMessage);
+    void __CreateAssert(llvm::Value * assertion, const llvm::Twine & failureMessage);
 
     llvm::Function * LinkFunction(llvm::StringRef name, llvm::FunctionType * type, void * functionPtr) const;

icGREP/icgrep-devel/icgrep/base64.cpp

@@ -53 +53 @@
     Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, int32Ty, outputType, nullptr));
     main->setCallingConv(CallingConv::C);
-    Function::arg_iterator args = main->arg_begin();
+    auto args = main->arg_begin();
     
     Value * const fileDescriptor = &*(args++);

icGREP/icgrep-devel/icgrep/grep_engine.cpp

@@ -85 +85 @@
     const unsigned n = filenames.size();
     mResultStrs.resize(n);
-    mFileStatus.resize(n);
-    for (unsigned i = 0; i < n; i++) {
-        mResultStrs[i] = make_unique<std::stringstream>();
-        mFileStatus[i] = FileStatus::Pending;
-    }
+    mFileStatus.resize(n, FileStatus::Pending);
     inputFiles = filenames;
 }
@@ -221 +217 @@
     friend class EmitMatchesEngine;
 public:
-    EmitMatch(std::string linePrefix, std::stringstream * strm) : mLinePrefix(linePrefix), mLineCount(0), mTerminated(true), mResultStr(strm) {}
+    EmitMatch(std::string linePrefix, std::ostringstream & strm) : mLinePrefix(linePrefix), mLineCount(0), mTerminated(true), mResultStr(strm) {}
     void accumulate_match(const size_t lineNum, char * line_start, char * line_end) override;
     void finalize_match(char * buffer_end) override;
@@ -228 +224 @@
     size_t mLineCount;
     bool mTerminated;
-    std::stringstream* mResultStr;
+    std::ostringstream & mResultStr;
 };
 
@@ -237 +233 @@
 void EmitMatch::accumulate_match (const size_t lineNum, char * line_start, char * line_end) {
     if (WithFilenameFlag) {
-        *mResultStr << mLinePrefix;
+        mResultStr << mLinePrefix;
     }
     if (LineNumberFlag) {
@@ -243 +239 @@
         // the line number so that lines are numbered from 1.
         if (InitialTabFlag) {
-            *mResultStr << lineNum+1 << "\t:";
+            mResultStr << lineNum+1 << "\t:";
         }
         else {
-            *mResultStr << lineNum+1 << ":";
+            mResultStr << lineNum+1 << ":";
         }
     }
     size_t bytes = line_end - line_start + 1;
-    mResultStr->write(line_start, bytes);
+    mResultStr.write(line_start, bytes);
     mLineCount++;
     unsigned last_byte = *line_end;
@@ -268 +264 @@
 
 void EmitMatch::finalize_match(char * buffer_end) {
-    if (!mTerminated) *mResultStr << "\n";
+    if (!mTerminated) mResultStr << "\n";
 }
 
@@ -321 +317 @@
     auto f = reinterpret_cast<GrepFunctionType>(mGrepDriver->getMain());
 
-    int32_t fileDescriptor = openFile(fileName, mResultStrs[fileIdx].get());
+    int32_t fileDescriptor = openFile(fileName, mResultStrs[fileIdx]);
     if (fileDescriptor == -1) return 0;
 
@@ -331 +327 @@
 uint64_t CountOnlyEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) {
     uint64_t grepResult = GrepEngine::doGrep(fileName, fileIdx);
-    if (WithFilenameFlag) *mResultStrs[fileIdx] << linePrefix(fileName);
-    *mResultStrs[fileIdx] << grepResult << "\n";
+    if (WithFilenameFlag) mResultStrs[fileIdx] << linePrefix(fileName);
+    mResultStrs[fileIdx] << grepResult << "\n";
     return grepResult;
 }
@@ -348 +344 @@
     uint64_t grepResult = GrepEngine::doGrep(fileName, fileIdx);
     if (grepResult == mRequiredCount) {
-       *mResultStrs[fileIdx] << linePrefix(fileName);
+       mResultStrs[fileIdx] << linePrefix(fileName);
     }
     return grepResult;
@@ -357 +353 @@
     auto f = reinterpret_cast<GrepFunctionType>(mGrepDriver->getMain());
 
-    int32_t fileDescriptor = openFile(fileName, mResultStrs[fileIdx].get());
+    int32_t fileDescriptor = openFile(fileName, mResultStrs[fileIdx]);
     if (fileDescriptor == -1) return 0;
-    EmitMatch accum(linePrefix(fileName), mResultStrs[fileIdx].get());
+    EmitMatch accum(linePrefix(fileName), mResultStrs[fileIdx]);
     f(fileDescriptor, reinterpret_cast<intptr_t>(&accum));
     close(fileDescriptor);
@@ -367 +363 @@
 
 // Open a file and return its file desciptor.
-int32_t GrepEngine::openFile(const std::string & fileName, std::stringstream * msgstrm) {
+int32_t GrepEngine::openFile(const std::string & fileName, std::ostringstream & msgstrm) {
     if (fileName == "-") {
         return STDIN_FILENO;
@@ -377 +373 @@
             if (!NoMessagesFlag) {
                 if (errno == EACCES) {
-                    *msgstrm << "icgrep: " << fileName << ": Permission denied.\n";
+                    msgstrm << "icgrep: " << fileName << ": Permission denied.\n";
                 }
                 else if (errno == ENOENT) {
-                    *msgstrm << "icgrep: " << fileName << ": No such file.\n";
+                    msgstrm << "icgrep: " << fileName << ": No such file.\n";
                 }
                 else {
-                    *msgstrm << "icgrep: " << fileName << ": Failed.\n";
+                    msgstrm << "icgrep: " << fileName << ": Failed.\n";
                 }
             }
@@ -390 +386 @@
         if (stat(fileName.c_str(), &sb) == 0 && S_ISDIR(sb.st_mode)) {
             if (!NoMessagesFlag) {
-                *msgstrm << "icgrep: " << fileName << ": Is a directory.\n";
+                msgstrm << "icgrep: " << fileName << ": Is a directory.\n";
             }
             close(fileDescriptor);
@@ -433 +429 @@
 void * GrepEngine::DoGrepThreadMethod() {
 
-    auto fileIdx = mNextFileToGrep++;
+    unsigned fileIdx = mNextFileToGrep++;
     while (fileIdx < inputFiles.size()) {
-        const size_t grepResult = doGrep(inputFiles[fileIdx], fileIdx);
+        const auto grepResult = doGrep(inputFiles[fileIdx], fileIdx);
         mFileStatus[fileIdx] = FileStatus::GrepComplete;
         if (grepResult > 0) {
@@ -449 +445 @@
     }
 
-    auto printIdx = mNextFileToPrint++;
+    unsigned printIdx = mNextFileToPrint++;
     while (printIdx < inputFiles.size()) {
         const bool readyToPrint = ((printIdx == 0) || (mFileStatus[printIdx - 1] == FileStatus::PrintComplete)) && (mFileStatus[printIdx] == FileStatus::GrepComplete);
         if (readyToPrint) {
-            const auto output = mResultStrs[printIdx]->str();
+            const auto output = mResultStrs[printIdx].str();
             if (!output.empty()) {
-                mWriteMutex.lock();
-                std::cout << output;
-                mWriteMutex.unlock();
+                llvm::outs() << output;
             }
             mFileStatus[printIdx] = FileStatus::PrintComplete;
             printIdx = mNextFileToPrint++;
         } else {
-            mCacheMutex.lock();
             mGrepDriver->performIncrementalCacheCleanupStep();
-            mCacheMutex.unlock();
         }
         sched_yield();

icGREP/icgrep-devel/icgrep/grep_engine.h

@@ -9 +9 @@
 #include <kernels/streamset.h>
 #include <toolchain/grep_pipeline.h>
-#include <string>       // for string
+#include <string>
 #include <vector>
 #include <sstream>
-#include <mutex>
 #include <atomic>
 
@@ -40 +39 @@
     virtual uint64_t doGrep(const std::string & fileName, const uint32_t fileIdx);
     std::string linePrefix(std::string fileName);
-    int32_t openFile(const std::string & fileName, std::stringstream * msgstrm);
+    int32_t openFile(const std::string & fileName, std::ostringstream & msgstrm);
 
     Driver * mGrepDriver;
@@ -47 +46 @@
     std::atomic<unsigned> mNextFileToPrint;
     std::vector<std::string> inputFiles;
-    std::vector<std::unique_ptr<std::stringstream>> mResultStrs;
+    std::vector<std::ostringstream> mResultStrs;
     std::vector<FileStatus> mFileStatus;
-    std::mutex mWriteMutex;
-    std::mutex mCacheMutex;
+    bool grepMatchFound;
 
-
-    bool grepMatchFound;
-    
     std::string mFileSuffix;
     bool mMoveMatchesToEOL;

icGREP/icgrep-devel/icgrep/kernels/kernel.cpp

@@ -680 +680 @@
     // Define and allocate the temporary buffer area in the prolog.
     const auto blockAlignment = b->getBitBlockWidth() / 8;
-    Value * temporaryInputBuffer[inputSetCount];
+    AllocaInst * temporaryInputBuffer[inputSetCount];
     for (unsigned i = 0; i < inputSetCount; ++i) {
-
-        // TODO: if this is a fixed rate input stream and the pipeline guarantees it will not call the kernel unless
-        // there is sufficient input and all buffers will be sized sufficiently for the input, we ought to be able to
-        // avoid the temporary buffer checks.
-
-        const ProcessingRate & rate = mStreamSetInputs[i].getRate();
-        Type * const ty = mStreamSetInputBuffers[i]->getStreamSetBlockType();
-        const auto ub = getUpperBound(rate);
-        if (ub.numerator() == 0) {
+        const auto & input = mStreamSetInputs[i];
+        const ProcessingRate & rate = input.getRate();
+        if (isTransitivelyUnknownRate(rate)) {
             report_fatal_error("MultiBlock kernels do not support unknown rate input streams or streams relative to an unknown rate input.");
-        } else {            
-            temporaryInputBuffer[i] = b->CreateAlignedAlloca(ty, blockAlignment, b->getSize(roundUp(ub)));
-            Type * const sty = temporaryInputBuffer[i]->getType()->getPointerElementType();
-            b->CreateStore(Constant::getNullValue(sty), temporaryInputBuffer[i]);
-        }        
-    }
-
-    Value * temporaryOutputBuffer[outputSetCount];
+        } else if (rate.isFixed() && input.nonDeferred() && !requiresBufferedFinalStride(input)) {
+            temporaryInputBuffer[i] = nullptr;
+        } else {
+            Type * const ty = mStreamSetInputBuffers[i]->getStreamSetBlockType();
+            const auto ub = getUpperBound(rate);
+            Constant * arraySize = b->getInt64(roundUp(ub));
+            AllocaInst * const ptr = b->CreateAlignedAlloca(ty, blockAlignment, arraySize);
+            assert (ptr->isStaticAlloca());
+            temporaryInputBuffer[i] = ptr;
+        }
+    }
+
+    AllocaInst * temporaryOutputBuffer[outputSetCount];
     for (unsigned i = 0; i < outputSetCount; i++) {
-        const ProcessingRate & rate = mStreamSetOutputs[i].getRate();
-        Type * const ty = mStreamSetOutputBuffers[i]->getStreamSetBlockType();
-        if (LLVM_UNLIKELY(isTransitivelyUnknownRate(rate))) {
+        const auto & output = mStreamSetOutputs[i];
+        const ProcessingRate & rate = output.getRate();
+        if (LLVM_UNLIKELY(isTransitivelyUnknownRate(rate) || (rate.isFixed() && output.nonDeferred() && !requiresBufferedFinalStride(output)))) {
             temporaryOutputBuffer[i] = nullptr;
         } else {            
@@ -710 +709 @@
                 ub += mStreamSetOutputBuffers[i]->overflowSize();
             }
-            temporaryOutputBuffer[i] = b->CreateAlignedAlloca(ty, blockAlignment, b->getSize(roundUp(ub)));
-            Type * const sty = temporaryOutputBuffer[i]->getType()->getPointerElementType();
-            b->CreateStore(Constant::getNullValue(sty), temporaryOutputBuffer[i]);
+            Type * const ty = mStreamSetOutputBuffers[i]->getStreamSetBlockType();
+            Constant * arraySize = b->getInt64(roundUp(ub));
+            AllocaInst * const ptr = b->CreateAlignedAlloca(ty, blockAlignment, arraySize);
+            assert (ptr->isStaticAlloca());
+            temporaryOutputBuffer[i] = ptr;
         }
     }
@@ -751 +752 @@
         Value * const ic = b->getProcessedItemCount(name);
         mInitialProcessedItemCount[i] = ic;
-        b->CreateAssert(b->CreateICmpUGE(mAvailableItemCount[i], ic), "processed item count cannot exceed the available item count");
+        if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
+            b->CreateAssert(b->CreateICmpUGE(mAvailableItemCount[i], ic),
+                            "processed item count cannot exceed the available item count");
+        }
         assert (ic->getType() == mAvailableItemCount[i]->getType());
         Value * const unprocessed = b->CreateSub(mAvailableItemCount[i], ic);
-
-        mStreamSetInputBaseAddress[i]  = b->getBlockAddress(name, b->CreateLShr(ic, LOG_2_BLOCK_WIDTH));
+        Value * baseBuffer  = b->getBlockAddress(name, b->CreateLShr(ic, LOG_2_BLOCK_WIDTH));
         mInitialAvailableItemCount[i] = mAvailableItemCount[i];
         mAvailableItemCount[i] = b->getLinearlyAccessibleItems(name, ic, unprocessed);
@@ -761 +764 @@
         // Are our linearly accessible items sufficient for a stride?
         inputStrideSize[i] = getStrideSize(b, rate);
-
         Value * accessibleStrides = b->CreateUDiv(mAvailableItemCount[i], inputStrideSize[i]);
-        if (!rate.isFixed() || (requiresBufferedFinalStride(input) && input.nonDeferred())) {
+        AllocaInst * const tempBuffer = temporaryInputBuffer[i];
+        if (tempBuffer) {
 
             // Since we trust that the pipeline won't call this kernel unless there is enough data to process a stride, whenever
@@ -777 +780 @@
             b->SetInsertPoint(copyFromBack);
             Value * const temporaryAvailable = b->CreateUMin(unprocessed, inputStrideSize[i]);
-
-            b->CreateAssert(b->CreateICmpULE(mAvailableItemCount[i], temporaryAvailable), "linearly available cannot be greater than temporarily available");
-            Value * const tempBufferPtr = temporaryInputBuffer[i];
+            if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
+                b->CreateAssert(b->CreateICmpULE(mAvailableItemCount[i], temporaryAvailable),
+                                "linearly available cannot be greater than temporarily available");
+            }
             Value * const offset = b->CreateAnd(ic, BLOCK_WIDTH_MASK);
+            Value * const bufferSize = b->CreateMul(ConstantExpr::getSizeOf(tempBuffer->getAllocatedType()), tempBuffer->getArraySize());
+            b->CreateMemZero(tempBuffer, bufferSize, blockAlignment);
             const auto copyAlignment = getItemAlignment(mStreamSetInputs[i]);
-            b->CreateMemZero(tempBufferPtr, ConstantExpr::getSizeOf(tempBufferPtr->getType()), blockAlignment);
-            b->CreateStreamCpy(name, tempBufferPtr, ZERO, mStreamSetInputBaseAddress[i] , offset, mAvailableItemCount[i], copyAlignment);
+            b->CreateStreamCpy(name, tempBuffer, ZERO, baseBuffer, offset, mAvailableItemCount[i], copyAlignment);
             Value * const temporaryStrides = b->CreateSelect(b->CreateICmpULT(unprocessed, inputStrideSize[i]), ZERO, ONE);
             BasicBlock * const copyToBackEnd = b->GetInsertBlock();
@@ -791 +796 @@
             Value * const remaining = b->CreateSub(temporaryAvailable, mAvailableItemCount[i]);
             Value * const baseAddress = b->getBaseAddress(name);
-            b->CreateStreamCpy(name, tempBufferPtr, mAvailableItemCount[i], baseAddress, ZERO, remaining, copyAlignment);
+            b->CreateStreamCpy(name, tempBuffer, mAvailableItemCount[i], baseAddress, ZERO, remaining, copyAlignment);
             BasicBlock * const copyToFrontEnd = b->GetInsertBlock();
             b->CreateBr(resume);
 
             b->SetInsertPoint(resume);
-            PHINode * const bufferPtr = b->CreatePHI(mStreamSetInputBaseAddress[i] ->getType(), 3);
-            bufferPtr->addIncoming(mStreamSetInputBaseAddress[i] , entry);
-            bufferPtr->addIncoming(tempBufferPtr, copyToBackEnd);
-            bufferPtr->addIncoming(tempBufferPtr, copyToFrontEnd);
-            mStreamSetInputBaseAddress[i] = bufferPtr;
+            PHINode * const bufferPtr = b->CreatePHI(baseBuffer->getType(), 3);
+            bufferPtr->addIncoming(baseBuffer , entry);
+            bufferPtr->addIncoming(tempBuffer, copyToBackEnd);
+            bufferPtr->addIncoming(tempBuffer, copyToFrontEnd);
+            baseBuffer = bufferPtr;
 
             PHINode * const phiAvailItemCount = b->CreatePHI(b->getSizeTy(), 3);
@@ -808 +813 @@
             mAvailableItemCount[i] = phiAvailItemCount;
 
-            PHINode * const phiNumOfStrides = b->CreatePHI(b->getSizeTy(), 2);
-            phiNumOfStrides->addIncoming(accessibleStrides, entry);
-            phiNumOfStrides->addIncoming(temporaryStrides, copyToBackEnd);
-            phiNumOfStrides->addIncoming(temporaryStrides, copyToFrontEnd);
-            accessibleStrides = phiNumOfStrides;
-        }
+            PHINode * const phiStrides = b->CreatePHI(b->getSizeTy(), 2);
+            phiStrides->addIncoming(accessibleStrides, entry);
+            phiStrides->addIncoming(temporaryStrides, copyToBackEnd);
+            phiStrides->addIncoming(temporaryStrides, copyToFrontEnd);
+            accessibleStrides = phiStrides;
+        }
+
+        mStreamSetInputBaseAddress[i] = baseBuffer;
         numOfStrides = b->CreateUMin(numOfStrides, accessibleStrides);
     }
@@ -819 +826 @@
     // Now determine the linearly writeable strides
     Value * linearlyWritable[outputSetCount];
-    Value * baseOutputBuffer[outputSetCount];
     Value * outputStrideSize[outputSetCount];
     mInitialProducedItemCount.resize(outputSetCount);
@@ -828 +834 @@
         const ProcessingRate & rate = output.getRate();
         Value * const ic = b->getProducedItemCount(name);
-        baseOutputBuffer[i] = b->getBlockAddress(name, b->CreateLShr(ic, LOG_2_BLOCK_WIDTH));
-        assert (baseOutputBuffer[i]->getType()->isPointerTy());
-        linearlyWritable[i] = b->getLinearlyWritableItems(name, ic);
+        Value * baseBuffer = b->getBlockAddress(name, b->CreateLShr(ic, LOG_2_BLOCK_WIDTH));
+        assert (baseBuffer->getType()->isPointerTy());
+        linearlyWritable[i] = b->getLinearlyWritableItems(name, ic);        
+        outputStrideSize[i] = getStrideSize(b, rate);
+        // Is the number of linearly writable items sufficient for a stride?
+        if (outputStrideSize[i]) {
+            AllocaInst * const tempBuffer = temporaryOutputBuffer[i];
+            Value * writableStrides = b->CreateUDiv(linearlyWritable[i], outputStrideSize[i]);
+            // Do we require a temporary buffer to write to?
+            if (tempBuffer) {
+                assert (tempBuffer->getType() == baseBuffer->getType());
+                BasicBlock * const entry = b->GetInsertBlock();
+                BasicBlock * const useTemporary = b->CreateBasicBlock(name + "UseTemporary");
+                BasicBlock * const resume = b->CreateBasicBlock(name + "Resume");
+                Value * const requiresCopy = b->CreateICmpEQ(writableStrides, ZERO);
+
+                b->CreateUnlikelyCondBr(requiresCopy, useTemporary, resume);
+
+                // Clear the buffer after use since we may end up reusing it within the same stride
+                b->SetInsertPoint(useTemporary);
+                Value * const bufferSize = b->CreateMul(ConstantExpr::getSizeOf(tempBuffer->getAllocatedType()), tempBuffer->getArraySize());
+                b->CreateMemZero(tempBuffer, bufferSize, blockAlignment);
+                b->CreateBr(resume);
+
+                b->SetInsertPoint(resume);
+                PHINode * const phiBuffer = b->CreatePHI(baseBuffer->getType(), 3);
+                phiBuffer->addIncoming(baseBuffer, entry);
+                phiBuffer->addIncoming(tempBuffer, useTemporary);
+                baseBuffer = phiBuffer;
+                PHINode * const phiStrides = b->CreatePHI(b->getSizeTy(), 2);
+                phiStrides->addIncoming(writableStrides, entry);
+                phiStrides->addIncoming(ONE, useTemporary);
+                writableStrides = phiStrides;
+
+            }
+            numOfStrides = b->CreateUMin(numOfStrides, writableStrides);
+        }
         mInitialProducedItemCount[i] = ic;
-        outputStrideSize[i] = nullptr;
-        if (temporaryOutputBuffer[i]) {
-            outputStrideSize[i] = getStrideSize(b, rate);
-            // Is the number of linearly writable items sufficient for a stride?
-            Value * writableStrides = b->CreateUDiv(linearlyWritable[i], outputStrideSize[i]);
-            if (!rate.isFixed() || requiresBufferedFinalStride(output)) {
-                Value * const requiresCopy = b->CreateICmpEQ(writableStrides, ZERO);
-                assert (temporaryOutputBuffer[i]->getType() == baseOutputBuffer[i]->getType());
-                baseOutputBuffer[i] = b->CreateSelect(requiresCopy, temporaryOutputBuffer[i], baseOutputBuffer[i]);
-                writableStrides = b->CreateSelect(requiresCopy, ONE, writableStrides);
-            }
-            numOfStrides = b->CreateUMin(numOfStrides, writableStrides);
-            assert (temporaryOutputBuffer[i]->getType() == baseOutputBuffer[i]->getType());
-        }
-        mStreamSetOutputBaseAddress[i] = baseOutputBuffer[i];
+        mStreamSetOutputBaseAddress[i] = baseBuffer;
     }
 
@@ -854 +880 @@
     if (LLVM_LIKELY(numOfStrides != nullptr)) {
         mIsFinal = b->CreateAnd(mIsFinal, b->CreateICmpEQ(numOfStrides, ZERO));
-        Value * const hasStride = b->CreateOr(b->CreateICmpNE(numOfStrides, ZERO), mIsFinal);
-        b->CreateAssert(hasStride, getName() + " has insufficient input data or output space for one stride");
+        if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
+            Value * const hasStride = b->CreateOr(b->CreateICmpNE(numOfStrides, ZERO), mIsFinal);
+            b->CreateAssert(hasStride, getName() + " has insufficient input data or output space for one stride");
+        }
         for (unsigned i = 0; i < inputSetCount; ++i) {
             const ProcessingRate & rate = mStreamSetInputs[i].getRate();
@@ -908 +936 @@
     // Copy back data to the actual output buffers.
     for (unsigned i = 0; i < outputSetCount; i++) {
-        Value * const tempBuffer = temporaryOutputBuffer[i];
+        AllocaInst * const tempBuffer = temporaryOutputBuffer[i];
         if (LLVM_UNLIKELY(tempBuffer == nullptr)) {
             continue;
         }
-        Value * const baseBuffer = baseOutputBuffer[i];
+        Value * const baseBuffer = mStreamSetOutputBaseAddress[i];
         assert ("stack corruption likely" && (tempBuffer->getType() == baseBuffer->getType()));
         const auto & name = mStreamSetOutputs[i].getName();
         BasicBlock * const copyToBack = b->CreateBasicBlock(name + "CopyToBack");
         BasicBlock * const copyToFront = b->CreateBasicBlock(name + "CopyToFront");
-        BasicBlock * const clearBuffer = b->CreateBasicBlock(name + "ClearBuffer");
         BasicBlock * const resume = b->CreateBasicBlock(name + "ResumeCopyBack");
         // If we used a temporary buffer, copy it back to the original output buffer
@@ -930 +957 @@
         b->CreateStreamCpy(name, baseBuffer, offset, tempBuffer, ZERO, toWrite, alignment);
         // If we required a temporary output buffer, we will probably need to write to the beginning of the buffer as well.
-        b->CreateLikelyCondBr(b->CreateICmpULT(toWrite, newlyProduced), copyToFront, clearBuffer);
+        b->CreateLikelyCondBr(b->CreateICmpULT(toWrite, newlyProduced), copyToFront, resume);
 
         b->SetInsertPoint(copyToFront);
@@ -936 +963 @@
         Value * const baseAddress = b->getBaseAddress(name);
         b->CreateStreamCpy(name, baseAddress, ZERO, tempBuffer, toWrite, remaining, alignment);
-        b->CreateBr(clearBuffer);
-        // Clear the buffer after use since we may end up reusing it within the same stride
-        b->SetInsertPoint(clearBuffer);
-
         b->CreateBr(resume);
 
@@ -970 +993 @@
         Value * const avail = mInitialAvailableItemCount[i];
         Value * const processed = b->getProcessedItemCount(name);
-        b->CreateAssert(b->CreateICmpULE(processed, avail), name + ": processed data cannot exceed available data");
+        if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
+            b->CreateAssert(b->CreateICmpULE(processed, avail), name + ": processed data cannot exceed available data");
+        }
         Value * const remaining = b->CreateSub(avail, processed);
         Value * const remainingStrides = b->CreateUDiv(remaining, inputStrideSize[i]);
@@ -987 +1012 @@
         if (LLVM_LIKELY(outputStrideSize[i] != nullptr)) {
             Value * const consumed = b->getConsumedItemCount(name);
-            b->CreateAssert(b->CreateICmpULE(consumed, produced), name + ": consumed data cannot exceed produced data");
+            if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
+                b->CreateAssert(b->CreateICmpULE(consumed, produced), name + ": consumed data cannot exceed produced data");
+            }
             Value * const unconsumed = b->CreateSub(produced, consumed);
             Value * const capacity = b->getCapacity(name);
-            b->CreateAssert(b->CreateICmpULE(unconsumed, capacity), name + ": unconsumed data cannot exceed capacity");
+            if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
+                b->CreateAssert(b->CreateICmpULE(unconsumed, capacity), name + ": unconsumed data cannot exceed capacity");
+            }
             Value * const remaining = b->CreateSub(capacity, unconsumed);
             Value * const remainingStrides = b->CreateUDiv(remaining, outputStrideSize[i]);
@@ -1181 +1210 @@
     BasicBlock * const doFinalBlock = b->CreateBasicBlock(getName() + "_doFinalBlock");
     BasicBlock * const segmentDone = b->CreateBasicBlock(getName() + "_segmentDone");
-    b->CreateAssert(b->CreateXor(b->CreateIsNotNull(numOfBlocks), mIsFinal),
-                    "numOfStrides cannot be 0 unless this is the final stride and must be 0 if it is");
+    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
+        b->CreateAssert(b->CreateXor(b->CreateIsNotNull(numOfBlocks), mIsFinal),
+                        "numOfStrides cannot be 0 unless this is the final stride and must be 0 if it is");
+    }
     const auto inputSetCount = mStreamSetInputs.size();
     Value * baseProcessedIndex[inputSetCount];

icGREP/icgrep-devel/icgrep/kernels/p2s_kernel.cpp

@@ -2 +2 @@
 #include <kernels/streamset.h>
 #include <kernels/kernel_builder.h>
+#include <toolchain/toolchain.h>
+#include <llvm/Support/Compiler.h>
 
 namespace llvm { class Value; }
@@ -131 +133 @@
         b->CreateAlignedStore(merge0, b->CreateBitCast(b->CreateGEP(outputPtr, offset), bitBlockPtrTy), 1);
         Value * const nextOffset1 = b->CreateZExt(b->CreateExtractElement(unitCounts, b->getInt32(2 * j)), i32Ty);
-        b->CreateAssert(b->CreateICmpULE(offset, nextOffset1), "deletion offset is not monotonically non-decreasing");
-
+        if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
+            b->CreateAssert(b->CreateICmpULE(offset, nextOffset1), "deletion offset is not monotonically non-decreasing");
+        }
         Value * const merge1 = b->bitCast(b->esimd_mergeh(8, hi_bytes[j], lo_bytes[j]));
         b->CreateAlignedStore(merge1, b->CreateBitCast(b->CreateGEP(outputPtr, nextOffset1), bitBlockPtrTy), 1);
         Value * const nextOffset2 = b->CreateZExt(b->CreateExtractElement(unitCounts, b->getInt32(2 * j + 1)), i32Ty);
-        b->CreateAssert(b->CreateICmpULE(nextOffset1, nextOffset2), "deletion offset is not monotonically non-decreasing");
-
+        if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
+            b->CreateAssert(b->CreateICmpULE(nextOffset1, nextOffset2), "deletion offset is not monotonically non-decreasing");
+        }
         offset = nextOffset2;
     }

icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

@@ -232 +232 @@
     Value * const bytesRead = b->CreateReadCall(fd, sourceBuffer, bytesToRead);
     Value * const itemsRead = b->CreateUDiv(bytesRead, codeUnitBytes);
-    b->CreateAssert(b->CreateICmpULE(itemsRead, itemsToRead), "read more items than expected");
     Value * const itemsBuffered = b->CreateAdd(buffered, itemsRead);
     b->setBufferedSize("sourceBuffer", itemsBuffered);

icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp

@@ -429 +429 @@
         b->SetInsertPoint(resume);
 
-        if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
-            b->CreateAssertZero(b->CreateOr(finalBorrow, finalCarry),
-                                       "CarryManager: loop post-condition violated: final borrow and carry must be zero!");
-        }
+        b->CreateAssertZero(b->CreateOr(finalBorrow, finalCarry),
+                                   "CarryManager: loop post-condition violated: final borrow and carry must be zero!");
 
         assert (!mLoopIndicies.empty());

icGREP/icgrep-devel/icgrep/pablo/carrypack_manager.cpp

@@ -452 +452 @@
         iBuilder->SetInsertPoint(resume);
 
-        if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
-            iBuilder->CreateAssertZero(iBuilder->CreateOr(finalBorrow, finalCarry),
-                                       "CarryPackManager: loop post-condition violated: final borrow and carry must be zero!");
-        }
+        iBuilder->CreateAssertZero(iBuilder->CreateOr(finalBorrow, finalCarry),
+                                   "CarryPackManager: loop post-condition violated: final borrow and carry must be zero!");
 
         assert (!mLoopIndicies.empty());

icGREP/icgrep-devel/icgrep/toolchain/object_cache.cpp

@@ -119 +119 @@
                 if (kernel->hasSignature()) {
                     const MDString * const sig = getSignature(M.get());
-                    assert ("signature is missing from kernel file: possible module naming conflict?" && sig);
+                    assert ("signature is missing from kernel file: possible module naming conflict or change in the LLVM metadata storage policy?" && sig);
                     if (LLVM_UNLIKELY(sig == nullptr || !sig->getString().equals(kernel->makeSignature(idb)))) {
                         goto invalid;
@@ -200 +200 @@
 
 void ParabixObjectCache::performIncrementalCacheCleanupStep() {
-    if (mCacheCleanupIterator != fs::directory_iterator()) {
-        const auto e = mCacheCleanupIterator->path();
-        mCacheCleanupIterator++;
+    mCleanupMutex.lock();
+    if (LLVM_UNLIKELY(mCleanupIterator == fs::directory_iterator())) {
+        mCleanupMutex.unlock();
+    } else {
+        const auto e = mCleanupIterator->path();
+        mCleanupIterator++;
+        mCleanupMutex.unlock();
+
         // Simple clean-up policy: files that haven't been touched by the
         // driver in MaxCacheEntryHours are deleted.
@@ -232 +237 @@
         sys::fs::create_directories(mCachePath);
     }
-    fs::directory_iterator it(p);
-    mCacheCleanupIterator = it;
+    mCleanupIterator = fs::directory_iterator(p);
 }
 

icGREP/icgrep-devel/icgrep/toolchain/object_cache.h

@@ -15 +15 @@
 #include <vector>
 #include <string>
+#include <mutex>
 
 namespace llvm { class Module; }
@@ -48 +49 @@
     void performIncrementalCacheCleanupStep();
 private:
+    std::mutex mCleanupMutex;
+    boost::filesystem::directory_iterator mCleanupIterator;
     ModuleCache mCachedObject;
     const Path mCachePath;
-    boost::filesystem::directory_iterator mCacheCleanupIterator;
 };
 

icGREP/icgrep-devel/icgrep/toolchain/toolchain.cpp

@@ -123 +123 @@
 TargetOptions Options;
 
-const cl::OptionCategory * codegen_flags() {
+const cl::OptionCategory * LLVM_READONLY codegen_flags() {
     return &CodeGenOptions;
 }
 
-bool DebugOptionIsSet(const DebugFlags flag) {
+bool LLVM_READONLY DebugOptionIsSet(const DebugFlags flag) {
     return DebugOptions.isSet(flag);
 }

icGREP/icgrep-devel/icgrep/toolchain/toolchain.h

@@ -38 +38 @@
 namespace codegen {
 
-const llvm::cl::OptionCategory * codegen_flags();
+const llvm::cl::OptionCategory * LLVM_READONLY codegen_flags();
 
 // Command Parameters
@@ -52 +52 @@
 };
 
-bool DebugOptionIsSet(const DebugFlags flag);
+bool LLVM_READONLY DebugOptionIsSet(const DebugFlags flag);
 
 extern bool PipelineParallel;