Ignore:
Timestamp:
May 10, 2017, 4:26:11 PM (2 years ago)
Author:
nmedfort
Message:

Large refactoring step. Removed IR generation code from Kernel (formally KernelBuilder?) and moved it into the new KernelBuilder? class.

Location:
icGREP/icgrep-devel/icgrep/pablo
Files:
10 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/pablo/builder.hpp

    r5436 r5440  
    197197        return mPb->createWhile(condition, builder.mPb);
    198198    }
    199 
    200 //    llvm::Type * getStreamTy(const unsigned FieldWidth = 1) {
    201 //        return mPb->getStreamTy(FieldWidth);
    202 //    }
    203    
    204 //    llvm::Type * getStreamSetTy(const unsigned NumElements = 1, const unsigned FieldWidth = 1) {
    205 //        return mPb->getStreamSetTy(NumElements, FieldWidth);
    206 //    }
    207199   
    208200    /// Statement Iterator Wrappers
  • icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp

    r5435 r5440  
    99#include <pablo/codegenstate.h>
    1010#include <llvm/IR/BasicBlock.h>
    11 #include <IR_Gen/idisa_builder.h>
    1211#include <llvm/IR/DerivedTypes.h>
    1312#include <pablo/branch.h>
     
    1615#include <pablo/pe_matchstar.h>
    1716#include <pablo/pe_var.h>
    18 
     17#include <kernels/kernel_builder.h>
    1918#include <llvm/Support/raw_ostream.h>
    2019
     
    6261 * @brief initializeCarryData
    6362 ** ------------------------------------------------------------------------------------------------------------- */
    64 void CarryManager::initializeCarryData(IDISA::IDISA_Builder * const builder, PabloKernel * const kernel) {
     63void CarryManager::initializeCarryData(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, PabloKernel * const kernel) {
    6564
    6665    // Each scope constructs its own CarryData struct, which will be added to the final "carries" struct
     
    8887    mCarryMetadata.resize(getScopeCount(mCurrentScope));
    8988
    90     Type * const carryStateTy = analyse(builder, mCurrentScope);
     89    Type * const carryStateTy = analyse(iBuilder, mCurrentScope);
    9190
    9291    kernel->addScalar(carryStateTy, "carries");
    9392
    9493    if (mHasLoop) {
    95         kernel->addScalar(builder->getInt32Ty(), "selector");
     94        kernel->addScalar(iBuilder->getInt32Ty(), "selector");
    9695    }
    9796    if (mHasLongAdvance) {
    98         kernel->addScalar(builder->getSizeTy(), "CarryBlockIndex");
     97        kernel->addScalar(iBuilder->getSizeTy(), "CarryBlockIndex");
    9998    }
    10099
     
    104103 * @brief initializeCodeGen
    105104 ** ------------------------------------------------------------------------------------------------------------- */
    106 void CarryManager::initializeCodeGen(IDISA::IDISA_Builder * const builder) {
     105void CarryManager::initializeCodeGen(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder) {
    107106
    108107    assert(!mCarryMetadata.empty());
     
    110109    assert (!mCarryInfo->hasSummary());
    111110
    112     mCurrentFrame = mKernel->getScalarFieldPtr("carries");
     111    mCurrentFrame = iBuilder->getScalarFieldPtr("carries");
    113112    mCurrentFrameIndex = 0;
    114113    mCarryScopes = 0;
     
    119118    assert (mCarrySummaryStack.empty());
    120119
    121     Type * const carryTy = builder->getBitBlockType();
     120    Type * const carryTy = iBuilder->getBitBlockType();
    122121
    123122    mCarrySummaryStack.push_back(Constant::getNullValue(carryTy));
    124123
    125124    if (mHasLoop) {       
    126         mLoopSelector = mKernel->getScalarField("selector");
    127         mNextLoopSelector = builder->CreateXor(mLoopSelector, ConstantInt::get(mLoopSelector->getType(), 1));
     125        mLoopSelector = iBuilder->getScalarField("selector");
     126        mNextLoopSelector = iBuilder->CreateXor(mLoopSelector, ConstantInt::get(mLoopSelector->getType(), 1));
    128127    }
    129128
     
    133132 * @brief finalizeCodeGen
    134133 ** ------------------------------------------------------------------------------------------------------------- */
    135 void CarryManager::finalizeCodeGen(IDISA::IDISA_Builder * const builder) {
     134void CarryManager::finalizeCodeGen(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder) {
    136135    if (mHasLoop) {
    137         mKernel->setScalarField("selector", mNextLoopSelector);
     136        iBuilder->setScalarField("selector", mNextLoopSelector);
    138137    }
    139138    if (mHasLongAdvance) {
    140         Value * idx = mKernel->getScalarField("CarryBlockIndex");
    141         idx = builder->CreateAdd(idx, builder->getSize(1));
    142         mKernel->setScalarField("CarryBlockIndex", idx);
     139        Value * idx = iBuilder->getScalarField("CarryBlockIndex");
     140        idx = iBuilder->CreateAdd(idx, iBuilder->getSize(1));
     141        iBuilder->setScalarField("CarryBlockIndex", idx);
    143142    }
    144143    assert (mCarryFrameStack.empty());   
     
    153152 * @brief enterLoopScope
    154153 ** ------------------------------------------------------------------------------------------------------------- */
    155 void CarryManager::enterLoopScope(IDISA::IDISA_Builder * const builder, const PabloBlock * const scope) {
     154void CarryManager::enterLoopScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope) {
    156155    assert (scope);
    157156    assert (mHasLoop);
    158157    ++mLoopDepth;
    159     enterScope(builder, scope);
     158    enterScope(iBuilder, scope);
    160159}
    161160
     
    163162 * @brief enterLoopBody
    164163 ** ------------------------------------------------------------------------------------------------------------- */
    165 void CarryManager::enterLoopBody(IDISA::IDISA_Builder * const builder, BasicBlock * const entryBlock) {
     164void CarryManager::enterLoopBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * const entryBlock) {
    166165    if (mCarryInfo->hasSummary()) {
    167         Type * const carryTy = builder->getBitBlockType();
    168         PHINode * phiCarryOutSummary = builder->CreatePHI(carryTy, 2, "summary");
     166        Type * const carryTy = iBuilder->getBitBlockType();
     167        PHINode * phiCarryOutSummary = iBuilder->CreatePHI(carryTy, 2, "summary");
    169168        assert (!mCarrySummaryStack.empty());
    170169        phiCarryOutSummary->addIncoming(mCarrySummaryStack.back(), entryBlock);
     
    180179        assert (mCarryInfo->hasSummary());
    181180
    182         Type * const carryTy = builder->getBitBlockType();
     181        Type * const carryTy = iBuilder->getBitBlockType();
    183182        PointerType * const carryPtrTy = carryTy->getPointerTo();
    184183
    185184        // Check whether we need to resize the carry state
    186         PHINode * index = builder->CreatePHI(builder->getSizeTy(), 2);
     185        PHINode * index = iBuilder->CreatePHI(iBuilder->getSizeTy(), 2);
    187186        mLoopIndicies.push_back(index);
    188         index->addIncoming(builder->getSize(0), entryBlock);
    189 
    190         Value * capacityPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(0)});
    191         Value * capacity = builder->CreateLoad(capacityPtr, false, "capacity");
     187        index->addIncoming(iBuilder->getSize(0), entryBlock);
     188
     189        Value * capacityPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
     190        Value * capacity = iBuilder->CreateLoad(capacityPtr, false, "capacity");
    192191        Constant * const ONE = ConstantInt::get(capacity->getType(), 1);
    193         Value * arrayPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(1)});
    194         Value * array = builder->CreateLoad(arrayPtr, false, "array");
    195 
    196         BasicBlock * const entry = builder->GetInsertBlock();
    197         BasicBlock * const resizeCarryState = mKernel->CreateBasicBlock("ResizeCarryState");
    198         BasicBlock * const reallocExisting = mKernel->CreateBasicBlock("ReallocExisting");
    199         BasicBlock * const createNew = mKernel->CreateBasicBlock("CreateNew");
    200         BasicBlock * const resumeKernel = mKernel->CreateBasicBlock("ResumeKernel");
    201 
    202         builder->CreateLikelyCondBr(builder->CreateICmpULT(index, capacity), resumeKernel, resizeCarryState);
     192        Value * arrayPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
     193        Value * array = iBuilder->CreateLoad(arrayPtr, false, "array");
     194
     195        BasicBlock * const entry = iBuilder->GetInsertBlock();
     196        BasicBlock * const resizeCarryState = iBuilder->CreateBasicBlock("ResizeCarryState");
     197        BasicBlock * const reallocExisting = iBuilder->CreateBasicBlock("ReallocExisting");
     198        BasicBlock * const createNew = iBuilder->CreateBasicBlock("CreateNew");
     199        BasicBlock * const resumeKernel = iBuilder->CreateBasicBlock("ResumeKernel");
     200
     201        iBuilder->CreateLikelyCondBr(iBuilder->CreateICmpULT(index, capacity), resumeKernel, resizeCarryState);
    203202
    204203        // RESIZE CARRY BLOCK
    205         builder->SetInsertPoint(resizeCarryState);
     204        iBuilder->SetInsertPoint(resizeCarryState);
    206205        const auto BlockWidth = carryTy->getPrimitiveSizeInBits() / 8;
    207206        const auto Log2BlockWidth = floor_log2(BlockWidth);
    208         Constant * const carryStateWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(array->getType()->getPointerElementType()), builder->getSizeTy(), false);
    209         Value * summaryPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(2)});
    210 
    211         Value * const hasCarryState = builder->CreateICmpNE(array, ConstantPointerNull::get(cast<PointerType>(array->getType())));
    212 
    213         builder->CreateLikelyCondBr(hasCarryState, reallocExisting, createNew);
     207        Constant * const carryStateWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(array->getType()->getPointerElementType()), iBuilder->getSizeTy(), false);
     208        Value * summaryPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(2)});
     209
     210        Value * const hasCarryState = iBuilder->CreateICmpNE(array, ConstantPointerNull::get(cast<PointerType>(array->getType())));
     211
     212        iBuilder->CreateLikelyCondBr(hasCarryState, reallocExisting, createNew);
    214213
    215214        // REALLOCATE EXISTING
    216         builder->SetInsertPoint(reallocExisting);
    217 
    218         Value * const capacitySize = builder->CreateMul(capacity, carryStateWidth);
    219         Value * const newCapacitySize = builder->CreateShl(capacitySize, 1); // x 2
    220 
    221 
    222         Value * newArray = builder->CreateAlignedMalloc(newCapacitySize, builder->getCacheAlignment());
    223         builder->CreateMemCpy(newArray, array, capacitySize, BlockWidth);
    224         builder->CreateMemZero(builder->CreateGEP(newArray, capacitySize), capacitySize, BlockWidth);
    225         builder->CreateAlignedFree(array);
    226         newArray = builder->CreatePointerCast(newArray, array->getType());
    227         builder->CreateStore(newArray, arrayPtr);
    228 
    229         Value * const log2capacity = builder->CreateAdd(builder->CreateCeilLog2(capacity), ONE);
    230         Value * const summarySize = builder->CreateShl(log2capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
    231         Value * const newLog2Capacity = builder->CreateAdd(log2capacity, ONE);
    232         Value * const newSummarySize = builder->CreateShl(newLog2Capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
    233 
    234         Value * const summary = builder->CreateLoad(summaryPtr, false);
    235         Value * newSummary = builder->CreateAlignedMalloc(newSummarySize, BlockWidth);
    236         builder->CreateMemCpy(newSummary, summary, summarySize, BlockWidth);
    237         builder->CreateMemZero(builder->CreateGEP(newSummary, summarySize), builder->getSize(2 * BlockWidth), BlockWidth);
    238         builder->CreateAlignedFree(summary);
    239 
    240         Value * ptr1 = builder->CreateGEP(newSummary, summarySize);
    241         ptr1 = builder->CreatePointerCast(ptr1, carryPtrTy);
    242 
    243         Value * ptr2 = builder->CreateGEP(newSummary, builder->CreateAdd(summarySize, builder->getSize(BlockWidth)));
    244         ptr2 = builder->CreatePointerCast(ptr2, carryPtrTy);
    245 
    246         newSummary = builder->CreatePointerCast(newSummary, carryPtrTy);
    247         builder->CreateStore(newSummary, summaryPtr);
    248         Value * const newCapacity = builder->CreateShl(ONE, log2capacity);
    249 
    250         builder->CreateStore(newCapacity, capacityPtr);
    251 
    252         builder->CreateBr(resumeKernel);
     215        iBuilder->SetInsertPoint(reallocExisting);
     216
     217        Value * const capacitySize = iBuilder->CreateMul(capacity, carryStateWidth);
     218        Value * const newCapacitySize = iBuilder->CreateShl(capacitySize, 1); // x 2
     219
     220
     221        Value * newArray = iBuilder->CreateAlignedMalloc(newCapacitySize, iBuilder->getCacheAlignment());
     222        iBuilder->CreateMemCpy(newArray, array, capacitySize, BlockWidth);
     223        iBuilder->CreateMemZero(iBuilder->CreateGEP(newArray, capacitySize), capacitySize, BlockWidth);
     224        iBuilder->CreateAlignedFree(array);
     225        newArray = iBuilder->CreatePointerCast(newArray, array->getType());
     226        iBuilder->CreateStore(newArray, arrayPtr);
     227
     228        Value * const log2capacity = iBuilder->CreateAdd(iBuilder->CreateCeilLog2(capacity), ONE);
     229        Value * const summarySize = iBuilder->CreateShl(log2capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
     230        Value * const newLog2Capacity = iBuilder->CreateAdd(log2capacity, ONE);
     231        Value * const newSummarySize = iBuilder->CreateShl(newLog2Capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
     232
     233        Value * const summary = iBuilder->CreateLoad(summaryPtr, false);
     234        Value * newSummary = iBuilder->CreateAlignedMalloc(newSummarySize, BlockWidth);
     235        iBuilder->CreateMemCpy(newSummary, summary, summarySize, BlockWidth);
     236        iBuilder->CreateMemZero(iBuilder->CreateGEP(newSummary, summarySize), iBuilder->getSize(2 * BlockWidth), BlockWidth);
     237        iBuilder->CreateAlignedFree(summary);
     238
     239        Value * ptr1 = iBuilder->CreateGEP(newSummary, summarySize);
     240        ptr1 = iBuilder->CreatePointerCast(ptr1, carryPtrTy);
     241
     242        Value * ptr2 = iBuilder->CreateGEP(newSummary, iBuilder->CreateAdd(summarySize, iBuilder->getSize(BlockWidth)));
     243        ptr2 = iBuilder->CreatePointerCast(ptr2, carryPtrTy);
     244
     245        newSummary = iBuilder->CreatePointerCast(newSummary, carryPtrTy);
     246        iBuilder->CreateStore(newSummary, summaryPtr);
     247        Value * const newCapacity = iBuilder->CreateShl(ONE, log2capacity);
     248
     249        iBuilder->CreateStore(newCapacity, capacityPtr);
     250
     251        iBuilder->CreateBr(resumeKernel);
    253252
    254253        // CREATE NEW
    255         builder->SetInsertPoint(createNew);
    256 
    257         Constant * const initialLog2Capacity = builder->getInt64(4);
     254        iBuilder->SetInsertPoint(createNew);
     255
     256        Constant * const initialLog2Capacity = iBuilder->getInt64(4);
    258257        Constant * const initialCapacity = ConstantExpr::getShl(ONE, initialLog2Capacity);
    259258        Constant * const initialCapacitySize = ConstantExpr::getMul(initialCapacity, carryStateWidth);
    260259
    261         Value * initialArray = builder->CreateAlignedMalloc(initialCapacitySize, builder->getCacheAlignment());
    262         builder->CreateMemZero(initialArray, initialCapacitySize, BlockWidth);
    263         initialArray = builder->CreatePointerCast(initialArray, array->getType());
    264         builder->CreateStore(initialArray, arrayPtr);
    265 
    266         Constant * initialSummarySize = ConstantExpr::getShl(ConstantExpr::getAdd(initialLog2Capacity, builder->getInt64(1)), builder->getInt64(Log2BlockWidth + 1));
    267         Value * initialSummary = builder->CreateAlignedMalloc(initialSummarySize, BlockWidth);
    268         builder->CreateMemZero(initialSummary, initialSummarySize, BlockWidth);
    269         initialSummary = builder->CreatePointerCast(initialSummary, carryPtrTy);
    270         builder->CreateStore(initialSummary, summaryPtr);
    271 
    272         builder->CreateStore(initialCapacity, capacityPtr);
    273 
    274         builder->CreateBr(resumeKernel);
     260        Value * initialArray = iBuilder->CreateAlignedMalloc(initialCapacitySize, iBuilder->getCacheAlignment());
     261        iBuilder->CreateMemZero(initialArray, initialCapacitySize, BlockWidth);
     262        initialArray = iBuilder->CreatePointerCast(initialArray, array->getType());
     263        iBuilder->CreateStore(initialArray, arrayPtr);
     264
     265        Constant * initialSummarySize = ConstantExpr::getShl(ConstantExpr::getAdd(initialLog2Capacity, iBuilder->getInt64(1)), iBuilder->getInt64(Log2BlockWidth + 1));
     266        Value * initialSummary = iBuilder->CreateAlignedMalloc(initialSummarySize, BlockWidth);
     267        iBuilder->CreateMemZero(initialSummary, initialSummarySize, BlockWidth);
     268        initialSummary = iBuilder->CreatePointerCast(initialSummary, carryPtrTy);
     269        iBuilder->CreateStore(initialSummary, summaryPtr);
     270
     271        iBuilder->CreateStore(initialCapacity, capacityPtr);
     272
     273        iBuilder->CreateBr(resumeKernel);
    275274
    276275        // RESUME KERNEL
    277         builder->SetInsertPoint(resumeKernel);
     276        iBuilder->SetInsertPoint(resumeKernel);
    278277        // Load the appropriate carry stat block
    279         PHINode * phiArrayPtr = builder->CreatePHI(array->getType(), 3);
     278        PHINode * phiArrayPtr = iBuilder->CreatePHI(array->getType(), 3);
    280279        phiArrayPtr->addIncoming(array, entry);
    281280        phiArrayPtr->addIncoming(initialArray, createNew);
     
    284283        // NOTE: the 3 here is only to pass the assertion later. It refers to the number of elements in the carry data struct.
    285284        mCarryFrameStack.emplace_back(mCurrentFrame, 3);
    286         mCurrentFrame = builder->CreateGEP(phiArrayPtr, index);
     285        mCurrentFrame = iBuilder->CreateGEP(phiArrayPtr, index);
    287286    }
    288287}
     
    291290 * @brief leaveLoopBody
    292291 ** ------------------------------------------------------------------------------------------------------------- */
    293 void CarryManager::leaveLoopBody(IDISA::IDISA_Builder * const builder, BasicBlock * /* exitBlock */) {
    294 
    295     Type * const carryTy = builder->getBitBlockType();
     292void CarryManager::leaveLoopBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * /* exitBlock */) {
     293
     294    Type * const carryTy = iBuilder->getBitBlockType();
    296295
    297296    if (LLVM_UNLIKELY(mCarryInfo->nonCarryCollapsingMode())) {
     
    299298        assert (mCarryInfo->hasSummary());
    300299
    301         ConstantInt * const summaryIndex = builder->getInt32(mCarryInfo->hasExplicitSummary() ? mCurrentFrameIndex : (mCurrentFrameIndex - 1));
    302 
    303         Value * const carryInAccumulator = readCarryInSummary(builder, summaryIndex);
     300        ConstantInt * const summaryIndex = iBuilder->getInt32(mCarryInfo->hasExplicitSummary() ? mCurrentFrameIndex : (mCurrentFrameIndex - 1));
     301
     302        Value * const carryInAccumulator = readCarryInSummary(iBuilder, summaryIndex);
    304303        Value * const carryOutAccumulator = mCarrySummaryStack.back();
    305304
    306305        if (mCarryInfo->hasExplicitSummary()) {
    307             writeCarryOutSummary(builder, carryOutAccumulator, summaryIndex);
     306            writeCarryOutSummary(iBuilder, carryOutAccumulator, summaryIndex);
    308307        }
    309308
     
    321320        // Otherwise we may end up with an incorrect result or being trapped in an infinite loop.
    322321
    323         Value * capacityPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(0)});
    324         Value * capacity = builder->CreateLoad(capacityPtr, false);
    325         Value * summaryPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(2)});
    326         Value * summary = builder->CreateLoad(summaryPtr, false);
     322        Value * capacityPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
     323        Value * capacity = iBuilder->CreateLoad(capacityPtr, false);
     324        Value * summaryPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(2)});
     325        Value * summary = iBuilder->CreateLoad(summaryPtr, false);
    327326
    328327        Constant * const ONE = ConstantInt::get(capacity->getType(), 1);
    329328
    330         Value * loopSelector = builder->CreateZExt(mLoopSelector, capacity->getType());
    331 
    332         BasicBlock * entry = builder->GetInsertBlock();
    333         BasicBlock * update = mKernel->CreateBasicBlock("UpdateNonCarryCollapsingSummary");
    334         BasicBlock * resume = mKernel->CreateBasicBlock("ResumeAfterUpdatingNonCarryCollapsingSummary");
    335 
    336         builder->CreateBr(update);
    337 
    338         builder->SetInsertPoint(update);
    339         PHINode * i = builder->CreatePHI(capacity->getType(), 2);
     329        Value * loopSelector = iBuilder->CreateZExt(mLoopSelector, capacity->getType());
     330
     331        BasicBlock * entry = iBuilder->GetInsertBlock();
     332        BasicBlock * update = iBuilder->CreateBasicBlock("UpdateNonCarryCollapsingSummary");
     333        BasicBlock * resume = iBuilder->CreateBasicBlock("ResumeAfterUpdatingNonCarryCollapsingSummary");
     334
     335        iBuilder->CreateBr(update);
     336
     337        iBuilder->SetInsertPoint(update);
     338        PHINode * i = iBuilder->CreatePHI(capacity->getType(), 2);
    340339        i->addIncoming(ConstantInt::getNullValue(capacity->getType()), entry);
    341         PHINode * const borrow = builder->CreatePHI(carryInAccumulator->getType(), 2);
     340        PHINode * const borrow = iBuilder->CreatePHI(carryInAccumulator->getType(), 2);
    342341        borrow->addIncoming(carryInAccumulator, entry);
    343         PHINode * const carry = builder->CreatePHI(carryOutAccumulator->getType(), 2);
     342        PHINode * const carry = iBuilder->CreatePHI(carryOutAccumulator->getType(), 2);
    344343        carry->addIncoming(carryOutAccumulator, entry);
    345344        // OR the updated carry in summary later for the summaryTest
    346         PHINode * const carryInSummary = builder->CreatePHI(carryTy, 2);
     345        PHINode * const carryInSummary = iBuilder->CreatePHI(carryTy, 2);
    347346        carryInSummary->addIncoming(Constant::getNullValue(carryTy), entry);
    348347
    349348        // half subtractor
    350         Value * const carryInOffset = builder->CreateOr(builder->CreateShl(i, 1), loopSelector);
    351         Value * const carryInPtr = builder->CreateGEP(summary, carryInOffset);
    352         Value * const carryIn = builder->CreateBlockAlignedLoad(carryInPtr);
    353         Value * const nextCarryIn = builder->CreateXor(carryIn, borrow);
    354         Value * const nextSummary = builder->CreateOr(carryInSummary, nextCarryIn);
    355         builder->CreateBlockAlignedStore(nextCarryIn, carryInPtr);
     349        Value * const carryInOffset = iBuilder->CreateOr(iBuilder->CreateShl(i, 1), loopSelector);
     350        Value * const carryInPtr = iBuilder->CreateGEP(summary, carryInOffset);
     351        Value * const carryIn = iBuilder->CreateBlockAlignedLoad(carryInPtr);
     352        Value * const nextCarryIn = iBuilder->CreateXor(carryIn, borrow);
     353        Value * const nextSummary = iBuilder->CreateOr(carryInSummary, nextCarryIn);
     354        iBuilder->CreateBlockAlignedStore(nextCarryIn, carryInPtr);
    356355        carryInSummary->addIncoming(nextSummary, update);
    357         Value * finalBorrow = builder->CreateAnd(builder->CreateNot(carryIn), borrow);
     356        Value * finalBorrow = iBuilder->CreateAnd(iBuilder->CreateNot(carryIn), borrow);
    358357        borrow->addIncoming(finalBorrow, update);
    359358
    360359        // half adder
    361         Value * const carryOutOffset = builder->CreateXor(carryInOffset, ConstantInt::get(carryInOffset->getType(), 1));
    362         Value * const carryOutPtr = builder->CreateGEP(summary, carryOutOffset);
    363         Value * const carryOut = builder->CreateBlockAlignedLoad(carryOutPtr);
    364         Value * const nextCarryOut = builder->CreateXor(carryOut, carry);
    365         builder->CreateBlockAlignedStore(nextCarryOut, carryOutPtr);
    366         Value * finalCarry = builder->CreateAnd(carryOut, carry);
     360        Value * const carryOutOffset = iBuilder->CreateXor(carryInOffset, ConstantInt::get(carryInOffset->getType(), 1));
     361        Value * const carryOutPtr = iBuilder->CreateGEP(summary, carryOutOffset);
     362        Value * const carryOut = iBuilder->CreateBlockAlignedLoad(carryOutPtr);
     363        Value * const nextCarryOut = iBuilder->CreateXor(carryOut, carry);
     364        iBuilder->CreateBlockAlignedStore(nextCarryOut, carryOutPtr);
     365        Value * finalCarry = iBuilder->CreateAnd(carryOut, carry);
    367366        carry->addIncoming(finalCarry, update);
    368367
    369368        // loop condition
    370         i->addIncoming(builder->CreateAdd(i, ONE), update);
    371         builder->CreateCondBr(builder->CreateICmpNE(builder->CreateShl(ONE, i), capacity), update, resume);
    372 
    373         builder->SetInsertPoint(resume);
    374 
    375         IntegerType * ty = IntegerType::get(builder->getContext(), carryTy->getPrimitiveSizeInBits());
    376         builder->CreateAssert(builder->CreateICmpEQ(builder->CreateBitCast(finalBorrow, ty), ConstantInt::getNullValue(ty)), "finalBorrow != 0");
    377         builder->CreateAssert(builder->CreateICmpEQ(builder->CreateBitCast(finalCarry, ty), ConstantInt::getNullValue(ty)), "finalCarry != 0");
     369        i->addIncoming(iBuilder->CreateAdd(i, ONE), update);
     370        iBuilder->CreateCondBr(iBuilder->CreateICmpNE(iBuilder->CreateShl(ONE, i), capacity), update, resume);
     371
     372        iBuilder->SetInsertPoint(resume);
     373
     374        IntegerType * ty = IntegerType::get(iBuilder->getContext(), carryTy->getPrimitiveSizeInBits());
     375        iBuilder->CreateAssert(iBuilder->CreateICmpEQ(iBuilder->CreateBitCast(finalBorrow, ty), ConstantInt::getNullValue(ty)), "finalBorrow != 0");
     376        iBuilder->CreateAssert(iBuilder->CreateICmpEQ(iBuilder->CreateBitCast(finalCarry, ty), ConstantInt::getNullValue(ty)), "finalCarry != 0");
    378377
    379378        assert (!mLoopIndicies.empty());
    380379        PHINode * index = mLoopIndicies.back();
    381         index->addIncoming(builder->CreateAdd(index, builder->getSize(1)), resume);
     380        index->addIncoming(iBuilder->CreateAdd(index, iBuilder->getSize(1)), resume);
    382381        mLoopIndicies.pop_back();
    383382
     
    389388        mCarrySummaryStack.pop_back();
    390389        PHINode * phiCarryOut = cast<PHINode>(mCarrySummaryStack.back());
    391         phiCarryOut->addIncoming(carryOut, builder->GetInsertBlock());
     390        phiCarryOut->addIncoming(carryOut, iBuilder->GetInsertBlock());
    392391        // If we're returning to the base scope, reset our accumulated summary value.
    393392        if (n == 2) {
     
    401400 * @brief leaveLoopScope
    402401 ** ------------------------------------------------------------------------------------------------------------- */
    403 void CarryManager::leaveLoopScope(IDISA::IDISA_Builder * const builder, BasicBlock * const /* entryBlock */, BasicBlock * const /* exitBlock */) {
     402void CarryManager::leaveLoopScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * const /* entryBlock */, BasicBlock * const /* exitBlock */) {
    404403    assert (mLoopDepth > 0);
    405404    --mLoopDepth;
    406     leaveScope(builder);
     405    leaveScope(iBuilder);
    407406}
    408407
     
    410409 * @brief enterIfScope
    411410 ** ------------------------------------------------------------------------------------------------------------- */
    412 void CarryManager::enterIfScope(IDISA::IDISA_Builder * const builder, const PabloBlock * const scope) {
     411void CarryManager::enterIfScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope) {
    413412    ++mIfDepth;
    414     enterScope(builder, scope);
     413    enterScope(iBuilder, scope);
    415414    // We zero-initialized the nested summary value and later OR in the current summary into the escaping summary
    416415    // so that upon processing the subsequent block iteration, we branch into this If scope iff a carry out was
     
    418417    if (LLVM_LIKELY(mCarryInfo->hasSummary())) {
    419418        assert (mCurrentFrameIndex == 0);
    420         mNextSummaryTest = readCarryInSummary(builder, builder->getInt32(0));
     419        mNextSummaryTest = readCarryInSummary(iBuilder, iBuilder->getInt32(0));
    421420        if (mCarryInfo->hasExplicitSummary()) {
    422421            mCurrentFrameIndex = 1;
    423422        }
    424423    }
    425     Type * const carryTy = builder->getBitBlockType();
     424    Type * const carryTy = iBuilder->getBitBlockType();
    426425    mCarrySummaryStack.push_back(Constant::getNullValue(carryTy));
    427426}
     
    430429 * @brief generateSummaryTest
    431430 ** ------------------------------------------------------------------------------------------------------------- */
    432 Value * CarryManager::generateSummaryTest(IDISA::IDISA_Builder * const builder, Value * condition) {
     431Value * CarryManager::generateSummaryTest(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * condition) {
    433432    if (LLVM_LIKELY(mCarryInfo->hasSummary())) {
    434433        assert ("summary test was not generated" && mNextSummaryTest);
    435         condition = builder->simd_or(condition, mNextSummaryTest);
     434        condition = iBuilder->simd_or(condition, mNextSummaryTest);
    436435        mNextSummaryTest = nullptr;
    437436    }
     
    443442 * @brief enterIfBody
    444443 ** ------------------------------------------------------------------------------------------------------------- */
    445 void CarryManager::enterIfBody(IDISA::IDISA_Builder * const /* builder */, BasicBlock * const entryBlock) {
     444void CarryManager::enterIfBody(const std::unique_ptr<kernel::KernelBuilder> &  /* iBuilder */, BasicBlock * const entryBlock) {
    446445    assert (entryBlock);
    447446}
     
    450449 * @brief leaveIfBody
    451450 ** ------------------------------------------------------------------------------------------------------------- */
    452 void CarryManager::leaveIfBody(IDISA::IDISA_Builder * const builder, BasicBlock * const exitBlock) {
     451void CarryManager::leaveIfBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * const exitBlock) {
    453452    assert (exitBlock);
    454453    const auto n = mCarrySummaryStack.size();
    455454    if (LLVM_LIKELY(mCarryInfo->hasExplicitSummary())) {
    456         writeCarryOutSummary(builder, mCarrySummaryStack[n - 1], builder->getInt32(0));
     455        writeCarryOutSummary(iBuilder, mCarrySummaryStack[n - 1], iBuilder->getInt32(0));
    457456    }
    458457    if (n > 2) {
    459         mCarrySummaryStack[n - 1] = builder->CreateOr(mCarrySummaryStack[n - 1], mCarrySummaryStack[n - 2], "summary");
     458        mCarrySummaryStack[n - 1] = iBuilder->CreateOr(mCarrySummaryStack[n - 1], mCarrySummaryStack[n - 2], "summary");
    460459    }
    461460}
     
    464463 * @brief leaveIfScope
    465464 ** ------------------------------------------------------------------------------------------------------------- */
    466 void CarryManager::leaveIfScope(IDISA::IDISA_Builder * const builder, BasicBlock * const entryBlock, BasicBlock * const exitBlock) {
     465void CarryManager::leaveIfScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * const entryBlock, BasicBlock * const exitBlock) {
    467466    assert (mIfDepth > 0);
    468467    if (LLVM_LIKELY(mCarryInfo->hasSummary())) {
     
    474473            Value * outer = mCarrySummaryStack[n - 2];
    475474            assert (nested->getType() == outer->getType());
    476             PHINode * const phi = builder->CreatePHI(nested->getType(), 2, "summary");
     475            PHINode * const phi = iBuilder->CreatePHI(nested->getType(), 2, "summary");
    477476            phi->addIncoming(outer, entryBlock);
    478477            phi->addIncoming(nested, exitBlock);
     
    481480    }
    482481    --mIfDepth;
    483     leaveScope(builder);
     482    leaveScope(iBuilder);
    484483    mCarrySummaryStack.pop_back();
    485484}
     
    488487 * @brief enterScope
    489488 ** ------------------------------------------------------------------------------------------------------------- */
    490 void CarryManager::enterScope(IDISA::IDISA_Builder * const builder, const PabloBlock * const scope) {
     489void CarryManager::enterScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope) {
    491490    assert (scope);
    492491    // Store the state of the current frame and update the scope state
     
    498497    // compilation or a memory corruption has occured.
    499498    assert (mCurrentFrameIndex < mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
    500     mCurrentFrame = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(mCurrentFrameIndex)});
     499    mCurrentFrame = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex)});
    501500    // Verify we're pointing to a carry frame struct
    502501    assert(mCurrentFrame->getType()->getPointerElementType()->isStructTy());
     
    507506 * @brief leaveScope
    508507 ** ------------------------------------------------------------------------------------------------------------- */
    509 void CarryManager::leaveScope(IDISA::IDISA_Builder * const /* builder */) {
     508void CarryManager::leaveScope(const std::unique_ptr<kernel::KernelBuilder> &  /* iBuilder */) {
    510509
    511510    // Did we use all of the packs in this carry struct?
     
    529528 * @brief addCarryInCarryOut
    530529 ** ------------------------------------------------------------------------------------------------------------- */
    531 Value * CarryManager::addCarryInCarryOut(IDISA::IDISA_Builder * const builder, const Statement * const operation, Value * const e1, Value * const e2) {
     530Value * CarryManager::addCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const Statement * const operation, Value * const e1, Value * const e2) {
    532531    assert (operation && (isNonAdvanceCarryGeneratingStatement(operation)));
    533     Value * const carryIn = getNextCarryIn(builder);
     532    Value * const carryIn = getNextCarryIn(iBuilder);
    534533    Value * carryOut, * result;
    535     std::tie(carryOut, result) = builder->bitblock_add_with_carry(e1, e2, carryIn);
    536     setNextCarryOut(builder, carryOut);
    537     assert (result->getType() == builder->getBitBlockType());
     534    std::tie(carryOut, result) = iBuilder->bitblock_add_with_carry(e1, e2, carryIn);
     535    setNextCarryOut(iBuilder, carryOut);
     536    assert (result->getType() == iBuilder->getBitBlockType());
    538537    return result;
    539538}
     
    542541 * @brief advanceCarryInCarryOut
    543542 ** ------------------------------------------------------------------------------------------------------------- */
    544 Value * CarryManager::advanceCarryInCarryOut(IDISA::IDISA_Builder * const builder, const Advance * const advance, Value * const value) {
     543Value * CarryManager::advanceCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const Advance * const advance, Value * const value) {
    545544    const auto shiftAmount = advance->getAmount();
    546545    if (LLVM_LIKELY(shiftAmount < LONG_ADVANCE_BREAKPOINT)) {
    547         Value * const carryIn = getNextCarryIn(builder);
     546        Value * const carryIn = getNextCarryIn(iBuilder);
    548547        Value * carryOut, * result;
    549         std::tie(carryOut, result) = builder->bitblock_advance(value, carryIn, shiftAmount);
    550         setNextCarryOut(builder, carryOut);
    551         assert (result->getType() == builder->getBitBlockType());
     548        std::tie(carryOut, result) = iBuilder->bitblock_advance(value, carryIn, shiftAmount);
     549        setNextCarryOut(iBuilder, carryOut);
     550        assert (result->getType() == iBuilder->getBitBlockType());
    552551        return result;
    553552    } else {
    554         return longAdvanceCarryInCarryOut(builder, value, shiftAmount);
     553        return longAdvanceCarryInCarryOut(iBuilder, value, shiftAmount);
    555554    }
    556555}
     
    559558 * @brief longAdvanceCarryInCarryOut
    560559 ** ------------------------------------------------------------------------------------------------------------- */
    561 inline Value * CarryManager::longAdvanceCarryInCarryOut(IDISA::IDISA_Builder * const builder, Value * const value, const unsigned shiftAmount) {
     560inline Value * CarryManager::longAdvanceCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * const value, const unsigned shiftAmount) {
    562561
    563562    assert (mHasLongAdvance);
    564563    assert (shiftAmount >= LONG_ADVANCE_BREAKPOINT);
    565564
    566     Type * const streamTy = builder->getIntNTy(builder->getBitBlockWidth());
     565    Type * const streamTy = iBuilder->getIntNTy(iBuilder->getBitBlockWidth());
    567566
    568567    if (mIfDepth > 0) {
    569         if (shiftAmount > builder->getBitBlockWidth()) {
     568        if (shiftAmount > iBuilder->getBitBlockWidth()) {
    570569            const auto frameIndex = mCurrentFrameIndex++;
    571             Value * carry = builder->CreateZExt(builder->bitblock_any(value), streamTy);
    572             const unsigned summarySize = ceil_udiv(shiftAmount, builder->getBitBlockWidth() * builder->getBitBlockWidth());
     570            Value * carry = iBuilder->CreateZExt(iBuilder->bitblock_any(value), streamTy);
     571            const unsigned summarySize = ceil_udiv(shiftAmount, iBuilder->getBitBlockWidth() * iBuilder->getBitBlockWidth());
    573572            for (unsigned i = 0;;++i) {
    574                 Value * const ptr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(frameIndex), builder->getInt32(i)});
    575                 Value * const prior = builder->CreateBitCast(builder->CreateBlockAlignedLoad(ptr), streamTy);
    576                 Value * const stream = builder->CreateBitCast(builder->CreateOr(builder->CreateShl(prior, 1), carry), builder->getBitBlockType());
     573                Value * const ptr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), iBuilder->getInt32(i)});
     574                Value * const prior = iBuilder->CreateBitCast(iBuilder->CreateBlockAlignedLoad(ptr), streamTy);
     575                Value * const stream = iBuilder->CreateBitCast(iBuilder->CreateOr(iBuilder->CreateShl(prior, 1), carry), iBuilder->getBitBlockType());
    577576                if (LLVM_LIKELY(i == summarySize)) {
    578                     Value * const maskedStream = builder->CreateAnd(stream, builder->bitblock_mask_from(builder->getInt32(summarySize % builder->getBitBlockWidth())));
    579                     addToCarryOutSummary(builder, maskedStream);
    580                     builder->CreateBlockAlignedStore(maskedStream, ptr);
     577                    Value * const maskedStream = iBuilder->CreateAnd(stream, iBuilder->bitblock_mask_from(iBuilder->getInt32(summarySize % iBuilder->getBitBlockWidth())));
     578                    addToCarryOutSummary(iBuilder, maskedStream);
     579                    iBuilder->CreateBlockAlignedStore(maskedStream, ptr);
    581580                    break;
    582581                }
    583                 addToCarryOutSummary(builder, stream);
    584                 builder->CreateBlockAlignedStore(stream, ptr);
    585                 carry = builder->CreateLShr(prior, builder->getBitBlockWidth() - 1);
     582                addToCarryOutSummary(iBuilder, stream);
     583                iBuilder->CreateBlockAlignedStore(stream, ptr);
     584                carry = iBuilder->CreateLShr(prior, iBuilder->getBitBlockWidth() - 1);
    586585            }
    587586        } else if (LLVM_LIKELY(mCarryInfo->hasExplicitSummary())) {
    588             addToCarryOutSummary(builder, value);
     587            addToCarryOutSummary(iBuilder, value);
    589588        }
    590589    }
    591590    const auto frameIndex = mCurrentFrameIndex++;
    592591    // special case using a single buffer entry and the carry_out value.
    593     if (LLVM_LIKELY((shiftAmount < builder->getBitBlockWidth()) && (mLoopDepth == 0))) {
    594         Value * const buffer = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(frameIndex), builder->getInt32(0)});
    595         assert (buffer->getType()->getPointerElementType() == builder->getBitBlockType());
    596         Value * carryIn = builder->CreateBlockAlignedLoad(buffer);
    597         builder->CreateBlockAlignedStore(value, buffer);
     592    if (LLVM_LIKELY((shiftAmount < iBuilder->getBitBlockWidth()) && (mLoopDepth == 0))) {
     593        Value * const buffer = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), iBuilder->getInt32(0)});
     594        assert (buffer->getType()->getPointerElementType() == iBuilder->getBitBlockType());
     595        Value * carryIn = iBuilder->CreateBlockAlignedLoad(buffer);
     596        iBuilder->CreateBlockAlignedStore(value, buffer);
    598597        /* Very special case - no combine */
    599         if (LLVM_UNLIKELY(shiftAmount == builder->getBitBlockWidth())) {
    600             return builder->CreateBitCast(carryIn, builder->getBitBlockType());
    601         }
    602         Value* block0_shr = builder->CreateLShr(builder->CreateBitCast(carryIn, streamTy), builder->getBitBlockWidth() - shiftAmount);
    603         Value* block1_shl = builder->CreateShl(builder->CreateBitCast(value, streamTy), shiftAmount);
    604         return builder->CreateBitCast(builder->CreateOr(block1_shl, block0_shr), builder->getBitBlockType());
     598        if (LLVM_UNLIKELY(shiftAmount == iBuilder->getBitBlockWidth())) {
     599            return iBuilder->CreateBitCast(carryIn, iBuilder->getBitBlockType());
     600        }
     601        Value* block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carryIn, streamTy), iBuilder->getBitBlockWidth() - shiftAmount);
     602        Value* block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(value, streamTy), shiftAmount);
     603        return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), iBuilder->getBitBlockType());
    605604    } else { //
    606         const unsigned blockShift = shiftAmount % builder->getBitBlockWidth();
    607         const unsigned blocks = ceil_udiv(shiftAmount, builder->getBitBlockWidth());
     605        const unsigned blockShift = shiftAmount % iBuilder->getBitBlockWidth();
     606        const unsigned blocks = ceil_udiv(shiftAmount, iBuilder->getBitBlockWidth());
    608607        // Create a mask to implement circular buffer indexing
    609         Value * indexMask = builder->getSize(nearest_pow2(blocks + ((mLoopDepth != 0) ? 1 : 0)) - 1);
    610         Value * blockIndex = mKernel->getScalarField("CarryBlockIndex");
    611         Value * carryIndex0 = builder->CreateSub(blockIndex, builder->getSize(blocks));
    612         Value * loadIndex0 = builder->CreateAnd(carryIndex0, indexMask);
    613         Value * const carryInPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(frameIndex), loadIndex0});
    614         Value * carryIn = builder->CreateBlockAlignedLoad(carryInPtr);
    615 
    616         Value * storeIndex = builder->CreateAnd(blockIndex, indexMask);
    617         Value * const carryOutPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(frameIndex), storeIndex});
    618         assert (carryIn->getType() == builder->getBitBlockType());
     608        Value * indexMask = iBuilder->getSize(nearest_pow2(blocks + ((mLoopDepth != 0) ? 1 : 0)) - 1);
     609        Value * blockIndex = iBuilder->getScalarField("CarryBlockIndex");
     610        Value * carryIndex0 = iBuilder->CreateSub(blockIndex, iBuilder->getSize(blocks));
     611        Value * loadIndex0 = iBuilder->CreateAnd(carryIndex0, indexMask);
     612        Value * const carryInPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), loadIndex0});
     613        Value * carryIn = iBuilder->CreateBlockAlignedLoad(carryInPtr);
     614
     615        Value * storeIndex = iBuilder->CreateAnd(blockIndex, indexMask);
     616        Value * const carryOutPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), storeIndex});
     617        assert (carryIn->getType() == iBuilder->getBitBlockType());
    619618
    620619        // If the long advance is an exact multiple of BitBlockWidth, we simply return the oldest
    621620        // block in the long advance carry data area.
    622621        if (LLVM_UNLIKELY(blockShift == 0)) {
    623             builder->CreateBlockAlignedStore(value, carryOutPtr);
     622            iBuilder->CreateBlockAlignedStore(value, carryOutPtr);
    624623            return carryIn;
    625624        } else { // Otherwise we need to combine data from the two oldest blocks.
    626             Value * carryIndex1 = builder->CreateSub(blockIndex, builder->getSize(blocks - 1));
    627             Value * loadIndex1 = builder->CreateAnd(carryIndex1, indexMask);
    628             Value * const carryInPtr2 = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(frameIndex), loadIndex1});
    629             Value * carry_block1 = builder->CreateBlockAlignedLoad(carryInPtr2);
    630             Value * block0_shr = builder->CreateLShr(builder->CreateBitCast(carryIn, streamTy), builder->getBitBlockWidth() - blockShift);
    631             Value * block1_shl = builder->CreateShl(builder->CreateBitCast(carry_block1, streamTy), blockShift);
    632             builder->CreateBlockAlignedStore(value, carryOutPtr);
    633             return builder->CreateBitCast(builder->CreateOr(block1_shl, block0_shr), builder->getBitBlockType());
     625            Value * carryIndex1 = iBuilder->CreateSub(blockIndex, iBuilder->getSize(blocks - 1));
     626            Value * loadIndex1 = iBuilder->CreateAnd(carryIndex1, indexMask);
     627            Value * const carryInPtr2 = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), loadIndex1});
     628            Value * carry_block1 = iBuilder->CreateBlockAlignedLoad(carryInPtr2);
     629            Value * block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carryIn, streamTy), iBuilder->getBitBlockWidth() - blockShift);
     630            Value * block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(carry_block1, streamTy), blockShift);
     631            iBuilder->CreateBlockAlignedStore(value, carryOutPtr);
     632            return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), iBuilder->getBitBlockType());
    634633        }
    635634    }
     
    639638 * @brief getNextCarryIn
    640639 ** ------------------------------------------------------------------------------------------------------------- */
    641 Value * CarryManager::getNextCarryIn(IDISA::IDISA_Builder * const builder) {
     640Value * CarryManager::getNextCarryIn(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder) {
    642641    assert (mCurrentFrameIndex < mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
    643642    if (mLoopDepth == 0) {
    644         mCarryPackPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(mCurrentFrameIndex)});
     643        mCarryPackPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex)});
    645644    } else {
    646         mCarryPackPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(mCurrentFrameIndex), mLoopSelector});
    647     }
    648     Type * const carryTy = builder->getBitBlockType();
     645        mCarryPackPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex), mLoopSelector});
     646    }
     647    Type * const carryTy = iBuilder->getBitBlockType();
    649648    assert (mCarryPackPtr->getType()->getPointerElementType() == carryTy);
    650     Value * const carryIn = builder->CreateBlockAlignedLoad(mCarryPackPtr);
     649    Value * const carryIn = iBuilder->CreateBlockAlignedLoad(mCarryPackPtr);
    651650    if (mLoopDepth > 0) {
    652         builder->CreateBlockAlignedStore(Constant::getNullValue(carryTy), mCarryPackPtr);
     651        iBuilder->CreateBlockAlignedStore(Constant::getNullValue(carryTy), mCarryPackPtr);
    653652    }
    654653    return carryIn;
     
    658657 * @brief setNextCarryOut
    659658 ** ------------------------------------------------------------------------------------------------------------- */
    660 void CarryManager::setNextCarryOut(IDISA::IDISA_Builder * const builder, Value * carryOut) {
    661     Type * const carryTy = builder->getBitBlockType();
     659void CarryManager::setNextCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * carryOut) {
     660    Type * const carryTy = iBuilder->getBitBlockType();
    662661    assert (mCurrentFrameIndex < mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
    663     carryOut = builder->CreateBitCast(carryOut, carryTy);
     662    carryOut = iBuilder->CreateBitCast(carryOut, carryTy);
    664663    if (mCarryInfo->hasSummary()) {
    665         addToCarryOutSummary(builder, carryOut);
     664        addToCarryOutSummary(iBuilder, carryOut);
    666665    }
    667666    if (mLoopDepth != 0) {
    668         mCarryPackPtr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), builder->getInt32(mCurrentFrameIndex), mNextLoopSelector});
     667        mCarryPackPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex), mNextLoopSelector});
    669668        if (LLVM_LIKELY(!mCarryInfo->nonCarryCollapsingMode())) {
    670             Value * accum = builder->CreateBlockAlignedLoad(mCarryPackPtr);
    671             carryOut = builder->CreateOr(carryOut, accum);
     669            Value * accum = iBuilder->CreateBlockAlignedLoad(mCarryPackPtr);
     670            carryOut = iBuilder->CreateOr(carryOut, accum);
    672671        }
    673672    }
    674673    ++mCurrentFrameIndex;
    675674    assert (mCarryPackPtr->getType()->getPointerElementType() == carryTy);
    676     builder->CreateBlockAlignedStore(carryOut, mCarryPackPtr);
     675    iBuilder->CreateBlockAlignedStore(carryOut, mCarryPackPtr);
    677676}
    678677
     
    680679 * @brief readCarryInSummary
    681680 ** ------------------------------------------------------------------------------------------------------------- */
    682 Value * CarryManager::readCarryInSummary(IDISA::IDISA_Builder * const builder, ConstantInt * index) const {
     681Value * CarryManager::readCarryInSummary(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, ConstantInt * index) const {
    683682    assert (mCarryInfo->hasSummary());
    684683    unsigned count = 2;
     
    693692    const unsigned length = (mLoopDepth == 0) ? count : (count + 1);
    694693    Value * indicies[length];
    695     std::fill(indicies, indicies + count - 1, builder->getInt32(0));
     694    std::fill(indicies, indicies + count - 1, iBuilder->getInt32(0));
    696695    indicies[count - 1] = index;
    697696    if (mLoopDepth != 0) {
     
    700699
    701700    ArrayRef<Value *> ar(indicies, length);
    702     Value * const ptr = builder->CreateGEP(mCurrentFrame, ar);
    703     Value * const summary = builder->CreateBlockAlignedLoad(ptr);
     701    Value * const ptr = iBuilder->CreateGEP(mCurrentFrame, ar);
     702    Value * const summary = iBuilder->CreateBlockAlignedLoad(ptr);
    704703    if (mLoopDepth != 0 && mCarryInfo->hasExplicitSummary()) {
    705         Type * const carryTy = builder->getBitBlockType();
    706         builder->CreateBlockAlignedStore(Constant::getNullValue(carryTy), ptr);
     704        Type * const carryTy = iBuilder->getBitBlockType();
     705        iBuilder->CreateBlockAlignedStore(Constant::getNullValue(carryTy), ptr);
    707706    }
    708707    return summary;
     
    712711 * @brief writeCarryOutSummary
    713712 ** ------------------------------------------------------------------------------------------------------------- */
    714 inline void CarryManager::writeCarryOutSummary(IDISA::IDISA_Builder * const builder, Value * const summary, ConstantInt * index) const {
     713inline void CarryManager::writeCarryOutSummary(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * const summary, ConstantInt * index) const {
    715714    Value * ptr = nullptr;
    716715    assert (mCarryInfo->hasExplicitSummary());
    717716    if (mLoopDepth > 0) {
    718         ptr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), index, mNextLoopSelector});
     717        ptr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), index, mNextLoopSelector});
    719718    } else {
    720         ptr = builder->CreateGEP(mCurrentFrame, {builder->getInt32(0), index});
    721     }
    722     builder->CreateBlockAlignedStore(summary, ptr);
     719        ptr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), index});
     720    }
     721    iBuilder->CreateBlockAlignedStore(summary, ptr);
    723722}
    724723
     
    726725 * @brief addToCarryOutSummary
    727726 ** ------------------------------------------------------------------------------------------------------------- */
    728 inline void CarryManager::addToCarryOutSummary(IDISA::IDISA_Builder * const builder, Value * const value) {
     727inline void CarryManager::addToCarryOutSummary(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * const value) {
    729728    assert ("cannot add null summary value!" && value);   
    730729    assert ("summary stack is empty!" && !mCarrySummaryStack.empty());
    731730    assert (mCarryInfo->hasSummary());
    732     mCarrySummaryStack.back() = builder->CreateOr(value, mCarrySummaryStack.back());
     731    mCarrySummaryStack.back() = iBuilder->CreateOr(value, mCarrySummaryStack.back());
    733732}
    734733
     
    777776 * @brief analyse
    778777 ** ------------------------------------------------------------------------------------------------------------- */
    779 StructType * CarryManager::analyse(IDISA::IDISA_Builder * const builder, const PabloBlock * const scope, const unsigned ifDepth, const unsigned loopDepth, const bool isNestedWithinNonCarryCollapsingLoop) {
     778StructType * CarryManager::analyse(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope, const unsigned ifDepth, const unsigned loopDepth, const bool isNestedWithinNonCarryCollapsingLoop) {
    780779    assert ("scope cannot be null!" && scope);
    781780    assert (mCarryScopes == 0 ? (scope == mKernel->getEntryBlock()) : (scope != mKernel->getEntryBlock()));
    782781    assert (mCarryScopes < mCarryMetadata.size());
    783     Type * const carryTy = builder->getBitBlockType();
    784     Type * const blockTy = builder->getBitBlockType();
     782    Type * const carryTy = iBuilder->getBitBlockType();
     783    Type * const blockTy = iBuilder->getBitBlockType();
    785784
    786785    const unsigned carryScopeIndex = mCarryScopes++;
     
    794793            Type * type = carryPackType;
    795794            if (LLVM_UNLIKELY(amount >= LONG_ADVANCE_BREAKPOINT)) {
    796                 const unsigned blocks = ceil_udiv(amount, builder->getBitBlockWidth());
     795                const unsigned blocks = ceil_udiv(amount, iBuilder->getBitBlockWidth());
    797796                type = ArrayType::get(blockTy, nearest_pow2(blocks + ((loopDepth != 0) ? 1 : 0)));
    798                 if (LLVM_UNLIKELY(ifDepth > 0 && amount > builder->getBitBlockWidth())) {
     797                if (LLVM_UNLIKELY(ifDepth > 0 && amount > iBuilder->getBitBlockWidth())) {
    799798                    // 1 bit will mark the presense of any bit in each block.
    800                     Type * carryType = ArrayType::get(blockTy, ceil_udiv(amount, std::pow(builder->getBitBlockWidth(), 2)));
     799                    Type * carryType = ArrayType::get(blockTy, ceil_udiv(amount, std::pow(iBuilder->getBitBlockWidth(), 2)));
    801800                    state.push_back(carryType);
    802801                }
     
    807806            state.push_back(carryPackType);
    808807        } else if (LLVM_UNLIKELY(isa<If>(stmt))) {
    809             state.push_back(analyse(builder, cast<If>(stmt)->getBody(), ifDepth + 1, loopDepth, nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop));
     808            state.push_back(analyse(iBuilder, cast<If>(stmt)->getBody(), ifDepth + 1, loopDepth, nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop));
    810809        } else if (LLVM_UNLIKELY(isa<While>(stmt))) {
    811810            mHasLoop = true;
    812             state.push_back(analyse(builder, cast<While>(stmt)->getBody(), ifDepth, loopDepth + 1, nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop));
     811            state.push_back(analyse(iBuilder, cast<While>(stmt)->getBody(), ifDepth, loopDepth + 1, nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop));
    813812        }
    814813    }
     
    818817    CarryData::SummaryType summaryType = CarryData::NoSummary;
    819818    if (LLVM_UNLIKELY(state.empty())) {
    820         carryState = StructType::get(builder->getContext());
     819        carryState = StructType::get(iBuilder->getContext());
    821820    } else {
    822821        //if (ifDepth > 0 || (nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop)) {
     
    834833            }           
    835834        }
    836         carryState = StructType::get(builder->getContext(), state);
     835        carryState = StructType::get(iBuilder->getContext(), state);
    837836        // If we're in a loop and cannot use collapsing carry mode, convert the carry state struct into a capacity,
    838837        // carry state pointer, and summary pointer struct.
    839838        if (LLVM_UNLIKELY(nonCarryCollapsingMode)) {
    840             carryState = StructType::get(builder->getSizeTy(), carryState->getPointerTo(), carryTy->getPointerTo(), nullptr);
     839            carryState = StructType::get(iBuilder->getSizeTy(), carryState->getPointerTo(), carryTy->getPointerTo(), nullptr);
    841840        }
    842841        cd.setNonCollapsingCarryMode(nonCarryCollapsingMode);
  • icGREP/icgrep-devel/icgrep/pablo/carry_manager.h

    r5435 r5440  
    2323namespace pablo { class PabloKernel; }
    2424namespace pablo { class Statement; }
     25namespace kernel { class KernelBuilder; }
    2526
    2627/*
     
    4546    CarryManager() noexcept;
    4647
    47     void initializeCarryData(IDISA::IDISA_Builder * const builder, PabloKernel * const kernel);
     48    void initializeCarryData(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, PabloKernel * const kernel);
    4849
    49     void initializeCodeGen(IDISA::IDISA_Builder * const builder);
     50    void initializeCodeGen(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder);
    5051
    51     void finalizeCodeGen(IDISA::IDISA_Builder * const builder);
     52    void finalizeCodeGen(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder);
    5253
    5354    /* Entering and leaving loops. */
    5455
    55     void enterLoopScope(IDISA::IDISA_Builder * const builder, const PabloBlock * const scope);
     56    void enterLoopScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope);
    5657
    57     void enterLoopBody(IDISA::IDISA_Builder * const builder, llvm::BasicBlock * const entryBlock);
     58    void enterLoopBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::BasicBlock * const entryBlock);
    5859
    59     void leaveLoopBody(IDISA::IDISA_Builder * const builder, llvm::BasicBlock * const exitBlock);
     60    void leaveLoopBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::BasicBlock * const exitBlock);
    6061
    61     void leaveLoopScope(IDISA::IDISA_Builder * const builder, llvm::BasicBlock * const entryBlock, llvm::BasicBlock * const exitBlock);
     62    void leaveLoopScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::BasicBlock * const entryBlock, llvm::BasicBlock * const exitBlock);
    6263
    6364    /* Entering and leaving ifs. */
    6465
    65     void enterIfScope(IDISA::IDISA_Builder * const builder, const PabloBlock * const scope);
     66    void enterIfScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope);
    6667
    67     void enterIfBody(IDISA::IDISA_Builder * const builder, llvm::BasicBlock * const entryBlock);
     68    void enterIfBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::BasicBlock * const entryBlock);
    6869
    69     void leaveIfBody(IDISA::IDISA_Builder * const builder, llvm::BasicBlock * const exitBlock);
     70    void leaveIfBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::BasicBlock * const exitBlock);
    7071
    71     void leaveIfScope(IDISA::IDISA_Builder * const builder, llvm::BasicBlock * const entryBlock, llvm::BasicBlock * const exitBlock);
     72    void leaveIfScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::BasicBlock * const entryBlock, llvm::BasicBlock * const exitBlock);
    7273
    7374    /* Methods for processing individual carry-generating operations. */
    7475   
    75     llvm::Value * addCarryInCarryOut(IDISA::IDISA_Builder * const builder, const Statement * operation, llvm::Value * const e1, llvm::Value * const e2);
     76    llvm::Value * addCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const Statement * operation, llvm::Value * const e1, llvm::Value * const e2);
    7677
    77     llvm::Value * advanceCarryInCarryOut(IDISA::IDISA_Builder * const builder, const Advance * advance, llvm::Value * const strm);
     78    llvm::Value * advanceCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const Advance * advance, llvm::Value * const strm);
    7879 
    7980    /* Methods for getting and setting carry summary values for If statements */
    8081         
    81     llvm::Value * generateSummaryTest(IDISA::IDISA_Builder * const builder, llvm::Value * condition);
     82    llvm::Value * generateSummaryTest(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::Value * condition);
    8283
    8384protected:
     
    8788    static bool hasIterationSpecificAssignment(const PabloBlock * const scope);
    8889
    89     llvm::StructType * analyse(IDISA::IDISA_Builder * const builder, const PabloBlock * const scope, const unsigned ifDepth = 0, const unsigned whileDepth = 0, const bool isNestedWithinNonCarryCollapsingLoop = false);
     90    llvm::StructType * analyse(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope, const unsigned ifDepth = 0, const unsigned whileDepth = 0, const bool isNestedWithinNonCarryCollapsingLoop = false);
    9091
    9192    /* Entering and leaving scopes. */
    92     void enterScope(IDISA::IDISA_Builder * const builder, const PabloBlock * const scope);
    93     void leaveScope(IDISA::IDISA_Builder * const builder);
     93    void enterScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope);
     94    void leaveScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder);
    9495
    9596    /* Methods for processing individual carry-generating operations. */
    96     llvm::Value * getNextCarryIn(IDISA::IDISA_Builder * const builder);
    97     void setNextCarryOut(IDISA::IDISA_Builder * const builder, llvm::Value * const carryOut);
    98     llvm::Value * longAdvanceCarryInCarryOut(IDISA::IDISA_Builder * const builder, llvm::Value * const value, const unsigned shiftAmount);
    99     llvm::Value * readCarryInSummary(IDISA::IDISA_Builder * const builder, llvm::ConstantInt *index) const;
    100     void writeCarryOutSummary(IDISA::IDISA_Builder * const builder, llvm::Value * const summary, llvm::ConstantInt * index) const;
     97    llvm::Value * getNextCarryIn(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder);
     98    void setNextCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::Value * const carryOut);
     99    llvm::Value * longAdvanceCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::Value * const value, const unsigned shiftAmount);
     100    llvm::Value * readCarryInSummary(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::ConstantInt *index) const;
     101    void writeCarryOutSummary(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::Value * const summary, llvm::ConstantInt * index) const;
    101102
    102103    /* Summary handling routines */
    103     void addToCarryOutSummary(IDISA::IDISA_Builder * const builder, llvm::Value * const value);
     104    void addToCarryOutSummary(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, llvm::Value * const value);
    104105
    105106private:
  • icGREP/icgrep-devel/icgrep/pablo/codegenstate.cpp

    r5436 r5440  
    186186LessThan * PabloBlock::createLessThan(PabloAST * expr1, PabloAST * expr2) {
    187187    CHECK_SAME_TYPE(expr1, expr2);
    188     return new (mAllocator) LessThan(getParent()->getBuilder()->getInt1Ty(), expr1, expr2, mAllocator);
     188    Type * type = getParent()->getBuilder()->getInt1Ty();
     189    return new (mAllocator) LessThan(type, expr1, expr2, mAllocator);
    189190}
    190191
  • icGREP/icgrep-devel/icgrep/pablo/pablo_compiler.cpp

    r5435 r5440  
    2525#include <pablo/ps_assign.h>
    2626#include <pablo/carry_manager.h>
    27 #include <IR_Gen/idisa_builder.h>
     27#include <kernels/kernel_builder.h>
    2828#include <kernels/streamset.h>
    2929#include <llvm/IR/Module.h>
     
    4444}
    4545
    46 void PabloCompiler::initializeKernelData(IDISA::IDISA_Builder * const builder) {
    47     assert ("PabloCompiler does not have a IDISA builder" && builder);
    48     examineBlock(builder, mKernel->getEntryBlock());
    49     mCarryManager->initializeCarryData(builder, mKernel);
    50 }
    51 
    52 void PabloCompiler::compile(IDISA::IDISA_Builder * const builder) {
    53     assert ("PabloCompiler does not have a IDISA builder" && builder);
    54     mCarryManager->initializeCodeGen(builder);
     46void PabloCompiler::initializeKernelData(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder) {
     47    assert ("PabloCompiler does not have a IDISA iBuilder" && iBuilder);
     48    examineBlock(iBuilder, mKernel->getEntryBlock());
     49    mCarryManager->initializeCarryData(iBuilder, mKernel);
     50}
     51
     52void PabloCompiler::compile(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder) {
     53    assert ("PabloCompiler does not have a IDISA iBuilder" && iBuilder);
     54    mCarryManager->initializeCodeGen(iBuilder);
    5555    PabloBlock * const entryBlock = mKernel->getEntryBlock(); assert (entryBlock);
    56     mMarker.emplace(entryBlock->createZeroes(), builder->allZeroes());
    57     mMarker.emplace(entryBlock->createOnes(), builder->allOnes());
    58     compileBlock(builder, entryBlock);
    59     mCarryManager->finalizeCodeGen(builder);
    60 }
    61 
    62 void PabloCompiler::examineBlock(IDISA::IDISA_Builder * const builder, const PabloBlock * const block) {
     56    mMarker.emplace(entryBlock->createZeroes(), iBuilder->allZeroes());
     57    mMarker.emplace(entryBlock->createOnes(), iBuilder->allOnes());
     58    compileBlock(iBuilder, entryBlock);
     59    mCarryManager->finalizeCodeGen(iBuilder);
     60}
     61
     62void PabloCompiler::examineBlock(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const block) {
    6363    for (const Statement * stmt : *block) {
    6464        if (LLVM_UNLIKELY(isa<Lookahead>(stmt))) {
     
    6969            }
    7070        } else if (LLVM_UNLIKELY(isa<Branch>(stmt))) {
    71             examineBlock(builder, cast<Branch>(stmt)->getBody());
     71            examineBlock(iBuilder, cast<Branch>(stmt)->getBody());
    7272        } else if (LLVM_UNLIKELY(isa<Count>(stmt))) {
    73             mAccumulator.insert(std::make_pair(stmt, builder->getInt32(mKernel->addUnnamedScalar(stmt->getType()))));
     73            mAccumulator.insert(std::make_pair(stmt, iBuilder->getInt32(mKernel->addUnnamedScalar(stmt->getType()))));
    7474        }
    7575    }   
    7676}
    7777
    78 inline void PabloCompiler::compileBlock(IDISA::IDISA_Builder * const builder, const PabloBlock * const block) {
     78inline void PabloCompiler::compileBlock(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const block) {
    7979    for (const Statement * statement : *block) {
    80         compileStatement(builder, statement);
    81     }
    82 }
    83 
    84 void PabloCompiler::compileIf(IDISA::IDISA_Builder * const builder, const If * const ifStatement) {
     80        compileStatement(iBuilder, statement);
     81    }
     82}
     83
     84void PabloCompiler::compileIf(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const If * const ifStatement) {
    8585    //
    8686    //  The If-ElseZero stmt:
     
    101101    //
    102102
    103     BasicBlock * const ifEntryBlock = builder->GetInsertBlock();
    104     BasicBlock * const ifBodyBlock = mKernel->CreateBasicBlock("if.body");
    105     BasicBlock * const ifEndBlock = mKernel->CreateBasicBlock("if.end");
     103    BasicBlock * const ifEntryBlock = iBuilder->GetInsertBlock();
     104    BasicBlock * const ifBodyBlock = iBuilder->CreateBasicBlock("if.body");
     105    BasicBlock * const ifEndBlock = iBuilder->CreateBasicBlock("if.end");
    106106   
    107107    std::vector<std::pair<const Var *, Value *>> incoming;
     
    111111            Value * marker = nullptr;
    112112            if (var->isScalar()) {
    113                 marker = mKernel->getScalarFieldPtr(var->getName());
     113                marker = iBuilder->getScalarFieldPtr(var->getName());
    114114            } else if (var->isReadOnly()) {
    115                 marker = mKernel->getInputStreamBlockPtr(var->getName(), builder->getInt32(0));
     115                marker = iBuilder->getInputStreamBlockPtr(var->getName(), iBuilder->getInt32(0));
    116116            } else if (var->isReadNone()) {
    117                 marker = mKernel->getOutputStreamBlockPtr(var->getName(), builder->getInt32(0));
     117                marker = iBuilder->getOutputStreamBlockPtr(var->getName(), iBuilder->getInt32(0));
    118118            }
    119119            mMarker[var] = marker;
     
    134134    const PabloBlock * ifBody = ifStatement->getBody();
    135135   
    136     mCarryManager->enterIfScope(builder, ifBody);
    137 
    138     Value * condition = compileExpression(builder, ifStatement->getCondition());
    139     if (condition->getType() == builder->getBitBlockType()) {
    140         condition = builder->bitblock_any(mCarryManager->generateSummaryTest(builder, condition));
     136    mCarryManager->enterIfScope(iBuilder, ifBody);
     137
     138    Value * condition = compileExpression(iBuilder, ifStatement->getCondition());
     139    if (condition->getType() == iBuilder->getBitBlockType()) {
     140        condition = iBuilder->bitblock_any(mCarryManager->generateSummaryTest(iBuilder, condition));
    141141    }
    142142   
    143     builder->CreateCondBr(condition, ifBodyBlock, ifEndBlock);
     143    iBuilder->CreateCondBr(condition, ifBodyBlock, ifEndBlock);
    144144   
    145145    // Entry processing is complete, now handle the body of the if.
    146     builder->SetInsertPoint(ifBodyBlock);
    147 
    148     mCarryManager->enterIfBody(builder, ifEntryBlock);
    149 
    150     compileBlock(builder, ifBody);
    151 
    152     mCarryManager->leaveIfBody(builder, builder->GetInsertBlock());
    153 
    154     BasicBlock * ifExitBlock = builder->GetInsertBlock();
    155 
    156     builder->CreateBr(ifEndBlock);
     146    iBuilder->SetInsertPoint(ifBodyBlock);
     147
     148    mCarryManager->enterIfBody(iBuilder, ifEntryBlock);
     149
     150    compileBlock(iBuilder, ifBody);
     151
     152    mCarryManager->leaveIfBody(iBuilder, iBuilder->GetInsertBlock());
     153
     154    BasicBlock * ifExitBlock = iBuilder->GetInsertBlock();
     155
     156    iBuilder->CreateBr(ifEndBlock);
    157157
    158158    ifEndBlock->moveAfter(ifExitBlock);
    159159
    160160    //End Block
    161     builder->SetInsertPoint(ifEndBlock);
    162 
    163     mCarryManager->leaveIfScope(builder, ifEntryBlock, ifExitBlock);
     161    iBuilder->SetInsertPoint(ifEndBlock);
     162
     163    mCarryManager->leaveIfScope(iBuilder, ifEntryBlock, ifExitBlock);
    164164
    165165    for (const auto i : incoming) {
     
    196196        }
    197197
    198         PHINode * phi = builder->CreatePHI(incoming->getType(), 2, var->getName());
     198        PHINode * phi = iBuilder->CreatePHI(incoming->getType(), 2, var->getName());
    199199        phi->addIncoming(incoming, ifEntryBlock);
    200200        phi->addIncoming(outgoing, ifExitBlock);
     
    203203}
    204204
    205 void PabloCompiler::compileWhile(IDISA::IDISA_Builder * const builder, const While * const whileStatement) {
     205void PabloCompiler::compileWhile(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const While * const whileStatement) {
    206206
    207207    const PabloBlock * const whileBody = whileStatement->getBody();
    208208
    209     BasicBlock * whileEntryBlock = builder->GetInsertBlock();
     209    BasicBlock * whileEntryBlock = iBuilder->GetInsertBlock();
    210210
    211211    const auto escaped = whileStatement->getEscaped();
     
    222222            Value * marker = nullptr;
    223223            if (var->isScalar()) {
    224                 marker = mKernel->getScalarFieldPtr(var->getName());
     224                marker = iBuilder->getScalarFieldPtr(var->getName());
    225225            } else if (var->isReadOnly()) {
    226                 marker = mKernel->getInputStreamBlockPtr(var->getName(), builder->getInt32(0));
     226                marker = iBuilder->getInputStreamBlockPtr(var->getName(), iBuilder->getInt32(0));
    227227            } else if (var->isReadNone()) {
    228                 marker = mKernel->getOutputStreamBlockPtr(var->getName(), builder->getInt32(0));
     228                marker = iBuilder->getOutputStreamBlockPtr(var->getName(), iBuilder->getInt32(0));
    229229            }
    230230            mMarker[var] = marker;
     
    232232    }
    233233
    234     mCarryManager->enterLoopScope(builder, whileBody);
    235 
    236     BasicBlock * whileBodyBlock = mKernel->CreateBasicBlock("while.body");
    237 
    238     builder->CreateBr(whileBodyBlock);
    239 
    240     builder->SetInsertPoint(whileBodyBlock);
     234    mCarryManager->enterLoopScope(iBuilder, whileBody);
     235
     236    BasicBlock * whileBodyBlock = iBuilder->CreateBasicBlock("while.body");
     237
     238    iBuilder->CreateBr(whileBodyBlock);
     239
     240    iBuilder->SetInsertPoint(whileBodyBlock);
    241241
    242242    //
     
    264264        }
    265265        Value * entryValue = f->second;
    266         PHINode * phi = builder->CreatePHI(entryValue->getType(), 2, var->getName());
     266        PHINode * phi = iBuilder->CreatePHI(entryValue->getType(), 2, var->getName());
    267267        phi->addIncoming(entryValue, whileEntryBlock);
    268268        f->second = phi;
     
    277277#endif
    278278
    279     mCarryManager->enterLoopBody(builder, whileEntryBlock);
    280 
    281     compileBlock(builder, whileBody);
     279    mCarryManager->enterLoopBody(iBuilder, whileEntryBlock);
     280
     281    compileBlock(iBuilder, whileBody);
    282282
    283283    // After the whileBody has been compiled, we may be in a different basic block.
    284284
    285     mCarryManager->leaveLoopBody(builder, builder->GetInsertBlock());
     285    mCarryManager->leaveLoopBody(iBuilder, iBuilder->GetInsertBlock());
    286286
    287287
     
    294294#endif
    295295
    296     BasicBlock * const whileExitBlock = builder->GetInsertBlock();
     296    BasicBlock * const whileExitBlock = iBuilder->GetInsertBlock();
    297297
    298298    // and for any variant nodes in the loop body
     
    329329    }
    330330
    331     BasicBlock * whileEndBlock = mKernel->CreateBasicBlock("while.end");
     331    BasicBlock * whileEndBlock = iBuilder->CreateBasicBlock("while.end");
    332332
    333333    // Terminate the while loop body with a conditional branch back.
    334     Value * condition = compileExpression(builder, whileStatement->getCondition());
    335     if (condition->getType() == builder->getBitBlockType()) {
    336         condition = builder->bitblock_any(mCarryManager->generateSummaryTest(builder, condition));
    337     }
    338 
    339     builder->CreateCondBr(condition, whileBodyBlock, whileEndBlock);
    340 
    341     builder->SetInsertPoint(whileEndBlock);
    342 
    343     mCarryManager->leaveLoopScope(builder, whileEntryBlock, whileExitBlock);
    344 
    345 }
    346 
    347 void PabloCompiler::compileStatement(IDISA::IDISA_Builder * const builder, const Statement * const stmt) {
     334    Value * condition = compileExpression(iBuilder, whileStatement->getCondition());
     335    if (condition->getType() == iBuilder->getBitBlockType()) {
     336        condition = iBuilder->bitblock_any(mCarryManager->generateSummaryTest(iBuilder, condition));
     337    }
     338
     339    iBuilder->CreateCondBr(condition, whileBodyBlock, whileEndBlock);
     340
     341    iBuilder->SetInsertPoint(whileEndBlock);
     342
     343    mCarryManager->leaveLoopScope(iBuilder, whileEntryBlock, whileExitBlock);
     344
     345}
     346
     347void PabloCompiler::compileStatement(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const Statement * const stmt) {
    348348
    349349    if (LLVM_UNLIKELY(isa<If>(stmt))) {
    350         compileIf(builder, cast<If>(stmt));
     350        compileIf(iBuilder, cast<If>(stmt));
    351351    } else if (LLVM_UNLIKELY(isa<While>(stmt))) {
    352         compileWhile(builder, cast<While>(stmt));
     352        compileWhile(iBuilder, cast<While>(stmt));
    353353    } else {
    354354        const PabloAST * expr = stmt;
    355355        Value * value = nullptr;
    356356        if (LLVM_UNLIKELY(isa<Assign>(stmt))) {
    357             value = compileExpression(builder, cast<Assign>(stmt)->getValue());
     357            value = compileExpression(iBuilder, cast<Assign>(stmt)->getValue());
    358358            expr = cast<Assign>(stmt)->getVariable();
    359359            Value * ptr = nullptr;
     
    373373                if (var->isKernelParameter()) {
    374374                    if (var->isScalar()) {
    375                         ptr = mKernel->getScalarFieldPtr(var->getName());
     375                        ptr = iBuilder->getScalarFieldPtr(var->getName());
    376376                    } else {
    377                         ptr = mKernel->getOutputStreamBlockPtr(var->getName(), builder->getInt32(0));
     377                        ptr = iBuilder->getOutputStreamBlockPtr(var->getName(), iBuilder->getInt32(0));
    378378                    }
    379379                }
     
    396396            }
    397397            if (ptr) {
    398                 builder->CreateAlignedStore(value, ptr, getAlignment(value));
     398                iBuilder->CreateAlignedStore(value, ptr, getAlignment(value));
    399399                value = ptr;
    400400            }
    401401        } else if (const Extract * extract = dyn_cast<Extract>(stmt)) {
    402             Value * index = compileExpression(builder, extract->getIndex());
     402            Value * index = compileExpression(iBuilder, extract->getIndex());
    403403            Var * const array = dyn_cast<Var>(extract->getArray());
    404404            if (LLVM_LIKELY(array && array->isKernelParameter())) {
    405405                if (array->isReadOnly()) {
    406                     value = mKernel->getInputStreamBlockPtr(array->getName(), index);
     406                    value = iBuilder->getInputStreamBlockPtr(array->getName(), index);
    407407                } else if (array->isReadNone()) {
    408                     value = mKernel->getOutputStreamBlockPtr(array->getName(), index);
     408                    value = iBuilder->getOutputStreamBlockPtr(array->getName(), index);
    409409                } else {
    410410                    std::string tmp;
     
    417417                }
    418418            } else {
    419                 Value * ptr = compileExpression(builder, extract->getArray(), false);
    420                 value = builder->CreateGEP(ptr, {ConstantInt::getNullValue(index->getType()), index}, "extract");
     419                Value * ptr = compileExpression(iBuilder, extract->getArray(), false);
     420                value = iBuilder->CreateGEP(ptr, {ConstantInt::getNullValue(index->getType()), index}, "extract");
    421421            }
    422422        } else if (isa<And>(stmt)) {
    423             value = compileExpression(builder, stmt->getOperand(0));
     423            value = compileExpression(iBuilder, stmt->getOperand(0));
    424424            for (unsigned i = 1; i < stmt->getNumOperands(); ++i) {
    425                 value = builder->simd_and(value, compileExpression(builder, stmt->getOperand(1)));
     425                value = iBuilder->simd_and(value, compileExpression(iBuilder, stmt->getOperand(1)));
    426426            }
    427427        } else if (isa<Or>(stmt)) {
    428             value = compileExpression(builder, stmt->getOperand(0));
     428            value = compileExpression(iBuilder, stmt->getOperand(0));
    429429            for (unsigned i = 1; i < stmt->getNumOperands(); ++i) {
    430                 value = builder->simd_or(value, compileExpression(builder, stmt->getOperand(1)));
     430                value = iBuilder->simd_or(value, compileExpression(iBuilder, stmt->getOperand(1)));
    431431            }
    432432        } else if (isa<Xor>(stmt)) {
    433             value = compileExpression(builder, stmt->getOperand(0));
     433            value = compileExpression(iBuilder, stmt->getOperand(0));
    434434            for (unsigned i = 1; i < stmt->getNumOperands(); ++i) {
    435                 value = builder->simd_xor(value, compileExpression(builder, stmt->getOperand(1)));
     435                value = iBuilder->simd_xor(value, compileExpression(iBuilder, stmt->getOperand(1)));
    436436            }
    437437        } else if (const Sel * sel = dyn_cast<Sel>(stmt)) {
    438             Value* ifMask = compileExpression(builder, sel->getCondition());
    439             Value* ifTrue = builder->simd_and(ifMask, compileExpression(builder, sel->getTrueExpr()));
    440             Value* ifFalse = builder->simd_and(builder->simd_not(ifMask), compileExpression(builder, sel->getFalseExpr()));
    441             value = builder->simd_or(ifTrue, ifFalse);
     438            Value* ifMask = compileExpression(iBuilder, sel->getCondition());
     439            Value* ifTrue = iBuilder->simd_and(ifMask, compileExpression(iBuilder, sel->getTrueExpr()));
     440            Value* ifFalse = iBuilder->simd_and(iBuilder->simd_not(ifMask), compileExpression(iBuilder, sel->getFalseExpr()));
     441            value = iBuilder->simd_or(ifTrue, ifFalse);
    442442        } else if (isa<Not>(stmt)) {
    443             value = builder->simd_not(compileExpression(builder, stmt->getOperand(0)));
     443            value = iBuilder->simd_not(compileExpression(iBuilder, stmt->getOperand(0)));
    444444        } else if (isa<Advance>(stmt)) {
    445445            const Advance * const adv = cast<Advance>(stmt);
    446446            // If our expr is an Extract op on a mutable Var then we need to pass the index value to the carry
    447447            // manager so that it properly selects the correct carry bit.
    448             value = mCarryManager->advanceCarryInCarryOut(builder, adv, compileExpression(builder, adv->getExpression()));
     448            value = mCarryManager->advanceCarryInCarryOut(iBuilder, adv, compileExpression(iBuilder, adv->getExpression()));
    449449        } else if (const MatchStar * mstar = dyn_cast<MatchStar>(stmt)) {
    450             Value * const marker = compileExpression(builder, mstar->getMarker());
    451             Value * const cc = compileExpression(builder, mstar->getCharClass());
    452             Value * const marker_and_cc = builder->simd_and(marker, cc);
    453             Value * const sum = mCarryManager->addCarryInCarryOut(builder, mstar, marker_and_cc, cc);
    454             value = builder->simd_or(builder->simd_xor(sum, cc), marker);
     450            Value * const marker = compileExpression(iBuilder, mstar->getMarker());
     451            Value * const cc = compileExpression(iBuilder, mstar->getCharClass());
     452            Value * const marker_and_cc = iBuilder->simd_and(marker, cc);
     453            Value * const sum = mCarryManager->addCarryInCarryOut(iBuilder, mstar, marker_and_cc, cc);
     454            value = iBuilder->simd_or(iBuilder->simd_xor(sum, cc), marker);
    455455        } else if (const ScanThru * sthru = dyn_cast<ScanThru>(stmt)) {
    456             Value * const from = compileExpression(builder, sthru->getScanFrom());
    457             Value * const thru = compileExpression(builder, sthru->getScanThru());
    458             Value * const sum = mCarryManager->addCarryInCarryOut(builder, sthru, from, thru);
    459             value = builder->simd_and(sum, builder->simd_not(thru));
     456            Value * const from = compileExpression(iBuilder, sthru->getScanFrom());
     457            Value * const thru = compileExpression(iBuilder, sthru->getScanThru());
     458            Value * const sum = mCarryManager->addCarryInCarryOut(iBuilder, sthru, from, thru);
     459            value = iBuilder->simd_and(sum, iBuilder->simd_not(thru));
    460460        } else if (const ScanTo * sthru = dyn_cast<ScanTo>(stmt)) {
    461             Value * const marker_expr = compileExpression(builder, sthru->getScanFrom());
    462             Value * const to = builder->simd_xor(compileExpression(builder, sthru->getScanTo()), mKernel->getScalarField("EOFmask"));
    463             Value * const sum = mCarryManager->addCarryInCarryOut(builder, sthru, marker_expr, builder->simd_not(to));
    464             value = builder->simd_and(sum, to);
     461            Value * const marker_expr = compileExpression(iBuilder, sthru->getScanFrom());
     462            Value * const to = iBuilder->simd_xor(compileExpression(iBuilder, sthru->getScanTo()), iBuilder->getScalarField("EOFmask"));
     463            Value * const sum = mCarryManager->addCarryInCarryOut(iBuilder, sthru, marker_expr, iBuilder->simd_not(to));
     464            value = iBuilder->simd_and(sum, to);
    465465        } else if (const AdvanceThenScanThru * sthru = dyn_cast<AdvanceThenScanThru>(stmt)) {
    466             Value * const from = compileExpression(builder, sthru->getScanFrom());
    467             Value * const thru = compileExpression(builder, sthru->getScanThru());
    468             Value * const sum = mCarryManager->addCarryInCarryOut(builder, sthru, from, builder->simd_or(from, thru));
    469             value = builder->simd_and(sum, builder->simd_not(thru));
     466            Value * const from = compileExpression(iBuilder, sthru->getScanFrom());
     467            Value * const thru = compileExpression(iBuilder, sthru->getScanThru());
     468            Value * const sum = mCarryManager->addCarryInCarryOut(iBuilder, sthru, from, iBuilder->simd_or(from, thru));
     469            value = iBuilder->simd_and(sum, iBuilder->simd_not(thru));
    470470        } else if (const AdvanceThenScanTo * sthru = dyn_cast<AdvanceThenScanTo>(stmt)) {
    471             Value * const from = compileExpression(builder, sthru->getScanFrom());
    472             Value * const to = builder->simd_xor(compileExpression(builder, sthru->getScanTo()), mKernel->getScalarField("EOFmask"));
    473             Value * const sum = mCarryManager->addCarryInCarryOut(builder, sthru, from, builder->simd_or(from, builder->simd_not(to)));
    474             value = builder->simd_and(sum, to);
     471            Value * const from = compileExpression(iBuilder, sthru->getScanFrom());
     472            Value * const to = iBuilder->simd_xor(compileExpression(iBuilder, sthru->getScanTo()), iBuilder->getScalarField("EOFmask"));
     473            Value * const sum = mCarryManager->addCarryInCarryOut(iBuilder, sthru, from, iBuilder->simd_or(from, iBuilder->simd_not(to)));
     474            value = iBuilder->simd_and(sum, to);
    475475        } else if (const InFile * e = dyn_cast<InFile>(stmt)) {
    476             Value * EOFmask = mKernel->getScalarField("EOFmask");
    477             value = builder->simd_and(compileExpression(builder, e->getExpr()), builder->simd_not(EOFmask));
     476            Value * EOFmask = iBuilder->getScalarField("EOFmask");
     477            value = iBuilder->simd_and(compileExpression(iBuilder, e->getExpr()), iBuilder->simd_not(EOFmask));
    478478        } else if (const AtEOF * e = dyn_cast<AtEOF>(stmt)) {
    479             Value * EOFbit = mKernel->getScalarField("EOFbit");
    480             value = builder->simd_and(compileExpression(builder, e->getExpr()), EOFbit);
     479            Value * EOFbit = iBuilder->getScalarField("EOFbit");
     480            value = iBuilder->simd_and(compileExpression(iBuilder, e->getExpr()), EOFbit);
    481481        } else if (const Count * c = dyn_cast<Count>(stmt)) {
    482             Value * EOFbit = mKernel->getScalarField("EOFbit");
    483             Value * EOFmask = mKernel->getScalarField("EOFmask");
    484         Value * const to_count = builder->simd_and(builder->simd_or(builder->simd_not(EOFmask), EOFbit), compileExpression(builder, c->getExpr()));
    485             const unsigned counterSize = builder->getSizeTy()->getBitWidth();
     482        Value * EOFbit = iBuilder->getScalarField("EOFbit");
     483        Value * EOFmask = iBuilder->getScalarField("EOFmask");
     484        Value * const to_count = iBuilder->simd_and(iBuilder->simd_or(iBuilder->simd_not(EOFmask), EOFbit), compileExpression(iBuilder, c->getExpr()));
     485            const unsigned counterSize = iBuilder->getSizeTy()->getBitWidth();
    486486            const auto f = mAccumulator.find(c);
    487487            if (LLVM_UNLIKELY(f == mAccumulator.end())) {
    488488                report_fatal_error("Unknown accumulator: " + c->getName().str());
    489489            }
    490             Value * ptr = mKernel->getScalarFieldPtr(f->second);
     490            Value * ptr = iBuilder->getScalarFieldPtr(f->second);
    491491            const auto alignment = getPointerElementAlignment(ptr);
    492             Value * count = builder->CreateAlignedLoad(ptr, alignment, c->getName() + "_accumulator");
    493             Value * const partial = builder->simd_popcount(counterSize, to_count);
     492            Value * count = iBuilder->CreateAlignedLoad(ptr, alignment, c->getName() + "_accumulator");
     493            Value * const partial = iBuilder->simd_popcount(counterSize, to_count);
    494494            if (LLVM_UNLIKELY(counterSize <= 1)) {
    495495                value = partial;
    496496            } else {
    497                 value = builder->mvmd_extract(counterSize, partial, 0);
    498                 const auto fields = (builder->getBitBlockWidth() / counterSize);
     497                value = iBuilder->mvmd_extract(counterSize, partial, 0);
     498                const auto fields = (iBuilder->getBitBlockWidth() / counterSize);
    499499                for (unsigned i = 1; i < fields; ++i) {
    500                     Value * temp = builder->mvmd_extract(counterSize, partial, i);
    501                     value = builder->CreateAdd(value, temp);
    502                 }
    503             }
    504             value = builder->CreateAdd(value, count);
    505             builder->CreateAlignedStore(value, ptr, alignment);
     500                    Value * temp = iBuilder->mvmd_extract(counterSize, partial, i);
     501                    value = iBuilder->CreateAdd(value, temp);
     502                }
     503            }
     504            value = iBuilder->CreateAdd(value, count);
     505            iBuilder->CreateAlignedStore(value, ptr, alignment);
    506506        } else if (const Lookahead * l = dyn_cast<Lookahead>(stmt)) {
    507507            Var * var = nullptr;
    508508            PabloAST * stream = l->getExpr();
    509             Value * index = builder->getInt32(0);
     509            Value * index = iBuilder->getInt32(0);
    510510            if (LLVM_UNLIKELY(isa<Extract>(stream))) {
    511511                var = dyn_cast<Var>(cast<Extract>(stream)->getArray());
    512                 index = compileExpression(builder, cast<Extract>(stream)->getIndex());
     512                index = compileExpression(iBuilder, cast<Extract>(stream)->getIndex());
    513513                if (!var->isKernelParameter() || var->isReadNone()) {
    514514                    std::string tmp;
     
    533533                }
    534534            }
    535             const auto bit_shift = (l->getAmount() % builder->getBitBlockWidth());
    536             const auto block_shift = (l->getAmount() / builder->getBitBlockWidth());
    537 
    538             Value * ptr = mKernel->getAdjustedInputStreamBlockPtr(builder->getSize(block_shift), var->getName(), index);
    539             Value * lookAhead = builder->CreateBlockAlignedLoad(ptr);
     535            const auto bit_shift = (l->getAmount() % iBuilder->getBitBlockWidth());
     536            const auto block_shift = (l->getAmount() / iBuilder->getBitBlockWidth());
     537
     538            Value * ptr = iBuilder->getAdjustedInputStreamBlockPtr(iBuilder->getSize(block_shift), var->getName(), index);
     539            Value * lookAhead = iBuilder->CreateBlockAlignedLoad(ptr);
    540540            if (bit_shift == 0) {  // Simple case with no intra-block shifting.
    541541                value = lookAhead;
    542542            } else { // Need to form shift result from two adjacent blocks.
    543                 Value * ptr = mKernel->getAdjustedInputStreamBlockPtr(builder->getSize(block_shift + 1), var->getName(), index);
    544                 Value * lookAhead1 = builder->CreateBlockAlignedLoad(ptr);
     543                Value * ptr = iBuilder->getAdjustedInputStreamBlockPtr(iBuilder->getSize(block_shift + 1), var->getName(), index);
     544                Value * lookAhead1 = iBuilder->CreateBlockAlignedLoad(ptr);
    545545                if (LLVM_UNLIKELY((bit_shift % 8) == 0)) { // Use a single whole-byte shift, if possible.
    546                     value = builder->mvmd_dslli(8, lookAhead1, lookAhead, (bit_shift / 8));
     546                    value = iBuilder->mvmd_dslli(8, lookAhead1, lookAhead, (bit_shift / 8));
    547547                } else {
    548                     Type  * const streamType = builder->getIntNTy(builder->getBitBlockWidth());
    549                     Value * b1 = builder->CreateBitCast(lookAhead1, streamType);
    550                     Value * b0 = builder->CreateBitCast(lookAhead, streamType);
    551                     Value * result = builder->CreateOr(builder->CreateShl(b1, builder->getBitBlockWidth() - bit_shift), builder->CreateLShr(b0, bit_shift));
    552                     value = builder->CreateBitCast(result, builder->getBitBlockType());
     548                    Type  * const streamType = iBuilder->getIntNTy(iBuilder->getBitBlockWidth());
     549                    Value * b1 = iBuilder->CreateBitCast(lookAhead1, streamType);
     550                    Value * b0 = iBuilder->CreateBitCast(lookAhead, streamType);
     551                    Value * result = iBuilder->CreateOr(iBuilder->CreateShl(b1, iBuilder->getBitBlockWidth() - bit_shift), iBuilder->CreateLShr(b0, bit_shift));
     552                    value = iBuilder->CreateBitCast(result, iBuilder->getBitBlockType());
    553553                }
    554554            }
     
    567567            const String & name = isa<Var>(expr) ? cast<Var>(expr)->getName() : cast<Statement>(expr)->getName();
    568568            if (value->getType()->isPointerTy()) {
    569                 value = builder->CreateLoad(value);
     569                value = iBuilder->CreateLoad(value);
    570570            }
    571571            if (value->getType()->isVectorTy()) {
    572                 builder->CallPrintRegister(name.str(), value);
     572                iBuilder->CallPrintRegister(name.str(), value);
    573573            } else if (value->getType()->isIntegerTy()) {
    574                 builder->CallPrintInt(name.str(), value);
    575             }
    576         }
    577     }
    578 }
    579 
    580 Value * PabloCompiler::compileExpression(IDISA::IDISA_Builder * const builder, const PabloAST * expr, const bool ensureLoaded) const {
     574                iBuilder->CallPrintInt(name.str(), value);
     575            }
     576        }
     577    }
     578}
     579
     580Value * PabloCompiler::compileExpression(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloAST * expr, const bool ensureLoaded) const {
    581581    if (LLVM_UNLIKELY(isa<Ones>(expr))) {
    582         return builder->allOnes();
     582        return iBuilder->allOnes();
    583583    } else if (LLVM_UNLIKELY(isa<Zeroes>(expr))) {
    584         return builder->allZeroes();
     584        return iBuilder->allZeroes();
    585585    } else if (LLVM_UNLIKELY(isa<Integer>(expr))) {
    586586        return ConstantInt::get(cast<Integer>(expr)->getType(), cast<Integer>(expr)->value());
    587587    } else if (LLVM_UNLIKELY(isa<Operator>(expr))) {
    588588        const Operator * op = cast<Operator>(expr);
    589         Value * lh = compileExpression(builder, op->getLH());
    590         Value * rh = compileExpression(builder, op->getRH());
     589        Value * lh = compileExpression(iBuilder, op->getLH());
     590        Value * rh = compileExpression(iBuilder, op->getRH());
    591591        if (LLVM_UNLIKELY(lh->getType() != rh->getType())) {
    592592            std::string tmp;
     
    603603        switch (op->getClassTypeId()) {
    604604            case TypeId::Add:
    605                 return builder->CreateAdd(lh, rh);
     605                return iBuilder->CreateAdd(lh, rh);
    606606            case TypeId::Subtract:
    607                 return builder->CreateSub(lh, rh);
     607                return iBuilder->CreateSub(lh, rh);
    608608            case TypeId::LessThan:
    609                 return builder->CreateICmpSLT(lh, rh);
     609                return iBuilder->CreateICmpSLT(lh, rh);
    610610            case TypeId::LessThanEquals:
    611                 return builder->CreateICmpSLE(lh, rh);
     611                return iBuilder->CreateICmpSLE(lh, rh);
    612612            case TypeId::Equals:
    613                 return builder->CreateICmpEQ(lh, rh);
     613                return iBuilder->CreateICmpEQ(lh, rh);
    614614            case TypeId::GreaterThanEquals:
    615                 return builder->CreateICmpSGE(lh, rh);
     615                return iBuilder->CreateICmpSGE(lh, rh);
    616616            case TypeId::GreaterThan:
    617                 return builder->CreateICmpSGT(lh, rh);
     617                return iBuilder->CreateICmpSGT(lh, rh);
    618618            case TypeId::NotEquals:
    619                 return builder->CreateICmpNE(lh, rh);
     619                return iBuilder->CreateICmpNE(lh, rh);
    620620            default: break;
    621621        }
     
    638638    Value * value = f->second;
    639639    if (LLVM_UNLIKELY(isa<GetElementPtrInst>(value) && ensureLoaded)) {
    640         value = builder->CreateAlignedLoad(value, getPointerElementAlignment(value));
     640        value = iBuilder->CreateAlignedLoad(value, getPointerElementAlignment(value));
    641641    }
    642642    return value;
  • icGREP/icgrep-devel/icgrep/pablo/pablo_compiler.h

    r5435 r5440  
    2020namespace pablo { class Statement; }
    2121namespace pablo { class While; }
     22namespace kernel { class KernelBuilder; }
    2223
    2324namespace pablo {
     
    3738protected:
    3839
    39     void initializeKernelData(IDISA::IDISA_Builder * const builder);
     40    void initializeKernelData(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder);
    4041
    41     void compile(IDISA::IDISA_Builder * const builder);
     42    void compile(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder);
    4243
    4344private:
    4445
    45     void examineBlock(IDISA::IDISA_Builder * const builder, const PabloBlock * const block);
     46    void examineBlock(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const block);
    4647
    47     void compileBlock(IDISA::IDISA_Builder * const builder, const PabloBlock * const block);
     48    void compileBlock(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const block);
    4849
    49     void compileStatement(IDISA::IDISA_Builder * const builder, const Statement * stmt);
     50    void compileStatement(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const Statement * stmt);
    5051
    51     void compileIf(IDISA::IDISA_Builder * const builder, const If * ifStmt);
     52    void compileIf(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const If * ifStmt);
    5253
    53     void compileWhile(IDISA::IDISA_Builder * const builder, const While * whileStmt);
     54    void compileWhile(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const While * whileStmt);
    5455
    55     llvm::Value * compileExpression(IDISA::IDISA_Builder * const builder, const PabloAST * expr, const bool ensureLoaded = true) const;
     56    llvm::Value * compileExpression(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloAST * expr, const bool ensureLoaded = true) const;
    5657
    5758private:
  • icGREP/icgrep-devel/icgrep/pablo/pablo_kernel.cpp

    r5436 r5440  
    1212#include <pablo/pablo_toolchain.h>
    1313#include <kernels/kernel_builder.h>
    14 #include "llvm/Support/Debug.h"
     14#include <llvm/IR/Module.h>
     15// #include "llvm/Support/Debug.h"
    1516
    1617using namespace pablo;
     
    5354
    5455Var * PabloKernel::addInput(const std::string & name, Type * const type) {
    55     Var * param = new (mAllocator) Var(mSymbolTable->makeString(type->getContext(), name), type, mAllocator, Var::KernelInputParameter);
     56    Var * param = new (mAllocator) Var(makeName(name), type, mAllocator, Var::KernelInputParameter);
    5657    param->addUser(this);
    5758    mInputs.push_back(param);
     
    6970
    7071Var * PabloKernel::addOutput(const std::string & name, Type * const type) {
    71     Var * result = new (mAllocator) Var(mSymbolTable->makeString(type->getContext(), name), type, mAllocator, Var::KernelOutputParameter);
     72    Var * result = new (mAllocator) Var(makeName(name), type, mAllocator, Var::KernelOutputParameter);
    7273    result->addUser(this);
    7374    mOutputs.push_back(result);
     
    9394Zeroes * PabloKernel::getNullValue(Type * type) {
    9495    if (type == nullptr) {
    95         type = iBuilder->getStreamTy();
     96        type = mBuilder->getStreamTy();
    9697    }
    9798    for (PabloAST * constant : mConstants) {
     
    107108Ones * PabloKernel::getAllOnesValue(Type * type) {
    108109    if (type == nullptr) {
    109         type = iBuilder->getStreamTy();
     110        type = mBuilder->getStreamTy();
    110111    }
    111112    for (PabloAST * constant : mConstants) {
     
    119120}
    120121
    121 void PabloKernel::prepareKernel() {
     122void PabloKernel::prepareKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    122123    if (DebugOptionIsSet(DumpTrace)) {
    123124        setName(getName() + "_DumpTrace");
    124125    }
     126    mBuilder = iBuilder.get();
    125127    generatePabloMethod();
     128    mBuilder = nullptr;
    126129    pablo_function_passes(this);
    127130    mPabloCompiler->initializeKernelData(iBuilder);
    128     BlockOrientedKernel::prepareKernel();
    129 }
    130 
    131 void PabloKernel::generateDoBlockMethod() {
     131    BlockOrientedKernel::prepareKernel(iBuilder);
     132}
     133
     134void PabloKernel::generateDoBlockMethod(const std::unique_ptr<KernelBuilder> & iBuilder) {
     135    mBuilder = iBuilder.get();
    132136    mPabloCompiler->compile(iBuilder);
    133 }
    134 
    135 void PabloKernel::generateFinalBlockMethod(Value * const remainingBytes) {
     137    mBuilder = nullptr;
     138}
     139
     140void PabloKernel::generateFinalBlockMethod(const std::unique_ptr<KernelBuilder> & iBuilder, Value * const remainingBytes) {
    136141    // Standard Pablo convention for final block processing: set a bit marking
    137142    // the position just past EOF, as well as a mask marking all positions past EOF.
    138     setScalarField("EOFbit", iBuilder->bitblock_set_bit(remainingBytes));
    139     setScalarField("EOFmask", iBuilder->bitblock_mask_from(remainingBytes));
    140     CreateDoBlockMethodCall();
     143    iBuilder->setScalarField("EOFbit", iBuilder->bitblock_set_bit(remainingBytes));
     144    iBuilder->setScalarField("EOFmask", iBuilder->bitblock_mask_from(remainingBytes));
     145    CreateDoBlockMethodCall(iBuilder);
    141146}
    142147
    143148String * PabloKernel::makeName(const llvm::StringRef & prefix) const {
    144     return mSymbolTable->makeString(iBuilder->getContext(), prefix);
     149    return mSymbolTable->makeString(prefix);
    145150}
    146151
    147152Integer * PabloKernel::getInteger(const int64_t value) const {
    148     return mSymbolTable->getInteger(iBuilder->getContext(), value);
     153    return mSymbolTable->getInteger(value);
    149154}
    150155
     
    161166, PabloAST(PabloAST::ClassTypeId::Kernel, nullptr, mAllocator)
    162167, mPabloCompiler(new PabloCompiler(this))
    163 , mSymbolTable(new SymbolGenerator(mAllocator))
    164 , mEntryBlock(PabloBlock::Create(this)) {
     168, mSymbolTable(new SymbolGenerator(b->getContext(), mAllocator))
     169, mEntryBlock(PabloBlock::Create(this))
     170, mBuilder(nullptr)
     171//, mSizeTy(b->getSizeTy())
     172//, mSizeTy(b->getSizeTy())
     173{
    165174    prepareStreamSetNameMap();
    166175    for (const Binding & ss : mStreamSetInputs) {
    167         Var * param = new (mAllocator) Var(mSymbolTable->makeString(b->getContext(), ss.name), ss.type, mAllocator, Var::KernelInputParameter);
     176        Var * param = new (mAllocator) Var(makeName(ss.name), ss.type, mAllocator, Var::KernelInputParameter);
    168177        param->addUser(this);
    169178        mInputs.push_back(param);
     
    171180    }
    172181    for (const Binding & ss : mStreamSetOutputs) {
    173         Var * result = new (mAllocator) Var(mSymbolTable->makeString(b->getContext(), ss.name), ss.type, mAllocator, Var::KernelOutputParameter);
     182        Var * result = new (mAllocator) Var(makeName(ss.name), ss.type, mAllocator, Var::KernelOutputParameter);
    174183        result->addUser(this);
    175184        mOutputs.push_back(result);
     
    177186    }
    178187    for (const Binding & ss : mScalarOutputs) {
    179         Var * result = new (mAllocator) Var(mSymbolTable->makeString(b->getContext(), ss.name), ss.type, mAllocator, Var::KernelOutputParameter);
     188        Var * result = new (mAllocator) Var(makeName(ss.name), ss.type, mAllocator, Var::KernelOutputParameter);
    180189        result->addUser(this);
    181190        mOutputs.push_back(result);
  • icGREP/icgrep-devel/icgrep/pablo/pablo_kernel.h

    r5437 r5440  
    3232
    3333public:
     34
     35    using KernelBuilder = kernel::KernelBuilder;
    3436
    3537    using Allocator = SlabAllocator<PabloAST *>;
     
    121123    Integer * getInteger(const int64_t value) const;
    122124
     125    kernel::KernelBuilder * getBuilder() {
     126        return mBuilder;
     127    }
     128
    123129protected:
    124130
     
    136142    // so that the carry data requirements may be accommodated before
    137143    // finalizing the KernelStateType.
    138     void prepareKernel() final;
     144    void prepareKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) final;
    139145
    140     void generateDoBlockMethod() final;
     146    void generateDoBlockMethod(const std::unique_ptr<KernelBuilder> & iBuilder) final;
    141147
    142148    // The default method for Pablo final block processing sets the
    143149    // EOFmark bit and then calls the standard DoBlock function.
    144150    // This may be overridden for specialized processing.
    145     void generateFinalBlockMethod(llvm::Value * remainingBytes) final;
     151    void generateFinalBlockMethod(const std::unique_ptr<KernelBuilder> & iBuilder, llvm::Value * remainingBytes) final;
    146152
    147153private:
     
    151157    SymbolGenerator *               mSymbolTable;
    152158    PabloBlock *                    mEntryBlock;
     159
     160    kernel::KernelBuilder *         mBuilder;
     161
     162//    llvm::IntegerType * const       mSizeTy;
     163//    llvm::VectorType * const        mStreamSetTy;
    153164
    154165    std::vector<Var *>              mInputs;
  • icGREP/icgrep-devel/icgrep/pablo/symbol_generator.cpp

    r5435 r5440  
    1212namespace pablo {
    1313
    14 String * SymbolGenerator::makeString(llvm::LLVMContext & C, const llvm::StringRef prefix) noexcept {
     14String * SymbolGenerator::makeString(const llvm::StringRef prefix) noexcept {
    1515    auto f = mPrefixMap.find(prefix);
    1616    if (f == mPrefixMap.end()) {   
     
    2020        llvm::StringRef name(data, prefix.size());
    2121        mPrefixMap.insert(std::make_pair(name, 1));
    22         return new (mAllocator) String(llvm::IntegerType::getInt8PtrTy(C), name, mAllocator);
     22        return new (mAllocator) String(llvm::IntegerType::getInt8PtrTy(mContext), name, mAllocator);
    2323    } else { // this string already exists; make a new string using the given prefix
    2424
     
    4040        }
    4141        *p = '_';
    42         return makeString(C, llvm::StringRef(name, length));
     42        return makeString(llvm::StringRef(name, length));
    4343    }
    4444}
    4545
    46 Integer * SymbolGenerator::getInteger(llvm::LLVMContext & C, const IntTy value) noexcept {
     46Integer * SymbolGenerator::getInteger(const IntTy value) noexcept {
    4747    auto f = mIntegerMap.find(value);
    4848    Integer * result;
    4949    if (f == mIntegerMap.end()) {       
    50         result = new (mAllocator) Integer(value, llvm::IntegerType::getInt64Ty(C), mAllocator);
     50        result = new (mAllocator) Integer(value, llvm::IntegerType::getInt64Ty(mContext), mAllocator);
    5151        assert (result->value() == value);
    5252        mIntegerMap.emplace(value, result);
  • icGREP/icgrep-devel/icgrep/pablo/symbol_generator.h

    r5435 r5440  
    2525public:
    2626    using IntTy = int64_t;
    27     String * makeString(llvm::LLVMContext & C, const llvm::StringRef prefix) noexcept;
    28     Integer * getInteger(llvm::LLVMContext & C, const IntTy value) noexcept;
     27    String * makeString(const llvm::StringRef prefix) noexcept;
     28    Integer * getInteger(const IntTy value) noexcept;
    2929    ~SymbolGenerator() { }
    3030protected:
    31     SymbolGenerator(Allocator & allocator) : mAllocator(allocator) { }
     31    SymbolGenerator(llvm::LLVMContext & C, Allocator & allocator)
     32    : mContext(C)
     33    , mAllocator(allocator) {
     34
     35    }
    3236private:
     37    llvm::LLVMContext &                          mContext;
    3338    Allocator &                                  mAllocator;
    3439    llvm::StringMap<IntTy>                       mPrefixMap;
Note: See TracChangeset for help on using the changeset viewer.