Ignore:
Timestamp:
Jul 7, 2015, 8:07:25 PM (4 years ago)
Author:
cameron
Message:

Carry Manager system integrated into Pablo compiler

Location:
icGREP/icgrep-devel/icgrep
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp

    r4644 r4647  
    1212#include <carry_manager.h>
    1313#include <pabloAST.h>
     14#include <iostream>
    1415
    1516namespace pablo {
     
    4647Value * CarryManager::getCarryOpCarryIn(PabloBlock * blk, int localIndex) {
    4748    PabloBlockCarryData & cd = blk->carryData;
     49    if (cd.getWhileDepth() == 0) {
     50       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd.carryOpCarryDataOffset(localIndex)));
     51       mCarryInVector[cd.carryOpCarryDataOffset(localIndex)] = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
     52    }
    4853    return mCarryInVector[cd.carryOpCarryDataOffset(localIndex)];
    4954}
     
    5156Value * CarryManager::getUnitAdvanceCarryIn(PabloBlock * blk, int localIndex) {
    5257    PabloBlockCarryData & cd = blk->carryData;
     58    if (cd.getWhileDepth() == 0) {
     59       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd.unitAdvanceCarryDataOffset(localIndex)));
     60       mCarryInVector[cd.unitAdvanceCarryDataOffset(localIndex)] = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
     61    }
    5362    return mCarryInVector[cd.unitAdvanceCarryDataOffset(localIndex)];
    5463}
     
    5665Value * CarryManager::getShortAdvanceCarryIn(PabloBlock * blk, int localIndex, int shift_amount) {
    5766    PabloBlockCarryData & cd = blk->carryData;
     67    if (cd.getWhileDepth() == 0) {
     68       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd.shortAdvanceCarryDataOffset(localIndex)));
     69       mCarryInVector[cd.shortAdvanceCarryDataOffset(localIndex)] = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
     70    }
    5871    return mCarryInVector[cd.shortAdvanceCarryDataOffset(localIndex)];
    5972}
    6073
     74void CarryManager::setCarryOpCarryOut(PabloBlock * blk, unsigned idx, Value * carry_out) {
     75    PabloBlockCarryData & cd = blk->carryData;
     76    mCarryOutVector[cd.carryOpCarryDataOffset(idx)] = carry_out;
     77    if (cd.getWhileDepth() == 0) {
     78       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd.carryOpCarryDataOffset(idx)));
     79       mBuilder->CreateAlignedStore(carry_out, packPtr, BLOCK_SIZE/8);
     80    }
     81}
     82
     83void CarryManager::setUnitAdvanceCarryOut(PabloBlock * blk, unsigned idx, Value * carry_out) {
     84    PabloBlockCarryData & cd = blk->carryData;
     85    mCarryOutVector[cd.unitAdvanceCarryDataOffset(idx)] = carry_out;
     86    if (cd.getWhileDepth() == 0) {
     87       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd.unitAdvanceCarryDataOffset(idx)));
     88       mBuilder->CreateAlignedStore(carry_out, packPtr, BLOCK_SIZE/8);
     89    }
     90}
     91
     92void CarryManager::setShortAdvanceCarryOut(PabloBlock * blk, unsigned idx, int shift_amount, Value * carry_out) {
     93    PabloBlockCarryData & cd = blk->carryData;
     94    mCarryOutVector[cd.shortAdvanceCarryDataOffset(idx)] = carry_out;
     95    if (cd.getWhileDepth() == 0) {
     96       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd.shortAdvanceCarryDataOffset(idx)));
     97       mBuilder->CreateAlignedStore(carry_out, packPtr, BLOCK_SIZE/8);
     98    }
     99}
     100   
    61101/*
    62 static unsigned power2ceil (unsigned v) {
    63     unsigned ceil = 1;
    64     while (ceil < v) ceil *= 2;
    65     return ceil;
    66 }
    67 
    68 unsigned longAdvanceEntries(unsigned shift_amount) const {
    69     return (shift_amount + BLOCK_SIZE - 1)/BLOCK_SIZE;
    70 }
    71    
    72 unsigned longAdvanceBufferSize(unsigned shift_amount)  const {
    73     return power2ceil(longAdvanceEntries(shift_amount));
    74 }
    75 */
    76 
    77 Value * CarryManager::getLongAdvanceCarryIn(PabloBlock * blk, int localIndex, int shift_amount) {
    78     PabloBlockCarryData & cd = blk->carryData;
     102 static unsigned power2ceil (unsigned v) {
     103 unsigned ceil = 1;
     104 while (ceil < v) ceil *= 2;
     105 return ceil;
     106 }
     107 
     108 unsigned longAdvanceEntries(unsigned shift_amount) const {
     109 return (shift_amount + BLOCK_SIZE - 1)/BLOCK_SIZE;
     110 }
     111 
     112 unsigned longAdvanceBufferSize(unsigned shift_amount)  const {
     113 return power2ceil(longAdvanceEntries(shift_amount));
     114 }
     115 */
     116
     117Value * CarryManager::longAdvanceCarryInCarryOut(PabloBlock * blk, int localIndex, int shift_amount, Value * carry_out) {
     118    PabloBlockCarryData & cd = blk->carryData;
     119    Value * advBaseIndex = mBuilder->getInt64(cd.longAdvanceCarryDataOffset(localIndex));
     120    if (shift_amount <= BLOCK_SIZE) {
     121        // special case using a single buffer entry and the carry_out value.
     122        Value * advanceDataPtr = mBuilder->CreateGEP(mCarryDataPtr, advBaseIndex);
     123        Value * carry_block0 = mBuilder->CreateAlignedLoad(advanceDataPtr, BLOCK_SIZE/8);
     124        mBuilder->CreateAlignedStore(carry_out, advanceDataPtr, BLOCK_SIZE/8);
     125        /* Very special case - no combine */
     126        if (shift_amount == BLOCK_SIZE) return carry_block0;
     127        Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - shift_amount);
     128        Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_out, mBuilder->getIntNTy(BLOCK_SIZE)), shift_amount);
     129        return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
     130    }
     131    // We need a buffer of at least two elements for storing the advance data.
    79132    const unsigned block_shift = shift_amount % BLOCK_SIZE;
    80133    const unsigned advanceEntries = cd.longAdvanceEntries(shift_amount);
    81134    const unsigned bufsize = cd.longAdvanceBufferSize(shift_amount);
    82135    Value * indexMask = mBuilder->getInt64(bufsize - 1);  // A mask to implement circular buffer indexing
    83     Value * advBaseIndex = mBuilder->getInt64(cd.longAdvanceCarryDataOffset(localIndex));
    84136    Value * loadIndex0 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries)), indexMask), advBaseIndex);
     137    Value * storeIndex = mBuilder->CreateAdd(mBuilder->CreateAnd(mBlockNo, indexMask), advBaseIndex);
    85138    Value * carry_block0 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex0), BLOCK_SIZE/8);
    86139    // If the long advance is an exact multiple of BLOCK_SIZE, we simply return the oldest
    87     // block in the long advance carry data area.
    88     if (block_shift == 0) return carry_block0;
    89     Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - block_shift);
    90     if (advanceEntries == 1) {
    91         return mBuilder->CreateBitCast(block0_shr, mBitBlockType);
     140    // block in the long advance carry data area. 
     141    if (block_shift == 0) {
     142        mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
     143        return carry_block0;
    92144    }
    93145    // Otherwise we need to combine data from the two oldest blocks.
    94146    Value * loadIndex1 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries-1)), indexMask), advBaseIndex);
    95147    Value * carry_block1 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex1), BLOCK_SIZE/8);
     148    Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - block_shift);
    96149    Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_block1, mBuilder->getIntNTy(BLOCK_SIZE)), block_shift);
     150    mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
    97151    return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
    98152}
    99 
    100 void CarryManager::setCarryOpCarryOut(PabloBlock * blk, unsigned idx, Value * carry_out) {
    101     PabloBlockCarryData & cd = blk->carryData;
    102     mCarryOutVector[cd.carryOpCarryDataOffset(idx)] = carry_out;
    103 }
    104 
    105 void CarryManager::setUnitAdvanceCarryOut(PabloBlock * blk, unsigned idx, Value * carry_out) {
    106     PabloBlockCarryData & cd = blk->carryData;
    107     mCarryOutVector[cd.unitAdvanceCarryDataOffset(idx)] = carry_out;
    108 }
    109 
    110 void CarryManager::setShortAdvanceCarryOut(PabloBlock * blk, unsigned idx, int shift_amount, Value * carry_out) {
    111     PabloBlockCarryData & cd = blk->carryData;
    112     mCarryOutVector[cd.shortAdvanceCarryDataOffset(idx)] = carry_out;
    113 }
    114 
    115 void CarryManager::setLongAdvanceCarryOut(PabloBlock * blk, unsigned idx, int shift_amount, Value * carry_out){
    116     PabloBlockCarryData & cd = blk->carryData;
    117     const unsigned bufsize = cd.longAdvanceBufferSize(shift_amount);
    118     Value * indexMask = mBuilder->getInt64(bufsize - 1);  // Mask to implement circular buffer indexing
    119     Value * advBaseIndex = mBuilder->getInt64(cd.longAdvanceCarryDataOffset(idx));
    120     Value * storeIndex = mBuilder->CreateAdd(mBuilder->CreateAnd(mBlockNo, indexMask), advBaseIndex);
    121     mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
    122 }
    123 
     153   
    124154
    125155/* Methods for getting and setting carry summary values */
     
    180210                carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[baseCarryDataIdx+i]);
    181211            }
    182        
     212        }
    183213        for (Statement * stmt : blk) {
    184214            if (If * innerIf = dyn_cast<If>(stmt)) {
     
    202232    Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(carrySummaryIndex));
    203233    mBuilder->CreateAlignedStore(carry_summary, packPtr, BLOCK_SIZE/8);
    204     }
    205234}
    206235
    207236void CarryManager::ensureCarriesLoadedLocal(PabloBlock & blk) {
     237#if 0
    208238    const PabloBlockCarryData & cd = blk.carryData;
    209239    const unsigned baseCarryDataIdx = cd.getBlockCarryDataIndex();
     
    216246        mCarryInVector[i] = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i)), BLOCK_SIZE/8, false);
    217247    }
     248#endif
    218249}
    219250
    220251void CarryManager::ensureCarriesStoredLocal(PabloBlock & blk) {
     252#if 0
    221253    const PabloBlockCarryData & cd = blk.carryData;
    222254    const unsigned baseCarryDataIdx = cd.getBlockCarryDataIndex();
     
    234266        mBuilder->CreateAlignedStore(mCarryOutVector[carrySummaryIndex], summaryPtr, BLOCK_SIZE/8, false);
    235267    }
     268#endif
    236269}
    237270
     
    241274    const unsigned baseCarryDataIdx = cd.getBlockCarryDataIndex();
    242275    const unsigned totalCarryDataSize = cd.getTotalCarryDataSize();
    243     if (cd.getWhileDepth() == 0) {
     276    if (cd.getWhileDepth() == 1) {
    244277        for (auto i = baseCarryDataIdx; i < baseCarryDataIdx + totalCarryDataSize; ++i) {
    245278            mCarryInVector[i] = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i)), BLOCK_SIZE/8, false);
     
    285318    const unsigned baseCarryDataIdx = cd.getBlockCarryDataIndex();
    286319    const unsigned totalCarryDataSize = cd.getTotalCarryDataSize();
    287     if (cd.getWhileDepth() == 0) {
     320    if (cd.getWhileDepth() == 1) {
    288321        for (auto i = baseCarryDataIdx; i < baseCarryDataIdx + totalCarryDataSize; ++i) {
    289322            Value * storePtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i));
  • icGREP/icgrep-devel/icgrep/pablo/carry_manager.h

    r4644 r4647  
    4545
    4646    Value * getShortAdvanceCarryIn(PabloBlock * blk, int localIndex, int shift_amount);
    47  
    48     Value * getLongAdvanceCarryIn(PabloBlock * blk, int localIndex, int shift_amount);
    49    
     47     
    5048    void setCarryOpCarryOut(PabloBlock * blk, unsigned idx, Value * carry_out);
    5149
     
    5452    void setShortAdvanceCarryOut(PabloBlock * blk, unsigned idx, int shift_amount, Value * carry_out);
    5553   
    56     void setLongAdvanceCarryOut(PabloBlock * blk, unsigned idx, int shift_amount, Value * carry_out);
     54    Value * longAdvanceCarryInCarryOut(PabloBlock * blk, int localIndex, int shift_amount, Value * carry_out);
    5755 
    5856    /* Methods for getting and setting carry summary values for If statements */
  • icGREP/icgrep-devel/icgrep/pablo/pablo_compiler.cpp

    r4643 r4647  
    88#include <pablo/codegenstate.h>
    99#include <pablo/carry_data.h>
     10#include <pablo/carry_manager.h>
    1011#include <pablo/printer_pablos.h>
    1112#include <cc/cc_namemap.hpp>
     
    7273#endif
    7374, mBuilder(&LLVM_Builder)
     75, mCarryManager(nullptr)
    7476, mExecutionEngine(nullptr)
    7577, mBitBlockType(VectorType::get(IntegerType::get(mMod->getContext(), 64), BLOCK_SIZE / 64))
    7678, mBasisBitsInputPtr(nullptr)
    7779, mCarryDataPtr(nullptr)
    78 , mBlockNo(nullptr)
    7980, mWhileDepth(0)
    8081, mIfDepth(0)
     
    120121    mIfDepth = 0;
    121122    mMaxWhileDepth = 0;
    122     // Get the total number of carry entries; add 1 extra element for the block number.
    123     unsigned totalCarryDataSize = pb.carryData.enumerate(pb) + 1;
    124     Examine(pb);
    125     mCarryInVector.resize(totalCarryDataSize);
    126     mCarryOutVector.resize(totalCarryDataSize);
    127     mCarryDataSummaryIdx.resize(totalCarryDataSize);
     123    mCarryManager = new CarryManager(mBuilder, mBitBlockType, mZeroInitializer, mOneInitializer);
     124   
    128125    std::string errMessage;
    129126#ifdef USE_LLVM_3_5
     
    144141    DeclareFunctions();
    145142
     143    Examine(pb);
    146144    DeclareCallFunctions();
    147145
     
    167165        mMarkerMap.insert(std::make_pair(mBasisBits[i], basisBit));
    168166    }
    169    
    170     // The block number is a 64-bit integer at the end of the carry data area.
    171     Value * blockNoPtr = mBuilder->CreateBitCast(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(totalCarryDataSize - 1)), Type::getInt64PtrTy(mBuilder->getContext()));
    172     mBlockNo = mBuilder->CreateLoad(blockNoPtr);
     167       
     168    unsigned totalCarryDataSize = mCarryManager->initialize(&pb, mCarryDataPtr);
     169   
    173170    //Generate the IR instructions for the function.
    174171    compileBlock(pb);
    175172   
    176     mBuilder->CreateStore(mBuilder->CreateAdd(mBlockNo, mBuilder->getInt64(1)), blockNoPtr);
     173    mCarryManager->generateBlockNoIncrement();
    177174
    178175    if (DumpTrace || TraceNext) {
    179         genPrintRegister("blockNo", genCarryDataLoad(totalCarryDataSize - 1));
    180     }
     176        genPrintRegister("mBlockNo", mBuilder->CreateAlignedLoad(mBuilder->CreateBitCast(mCarryManager->getBlockNoPtr(), PointerType::get(mBitBlockType, 0)), BLOCK_SIZE/8, false));
     177    }
     178   
    181179    if (LLVM_UNLIKELY(mWhileDepth != 0)) {
    182180        throw std::runtime_error("Non-zero nesting depth error (" + std::to_string(mWhileDepth) + ")");
     
    416414
    417415void PabloCompiler::compileBlock(PabloBlock & block) {
    418     mPabloBlock = &block;
     416    mCarryManager->ensureCarriesLoadedLocal(block);
     417    mPabloBlock = & block;
    419418    for (const Statement * statement : block) {
    420419        compileStatement(statement);
    421420    }
    422421    mPabloBlock = block.getParent();
     422    mCarryManager->ensureCarriesStoredLocal(block);
    423423}
    424424
    425425
    426426void PabloCompiler::compileIf(const If * ifStatement) {       
    427         //
    428         //  The If-ElseZero stmt:
    429         //  if <predicate:expr> then <body:stmt>* elsezero <defined:var>* endif
    430         //  If the value of the predicate is nonzero, then determine the values of variables
    431         //  <var>* by executing the given statements.  Otherwise, the value of the
    432         //  variables are all zero.  Requirements: (a) no variable that is defined within
    433         //  the body of the if may be accessed outside unless it is explicitly
    434         //  listed in the variable list, (b) every variable in the defined list receives
    435         //  a value within the body, and (c) the logical consequence of executing
    436         //  the statements in the event that the predicate is zero is that the
    437         //  values of all defined variables indeed work out to be 0.
    438         //
    439         //  Simple Implementation with Phi nodes:  a phi node in the if exit block
    440         //  is inserted for each variable in the defined variable list.  It receives
    441         //  a zero value from the ifentry block and the defined value from the if
    442         //  body.
    443         //
    444         BasicBlock * ifEntryBlock = mBuilder->GetInsertBlock();
    445         BasicBlock * ifBodyBlock = BasicBlock::Create(mMod->getContext(), "if.body", mFunction, 0);
    446         BasicBlock * ifEndBlock = BasicBlock::Create(mMod->getContext(), "if.end", mFunction, 0);
    447        
    448         const PabloBlockCarryData & cd = ifStatement -> getBody().carryData;
    449    
    450         const unsigned baseCarryDataIdx = cd.getBlockCarryDataIndex();
    451         const unsigned carrySummaryIndex = cd.summaryCarryDataIndex();
    452        
    453         Value* if_test_value = compileExpression(ifStatement->getCondition());
    454         if (cd.blockHasCarries()) {
    455             // load the summary variable
    456             Value* last_if_pending_data = genCarryDataLoad(carrySummaryIndex);
    457             if_test_value = mBuilder->CreateOr(if_test_value, last_if_pending_data);
    458         }
    459         mBuilder->CreateCondBr(genBitBlockAny(if_test_value), ifEndBlock, ifBodyBlock);
    460 
    461         // Entry processing is complete, now handle the body of the if.
    462         mBuilder->SetInsertPoint(ifBodyBlock);
    463         compileBlock(ifStatement -> getBody());
    464    
    465         if (cd.explicitSummaryRequired()) {
    466             // If there was only one carry entry, then it also serves as the summary variable.
    467             // Otherwise, we need to combine entries to compute the summary.
    468             Value * carry_summary = mZeroInitializer;
    469             for (int c = baseCarryDataIdx; c < carrySummaryIndex; c++) {
    470                 int s = mCarryDataSummaryIdx[c];
    471                 if (s == -1) {
    472                     Value* carryq_value = mCarryOutVector[c];
    473                     if (carry_summary == mZeroInitializer) {
    474                         carry_summary = carryq_value;
    475                     }
    476                     else {
    477                         carry_summary = mBuilder->CreateOr(carry_summary, carryq_value);
    478                     }
    479                     mCarryDataSummaryIdx[c] = carrySummaryIndex;
    480                 }
    481             }
    482             genCarryDataStore(carry_summary, carrySummaryIndex);
    483         }
    484         BasicBlock * ifBodyFinalBlock = mBuilder->GetInsertBlock();
    485         mBuilder->CreateBr(ifEndBlock);
    486         //End Block
    487         mBuilder->SetInsertPoint(ifEndBlock);
    488         for (const PabloAST * node : ifStatement->getDefined()) {
    489             const Assign * assign = cast<Assign>(node);
    490             PHINode * phi = mBuilder->CreatePHI(mBitBlockType, 2, assign->getName()->value());
    491             auto f = mMarkerMap.find(assign);
    492             assert (f != mMarkerMap.end());
    493             phi->addIncoming(mZeroInitializer, ifEntryBlock);
    494             phi->addIncoming(f->second, ifBodyFinalBlock);
    495             mMarkerMap[assign] = phi;
    496         }
    497         // Create the phi Node for the summary variable, if needed.
    498         if (cd.summaryNeededInParentBlock()) {
    499             PHINode * summary_phi = mBuilder->CreatePHI(mBitBlockType, 2, "summary");
    500             summary_phi->addIncoming(mZeroInitializer, ifEntryBlock);
    501             summary_phi->addIncoming(mCarryOutVector[carrySummaryIndex], ifBodyFinalBlock);
    502             mCarryOutVector[carrySummaryIndex] = summary_phi;
    503         }
    504        
    505 }
    506 
    507 //#define SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
    508 
    509 #define LOAD_WHILE_CARRIES \
    510         if (mWhileDepth == 0) { \
    511             for (auto i = baseCarryDataIdx; i < baseCarryDataIdx + carryDataSize; ++i) { \
    512                 mCarryInVector[i] = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i)), BLOCK_SIZE/8, false); \
    513             } \
    514         }
    515 
    516 #define INITIALIZE_CARRY_IN_PHIS  \
    517         std::vector<PHINode *> carryInPhis(carryDataSize);  \
    518         for (unsigned index = 0; index < carryDataSize; ++index) {  \
    519             PHINode * phi_in = mBuilder->CreatePHI(mBitBlockType, 2); \
    520             phi_in->addIncoming(mCarryInVector[baseCarryDataIdx + index], whileEntryBlock); \
    521             carryInPhis[index] = phi_in; \
    522         }
    523 
    524 #define INITIALIZE_CARRY_OUT_ACCUMULATOR_PHIS \
    525         std::vector<PHINode *> carryOutAccumPhis(carryDataSize); \
    526         for (unsigned index = 0; index < carryDataSize; ++index) { \
    527             PHINode * phi_out = mBuilder->CreatePHI(mBitBlockType, 2); \
    528             phi_out->addIncoming(mZeroInitializer, whileEntryBlock); \
    529             carryOutAccumPhis[index] = phi_out; \
    530             mCarryOutVector[baseCarryDataIdx + index] = mZeroInitializer; \
    531         }
    532 
    533 #define EXTEND_CARRY_IN_PHIS_TO_ZERO_FROM_WHILE_BODY_FINAL_BLOCK \
    534         for (unsigned index = 0; index < carryDataSize; ++index) { \
    535             carryInPhis[index]->addIncoming(mZeroInitializer, whileBodyFinalBlock); \
    536         }
    537 
    538 #define EXTEND_CARRY_OUT_ACCUMULATOR_PHIS_TO_OR_COMBINE_CARRY_OUT \
    539         for (unsigned index = 0; index < carryDataSize; ++index) { \
    540             PHINode * phi = carryOutAccumPhis[index]; \
    541             Value * carryOut = mBuilder->CreateOr(phi, mCarryOutVector[baseCarryDataIdx + index]); \
    542             phi->addIncoming(carryOut, whileBodyFinalBlock); \
    543             mCarryOutVector[baseCarryDataIdx + index] = carryOut; \
    544         }
    545 
    546 #define STORE_WHILE_CARRY_DATA \
    547         if (mWhileDepth == 0) { \
    548             for (unsigned index = baseCarryDataIdx; index < baseCarryDataIdx + carryDataSize; ++index) { \
    549                 mBuilder->CreateAlignedStore(mCarryOutVector[index], mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(index)), BLOCK_SIZE/8, false); \
    550             } \
    551         }
    552 
     427    //
     428    //  The If-ElseZero stmt:
     429    //  if <predicate:expr> then <body:stmt>* elsezero <defined:var>* endif
     430    //  If the value of the predicate is nonzero, then determine the values of variables
     431    //  <var>* by executing the given statements.  Otherwise, the value of the
     432    //  variables are all zero.  Requirements: (a) no variable that is defined within
     433    //  the body of the if may be accessed outside unless it is explicitly
     434    //  listed in the variable list, (b) every variable in the defined list receives
     435    //  a value within the body, and (c) the logical consequence of executing
     436    //  the statements in the event that the predicate is zero is that the
     437    //  values of all defined variables indeed work out to be 0.
     438    //
     439    //  Simple Implementation with Phi nodes:  a phi node in the if exit block
     440    //  is inserted for each variable in the defined variable list.  It receives
     441    //  a zero value from the ifentry block and the defined value from the if
     442    //  body.
     443    //
     444    BasicBlock * ifEntryBlock = mBuilder->GetInsertBlock();
     445    BasicBlock * ifBodyBlock = BasicBlock::Create(mMod->getContext(), "if.body", mFunction, 0);
     446    BasicBlock * ifEndBlock = BasicBlock::Create(mMod->getContext(), "if.end", mFunction, 0);
     447   
     448    PabloBlock & ifBody = ifStatement -> getBody();
     449   
     450    Value* if_test_value = compileExpression(ifStatement->getCondition());
     451    if (mCarryManager->blockHasCarries(ifBody)) {
     452        // load the summary variable
     453        Value* last_if_pending_data = mCarryManager->getCarrySummaryExpr(ifBody);
     454        if_test_value = mBuilder->CreateOr(if_test_value, last_if_pending_data);
     455    }
     456    mBuilder->CreateCondBr(genBitBlockAny(if_test_value), ifEndBlock, ifBodyBlock);
     457    // Entry processing is complete, now handle the body of the if.
     458    mBuilder->SetInsertPoint(ifBodyBlock);
     459   
     460    ++mIfDepth;
     461    compileBlock(ifBody);
     462    --mIfDepth;
     463    if (mCarryManager->blockHasCarries(ifBody)) {
     464        mCarryManager->generateCarryOutSummaryCode(ifBody);
     465    }
     466    BasicBlock * ifBodyFinalBlock = mBuilder->GetInsertBlock();
     467    mBuilder->CreateBr(ifEndBlock);
     468    //End Block
     469    mBuilder->SetInsertPoint(ifEndBlock);
     470    for (const PabloAST * node : ifStatement->getDefined()) {
     471        const Assign * assign = cast<Assign>(node);
     472        PHINode * phi = mBuilder->CreatePHI(mBitBlockType, 2, assign->getName()->value());
     473        auto f = mMarkerMap.find(assign);
     474        assert (f != mMarkerMap.end());
     475        phi->addIncoming(mZeroInitializer, ifEntryBlock);
     476        phi->addIncoming(f->second, ifBodyFinalBlock);
     477        mMarkerMap[assign] = phi;
     478    }
     479    // Create the phi Node for the summary variable, if needed.
     480    if (mCarryManager->summaryNeededInParentBlock(ifBody)) {
     481        mCarryManager->addSummaryPhi(ifBody, ifEntryBlock, ifBodyFinalBlock);
     482    }
     483}
    553484
    554485void PabloCompiler::compileWhile(const While * whileStatement) {
    555         //BasicBlock* whileEntryBlock = mBasicBlock;
    556         BasicBlock * whileEntryBlock = mBuilder->GetInsertBlock();
    557         BasicBlock * whileBodyBlock = BasicBlock::Create(mMod->getContext(), "while.body", mFunction, 0);
    558         BasicBlock * whileEndBlock = BasicBlock::Create(mMod->getContext(), "while.end", mFunction, 0);
    559    
    560    
    561         const PabloBlockCarryData & cd = whileStatement -> getBody().carryData;
    562         const unsigned baseCarryDataIdx = cd.getBlockCarryDataIndex();
    563         const unsigned carryDataSize = cd.getTotalCarryDataSize();
    564 
    565 
    566         LOAD_WHILE_CARRIES
    567 
    568         const auto & nextNodes = whileStatement->getVariants();
    569         std::vector<PHINode *> nextPhis;
    570         nextPhis.reserve(nextNodes.size());
    571    
    572         // On entry to the while structure, proceed to execute the first iteration
    573         // of the loop body unconditionally.   The while condition is tested at the end of
    574         // the loop.
    575 
    576         mBuilder->CreateBr(whileBodyBlock);
    577         mBuilder->SetInsertPoint(whileBodyBlock);
    578    
    579         //
    580         // There are 3 sets of Phi nodes for the while loop.
    581         // (1) Carry-ins: (a) incoming carry data first iterations, (b) zero thereafter
    582         // (2) Carry-out accumulators: (a) zero first iteration, (b) |= carry-out of each iteration
    583         // (3) Next nodes: (a) values set up before loop, (b) modified values calculated in loop.
    584 
    585 #ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
    586         INITIALIZE_CARRY_IN_PHIS
    587 #endif
    588 
    589         INITIALIZE_CARRY_OUT_ACCUMULATOR_PHIS
    590    
    591         // for any Next nodes in the loop body, initialize to (a) pre-loop value.
    592         for (const Next * n : nextNodes) {
    593             PHINode * phi = mBuilder->CreatePHI(mBitBlockType, 2, n->getName()->value());
    594             auto f = mMarkerMap.find(n->getInitial());
    595             assert (f != mMarkerMap.end());
    596             phi->addIncoming(f->second, whileEntryBlock);
    597             mMarkerMap[n->getInitial()] = phi;
    598             nextPhis.push_back(phi);
    599         }
    600 
    601         //
    602         // Now compile the loop body proper.  Carry-out accumulated values
    603         // and iterated values of Next nodes will be computed.
    604         ++mWhileDepth;
    605         compileBlock(whileStatement->getBody());
    606    
    607         BasicBlock * whileBodyFinalBlock = mBuilder->GetInsertBlock();
    608 
    609 #ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
    610         EXTEND_CARRY_IN_PHIS_TO_ZERO_FROM_WHILE_BODY_FINAL_BLOCK
    611 #endif
    612 
    613         EXTEND_CARRY_OUT_ACCUMULATOR_PHIS_TO_OR_COMBINE_CARRY_OUT
    614 
    615         // Terminate the while loop body with a conditional branch back.
    616         mBuilder->CreateCondBr(genBitBlockAny(compileExpression(whileStatement->getCondition())), whileEndBlock, whileBodyBlock);
    617 
    618         // and for any Next nodes in the loop body
    619         for (unsigned i = 0; i < nextNodes.size(); i++) {
    620             const Next * n = nextNodes[i];
    621             auto f = mMarkerMap.find(n->getExpr());
    622             if (LLVM_UNLIKELY(f == mMarkerMap.end())) {
    623                 throw std::runtime_error("Next node expression was not compiled!");
    624             }
    625             nextPhis[i]->addIncoming(f->second, whileBodyFinalBlock);
    626         }
    627 
    628         // EXIT BLOCK
    629         mBuilder->SetInsertPoint(whileEndBlock);
    630         --mWhileDepth;
    631 
    632         STORE_WHILE_CARRY_DATA
     486
     487    PabloBlock & whileBody = whileStatement -> getBody();
     488   
     489    BasicBlock * whileEntryBlock = mBuilder->GetInsertBlock();
     490    BasicBlock * whileBodyBlock = BasicBlock::Create(mMod->getContext(), "while.body", mFunction, 0);
     491    BasicBlock * whileEndBlock = BasicBlock::Create(mMod->getContext(), "while.end", mFunction, 0);
     492
     493    mCarryManager->ensureCarriesLoadedRecursive(whileBody);
     494
     495    const auto & nextNodes = whileStatement->getVariants();
     496    std::vector<PHINode *> nextPhis;
     497    nextPhis.reserve(nextNodes.size());
     498
     499    // On entry to the while structure, proceed to execute the first iteration
     500    // of the loop body unconditionally.   The while condition is tested at the end of
     501    // the loop.
     502
     503    mBuilder->CreateBr(whileBodyBlock);
     504    mBuilder->SetInsertPoint(whileBodyBlock);
     505
     506    //
     507    // There are 3 sets of Phi nodes for the while loop.
     508    // (1) Carry-ins: (a) incoming carry data first iterations, (b) zero thereafter
     509    // (2) Carry-out accumulators: (a) zero first iteration, (b) |= carry-out of each iteration
     510    // (3) Next nodes: (a) values set up before loop, (b) modified values calculated in loop.
     511
     512    mCarryManager->initializeCarryDataPhisAtWhileEntry(whileBody, whileEntryBlock);
     513
     514    // for any Next nodes in the loop body, initialize to (a) pre-loop value.
     515    for (const Next * n : nextNodes) {
     516        PHINode * phi = mBuilder->CreatePHI(mBitBlockType, 2, n->getName()->value());
     517        auto f = mMarkerMap.find(n->getInitial());
     518        assert (f != mMarkerMap.end());
     519        phi->addIncoming(f->second, whileEntryBlock);
     520        mMarkerMap[n->getInitial()] = phi;
     521        nextPhis.push_back(phi);
     522    }
     523
     524    //
     525    // Now compile the loop body proper.  Carry-out accumulated values
     526    // and iterated values of Next nodes will be computed.
     527    ++mWhileDepth;
     528    compileBlock(whileBody);
     529
     530    BasicBlock * whileBodyFinalBlock = mBuilder->GetInsertBlock();
     531
     532    mCarryManager->extendCarryDataPhisAtWhileBodyFinalBlock(whileBody, whileBodyFinalBlock);
     533
     534    // Terminate the while loop body with a conditional branch back.
     535    mBuilder->CreateCondBr(genBitBlockAny(compileExpression(whileStatement->getCondition())), whileEndBlock, whileBodyBlock);
     536
     537    // and for any Next nodes in the loop body
     538    for (unsigned i = 0; i < nextNodes.size(); i++) {
     539        const Next * n = nextNodes[i];
     540        auto f = mMarkerMap.find(n->getExpr());
     541        if (LLVM_UNLIKELY(f == mMarkerMap.end())) {
     542            throw std::runtime_error("Next node expression was not compiled!");
     543        }
     544        nextPhis[i]->addIncoming(f->second, whileBodyFinalBlock);
     545    }
     546
     547    mBuilder->SetInsertPoint(whileEndBlock);
     548    --mWhileDepth;
     549
     550    mCarryManager->ensureCarriesStoredRecursive(whileBody);
    633551}
    634552
     
    638556    if (const Assign * assign = dyn_cast<const Assign>(stmt)) {
    639557        expr = compileExpression(assign->getExpr());
    640         if (DumpTrace) {
    641             genPrintRegister(assign->getName()->to_string(), expr);
    642         }
    643558        if (LLVM_UNLIKELY(assign->isOutputAssignment())) {
    644559            SetOutputValue(expr, assign->getOutputIndex());
     
    672587    else if (const And * pablo_and = dyn_cast<And>(stmt)) {
    673588        expr = mBuilder->CreateAnd(compileExpression(pablo_and->getExpr1()), compileExpression(pablo_and->getExpr2()), "and");
    674         if (DumpTrace) {
    675             genPrintRegister(stmt->getName()->to_string(), expr);
    676         }
    677589    }
    678590    else if (const Or * pablo_or = dyn_cast<Or>(stmt)) {
    679591        expr = mBuilder->CreateOr(compileExpression(pablo_or->getExpr1()), compileExpression(pablo_or->getExpr2()), "or");
    680         if (DumpTrace) {
    681             genPrintRegister(stmt->getName()->to_string(), expr);
    682         }
    683592    }
    684593    else if (const Xor * pablo_xor = dyn_cast<Xor>(stmt)) {
     
    690599        Value* ifFalse = mBuilder->CreateAnd(genNot(ifMask), compileExpression(sel->getFalseExpr()));
    691600        expr = mBuilder->CreateOr(ifTrue, ifFalse);
    692         if (DumpTrace) {
    693             genPrintRegister(stmt->getName()->to_string(), expr);
    694         }
    695601    }
    696602    else if (const Not * pablo_not = dyn_cast<Not>(stmt)) {
    697603        expr = genNot(compileExpression(pablo_not->getExpr()));
    698         if (DumpTrace) {
    699             genPrintRegister(stmt->getName()->to_string(), expr);
    700         }
    701604    }
    702605    else if (const Advance * adv = dyn_cast<Advance>(stmt)) {
    703         Value* value = compileExpression(adv->getExpr());
     606        Value* strm_value = compileExpression(adv->getExpr());
    704607        int shift = adv->getAdvanceAmount();
    705608        unsigned advance_index = adv->getLocalAdvanceIndex();
    706         expr = genAdvanceWithCarry(value, shift, advance_index);
    707         if (DumpTrace) {
    708             genPrintRegister(stmt->getName()->to_string(), expr);
     609        if (shift == 1) {
     610            expr = genUnitAdvanceWithCarry(strm_value, advance_index);
     611        }
     612        else if (shift < LongAdvanceBase) {
     613            expr = genShortAdvanceWithCarry(strm_value, advance_index, shift);
     614        }
     615        else {
     616            expr = genLongAdvanceWithCarry(strm_value, advance_index, shift);
    709617        }
    710618    }
     
    715623        unsigned carry_index = mstar->getLocalCarryIndex();
    716624        expr = mBuilder->CreateOr(mBuilder->CreateXor(genAddWithCarry(marker_and_cc, cc, carry_index), cc), marker, "matchstar");
    717         if (DumpTrace) {
    718             genPrintRegister(stmt->getName()->to_string(), expr);
    719         }
    720625    }
    721626    else if (const ScanThru * sthru = dyn_cast<ScanThru>(stmt)) {
     
    724629        unsigned carry_index = sthru->getLocalCarryIndex();
    725630        expr = mBuilder->CreateAnd(genAddWithCarry(marker_expr, cc_expr, carry_index), genNot(cc_expr), "scanthru");
    726         if (DumpTrace) {
    727             genPrintRegister(stmt->getName()->to_string(), expr);
    728         }
    729631    }
    730632    else {
     
    734636    }
    735637    mMarkerMap[stmt] = expr;
     638    if (DumpTrace) {
     639        genPrintRegister(stmt->getName()->to_string(), expr);
     640    }
     641   
    736642}
    737643
     
    809715
    810716Value* PabloCompiler::genAddWithCarry(Value* e1, Value* e2, unsigned localIndex) {
    811     const PabloBlockCarryData & cd = mPabloBlock->carryData;
    812     const unsigned carryIdx = cd.carryOpCarryDataOffset(localIndex);
    813     Value* carryq_value = genCarryDataLoad(carryIdx);
     717    Value * carryq_value = mCarryManager->getCarryOpCarryIn(mPabloBlock, localIndex);
    814718#ifdef USE_TWO_UADD_OVERFLOW
    815719    //This is the ideal implementation, which uses two uadd.with.overflow
     
    868772#endif //USE_TWO_UADD_OVERFLOW
    869773
    870     genCarryDataStore(carry_out, carryIdx);
     774    mCarryManager->setCarryOpCarryOut(mPabloBlock, localIndex, carry_out);
    871775    return sum;
    872 }
    873 //#define CARRY_DEBUG
    874 Value* PabloCompiler::genCarryDataLoad(const unsigned index) {
    875     assert (index < mCarryInVector.size());
    876     if (mWhileDepth == 0) {
    877         mCarryInVector[index] = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(index)), BLOCK_SIZE/8, false);
    878     }
    879 #ifdef CARRY_DEBUG
    880     std::cerr << "genCarryDataLoad " << index << std::endl;
    881     genPrintRegister("carry_in_" + std::to_string(index), mCarryInVector[index]);
    882 #endif
    883     return mCarryInVector[index];
    884 }
    885 
    886 void PabloCompiler::genCarryDataStore(Value* carryOut, const unsigned index ) {
    887     assert (carryOut);
    888     assert (index < mCarryOutVector.size());
    889     if (mWhileDepth == 0) {
    890         mBuilder->CreateAlignedStore(carryOut, mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(index)), BLOCK_SIZE/8, false);
    891     }
    892     mCarryDataSummaryIdx[index] = -1;
    893     mCarryOutVector[index] = carryOut;
    894 #ifdef CARRY_DEBUG
    895     std::cerr << "genCarryDataStore " << index << std::endl;
    896     genPrintRegister("carry_out_" + std::to_string(index), mCarryOutVector[index]);
    897 #endif
    898     //std::cerr << "mCarryOutVector[" << index << "]]\n";
    899776}
    900777
     
    920797}
    921798
    922 Value* PabloCompiler::genAdvanceWithCarry(Value* strm_value, int shift_amount, unsigned localIndex) {
    923     if (shift_amount >= LongAdvanceBase) {
    924         return genLongAdvanceWithCarry(strm_value, shift_amount, localIndex);
    925     }
    926     else if (shift_amount == 1) {
    927         return genUnitAdvanceWithCarry(strm_value, localIndex);
    928     }
    929     const PabloBlockCarryData & cd = mPabloBlock->carryData;
    930     const auto advanceIndex = cd.shortAdvanceCarryDataOffset(localIndex);
     799Value* PabloCompiler::genUnitAdvanceWithCarry(Value* strm_value, unsigned localIndex) {
     800    Value * carry_in = mCarryManager->getUnitAdvanceCarryIn(mPabloBlock, localIndex);
    931801    Value* result_value;
    932802   
    933     if (shift_amount == 0) {
    934         result_value = genCarryDataLoad(advanceIndex);
    935     }
    936     else {
    937         Value* advanceq_longint = mBuilder->CreateBitCast(genCarryDataLoad(advanceIndex), mBuilder->getIntNTy(BLOCK_SIZE));
    938         Value* strm_longint = mBuilder->CreateBitCast(strm_value, mBuilder->getIntNTy(BLOCK_SIZE));
    939         Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, shift_amount), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - shift_amount), "advance");
    940         result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
    941     }
    942     genCarryDataStore(strm_value, advanceIndex);
    943     return result_value;
    944 }
    945                    
    946 Value* PabloCompiler::genUnitAdvanceWithCarry(Value* strm_value, unsigned localIndex) {
    947     const PabloBlockCarryData & cd = mPabloBlock->carryData;
    948     const auto advanceIndex = cd.unitAdvanceCarryDataOffset(localIndex);
    949     Value* result_value;
    950    
    951803#if (BLOCK_SIZE == 128) && !defined(USE_LONG_INTEGER_SHIFT)
    952     Value* advanceq_value = genShiftHighbitToLow(BLOCK_SIZE, genCarryDataLoad(advanceIndex));
     804    Value* advanceq_value = genShiftHighbitToLow(BLOCK_SIZE, carry_in);
    953805    Value* srli_1_value = mBuilder->CreateLShr(strm_value, 63);
    954806    Value* packed_shuffle;
     
    963815    result_value = mBuilder->CreateOr(shl_value, packed_shuffle, "advance");
    964816#else
    965     Value* advanceq_longint = mBuilder->CreateBitCast(genCarryDataLoad(advanceIndex), mBuilder->getIntNTy(BLOCK_SIZE));
     817    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
    966818    Value* strm_longint = mBuilder->CreateBitCast(strm_value, mBuilder->getIntNTy(BLOCK_SIZE));
    967819    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - 1), "advance");
     
    969821   
    970822#endif
    971     genCarryDataStore(strm_value, advanceIndex);
     823    mCarryManager->setUnitAdvanceCarryOut(mPabloBlock, localIndex, strm_value);
    972824    return result_value;
    973825}
    974826                   
    975 //
    976 // Generate code for long advances >= LongAdvanceBase
    977 //
    978 Value* PabloCompiler::genLongAdvanceWithCarry(Value* strm_value, int shift_amount, unsigned localIndex) {
    979     const PabloBlockCarryData & cd = mPabloBlock->carryData;
    980     const unsigned block_shift = shift_amount % BLOCK_SIZE;
    981     const unsigned advanceEntries = cd.longAdvanceEntries(shift_amount);
    982     const unsigned bufsize = cd.longAdvanceBufferSize(shift_amount);
    983     //std::cerr << "shift_amount = " << shift_amount << " bufsize = " << bufsize << std::endl;
    984     Value * indexMask = mBuilder->getInt64(bufsize - 1);  // A mask to implement circular buffer indexing
    985     Value * advBaseIndex = mBuilder->getInt64(cd.longAdvanceCarryDataOffset(localIndex));
    986     Value * storeIndex = mBuilder->CreateAdd(mBuilder->CreateAnd(mBlockNo, indexMask), advBaseIndex);
    987     Value * loadIndex = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries)), indexMask), advBaseIndex);
    988     Value * storePtr = mBuilder->CreateGEP(mCarryDataPtr, storeIndex);
    989     Value * loadPtr = mBuilder->CreateGEP(mCarryDataPtr, loadIndex);
    990     Value* result_value;
    991 
    992     if (block_shift == 0) {
    993         result_value = mBuilder->CreateAlignedLoad(loadPtr, BLOCK_SIZE/8);
    994     }
    995     else if (advanceEntries == 1) {
    996         Value* advanceq_longint = mBuilder->CreateBitCast(mBuilder->CreateAlignedLoad(loadPtr, BLOCK_SIZE/8), mBuilder->getIntNTy(BLOCK_SIZE));
    997         Value* strm_longint = mBuilder->CreateBitCast(strm_value, mBuilder->getIntNTy(BLOCK_SIZE));
    998         Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, block_shift), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - block_shift), "advance");
    999         result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
    1000     }
    1001     else {
    1002         // The advance is based on the two oldest bit blocks in the advance buffer.
    1003         // The buffer is maintained as a circular buffer of size bufsize.
    1004         // Indexes within the buffer are computed by bitwise and with the indexMask.
    1005         Value * loadIndex2 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries-1)), indexMask), advBaseIndex);
    1006         Value * loadPtr2 = mBuilder->CreateGEP(mCarryDataPtr, loadIndex2);
    1007         Value* advanceq_longint = mBuilder->CreateBitCast(mBuilder->CreateAlignedLoad(loadPtr, BLOCK_SIZE/8), mBuilder->getIntNTy(BLOCK_SIZE));
    1008         //genPrintRegister("advanceq_longint", mBuilder->CreateBitCast(advanceq_longint, mBitBlockType));
    1009         Value* strm_longint = mBuilder->CreateBitCast(mBuilder->CreateAlignedLoad(loadPtr2, BLOCK_SIZE/8), mBuilder->getIntNTy(BLOCK_SIZE));
    1010         //genPrintRegister("strm_longint", mBuilder->CreateBitCast(strm_longint, mBitBlockType));
    1011         Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, block_shift), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - block_shift), "longadvance");
    1012         result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
    1013     }
    1014     mBuilder->CreateAlignedStore(strm_value, storePtr, BLOCK_SIZE/8);
     827Value* PabloCompiler::genShortAdvanceWithCarry(Value* strm_value, unsigned localIndex, int shift_amount) {
     828    Value * carry_in = mCarryManager->getShortAdvanceCarryIn(mPabloBlock, localIndex, shift_amount);
     829    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
     830    Value* strm_longint = mBuilder->CreateBitCast(strm_value, mBuilder->getIntNTy(BLOCK_SIZE));
     831    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, shift_amount), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - shift_amount), "advance");
     832    Value* result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
     833    mCarryManager->setShortAdvanceCarryOut(mPabloBlock, localIndex, shift_amount, strm_value);
    1015834    return result_value;
     835}
     836                   
     837Value* PabloCompiler::genLongAdvanceWithCarry(Value* strm_value, unsigned localIndex, int shift_amount) {
     838    return mCarryManager->longAdvanceCarryInCarryOut(mPabloBlock, localIndex, shift_amount, strm_value);
    1016839}
    1017840   
  • icGREP/icgrep-devel/icgrep/pablo/pablo_compiler.h

    r4643 r4647  
    2525#include <unordered_map>
    2626#include <pablo/pe_string.h>
     27#include <pablo/carry_manager.h>
    2728#include <llvm/ADT/Twine.h>
    2829#include <llvm/IR/IRBuilder.h>
     
    113114    void compileWhile(const While * whileStmt);
    114115    Value* compileExpression(const PabloAST * expr);
    115     Value* genCarryDataLoad(const unsigned index);
    116     void   genCarryDataStore(Value* carryOut, const unsigned index);
    117116    Value* genAddWithCarry(Value* e1, Value* e2, unsigned localIndex);
    118     Value* genAdvanceWithCarry(Value* e1, int shift_amount, unsigned localIndex);
    119117    Value* genUnitAdvanceWithCarry(Value* e1, unsigned localIndex);
    120     Value* genLongAdvanceWithCarry(Value* e1, int shift_amount, unsigned localIndex);
     118    Value* genShortAdvanceWithCarry(Value* e1, unsigned localIndex, int shift_amount);
     119    Value* genLongAdvanceWithCarry(Value* e1, unsigned localIndex, int shift_amount);
    121120    Value* genBitBlockAny(Value* test);
    122121    Value* genShiftHighbitToLow(unsigned FieldWidth, Value * op);
     
    138137    CarryQueueVector                    mCarryInVector;
    139138    CarryQueueVector                    mCarryOutVector;
    140     std::vector<int>                    mCarryDataSummaryIdx;
    141139
    142140    const std::vector<Var *> &          mBasisBits;
     
    148146#endif
    149147    IRBuilder <> *                      mBuilder;
     148    CarryManager *                      mCarryManager;
    150149    ExecutionEngine*                    mExecutionEngine;
    151150
     
    156155   
    157156    Value*                              mCarryDataPtr;
    158     Value*                              mBlockNo;
    159157    unsigned                            mWhileDepth;
    160158    unsigned                            mIfDepth;
  • icGREP/icgrep-devel/icgrep/re/re_compiler.cpp

    r4646 r4647  
    3838static cl::opt<bool> DisableUnicodeLineBreak("disable-unicode-linebreak", cl::init(false),
    3939                     cl::desc("disable Unicode line breaks - use LF only"), cl::cat(fREcompilationOptions));
    40 static cl::opt<bool> DisablePregeneratedUnicode("disable-pregenerated-unicode", cl::init(true),
     40static cl::opt<bool> DisablePregeneratedUnicode("disable-pregenerated-unicode", cl::init(false),
    4141                     cl::desc("disable use of pregenerated Unicode character class sets"), cl::cat(fREcompilationOptions));
    4242
Note: See TracChangeset for help on using the changeset viewer.