Ignore:
Timestamp:
Mar 26, 2015, 8:09:48 PM (4 years ago)
Author:
cameron
Message:

New carry data system -- first stage

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/pablo/pablo_compiler.cpp

    r4539 r4541  
    6565, mBitBlockType(VectorType::get(IntegerType::get(mMod->getContext(), 64), BLOCK_SIZE / 64))
    6666, mBasisBitsInputPtr(nullptr)
    67 , mCarryQueueIdx(0)
    6867, mCarryDataPtr(nullptr)
    6968, mNestingDepth(0)
    70 , mCarryQueueSize(0)
    71 , mAdvanceQueueIdx(0)
    72 , mAdvanceQueueSize(0)
    7369, mZeroInitializer(ConstantAggregateZero::get(mBitBlockType))
    7470, mOneInitializer(ConstantVector::getAllOnesValue(mBitBlockType))
     
    10197    mNestingDepth = 0;
    10298    mMaxNestingDepth = 0;
    103     mCarryQueueSize = 0;
    104     mAdvanceQueueSize = 0;
    105     Examine(pb);
    106     mCarryQueueVector.resize(mCarryQueueSize);
    107     mAdvanceQueueVector.resize(mAdvanceQueueSize);
    108     mCarryQueueSummaryIdx.resize(mCarryQueueSize);
    109     mAdvanceQueueSummaryIdx.resize(mAdvanceQueueSize);
     99    unsigned totalCarryDataSize = Examine(pb, 0);
     100    mCarryDataVector.resize(totalCarryDataSize);
     101    mCarryDataSummaryIdx.resize(totalCarryDataSize);
    110102    std::string errMessage;
    111103    EngineBuilder builder(mMod);
     
    130122    mOutputAddrPtr->setName("output");
    131123
    132     //Create the carry and advance queues.
    133     mCarryQueueIdx = 0;
    134     mAdvanceQueueIdx = 0;
    135124    mNestingDepth = 0;
    136125    mMaxNestingDepth = 0;
     
    149138    compileBlock(pb);
    150139
    151     if (LLVM_UNLIKELY(mCarryQueueIdx != mCarryQueueSize)) {
    152         throw std::runtime_error("Actual carry queue size (" + std::to_string(mCarryQueueIdx) + ") does not match expected (" + std::to_string(mCarryQueueSize) + ")");
    153     }
    154     if (LLVM_UNLIKELY(mAdvanceQueueIdx != mAdvanceQueueSize)) {
    155         throw std::runtime_error("Actual advance queue size (" + std::to_string(mAdvanceQueueIdx) + ") does not match expected (" + std::to_string(mAdvanceQueueSize) + ")");
    156     }
    157140    if (LLVM_UNLIKELY(mNestingDepth != 0)) {
    158141        throw std::runtime_error("Non-zero nesting depth error (" + std::to_string(mNestingDepth) + ")");
     
    172155
    173156    //Return the required size of the carry data area to the process_block function.
    174     return CompiledPabloFunction((mCarryQueueSize + mAdvanceQueueSize) * sizeof(BitBlock), mFunction, mExecutionEngine);
     157    // Reserve 1 element in the carry data area for current block number (future). TODO
     158    return CompiledPabloFunction((totalCarryDataSize + 1) * sizeof(BitBlock), mFunction, mExecutionEngine);
    175159}
    176160
     
    351335    mFunction->setAttributes(AttrSet);
    352336}
    353 
     337   
     338// CarryDataNumbering
    354339//
    355 // CarryNumbering: sequential numbers associated with each
    356 // carry-generating operation encountered in a traversal of the
    357 // Pablo AST.    Carry-generating operations are MatchStar, ScanThru,
    358 // and so on.
    359 // AdvanceNumbering: sequential numbers associated with each Advance
    360 // operation encountered in tree traversal, with the following modifications.
    361 //   (a) an additional AdvanceQueue entry is created for each if-statement
     340// For each PabloBlock, a contiguous CarryData area holds carry,
     341// and advance values that are generated in one block for use in the
     342// next.  For a given block, the carry data area contains the
     343// carries, the advances and the nested data for contained blocks,
     344// if any.
     345// Notes:
     346//   (a) an additional data entry is created for each if-statement
    362347//       having more than one carry or advance opreation within it.  This
    363348//       additional entry is a summary entry which must be nonzero to
     
    366351//   (b) advancing by a large amount may require multiple advance entries.
    367352//       the number of advance entries for an operation Adv(x, n) is
    368 //       (n - 1) / BLOCK_SIZE + 1
    369 //
    370 // Note that the initial carry/advance numbering is determined by the
    371 // Examine function.  The values determined at this stage must be consistent
    372 // with the later numbering calculated during actual statement compilation.
     353//       (n + BLOCK_SIZE - 1) / BLOCK_SIZE
    373354//
    374355// Examine precomputes some CarryNumbering and AdvanceNumbering, as
    375356// well as mMaxNestingDepth of while loops.
    376357//
    377 void PabloCompiler::Examine(PabloBlock & blk) {
     358unsigned PabloCompiler::Examine(PabloBlock & blk, unsigned carryDataIndexIn) {
    378359    // Count local carries and advances at this level.
     360    unsigned carryDataIndex = carryDataIndexIn;
    379361    unsigned localCarries = 0;
    380362    unsigned localAdvances = 0;
     363    unsigned nestedCarryDataSize = 0;
    381364    for (Statement * stmt : blk) {
    382365        if (Advance * adv = dyn_cast<Advance>(stmt)) {
     366            adv->setLocalAdvanceIndex(localAdvances);
    383367            localAdvances += (adv->getAdvanceAmount() + BLOCK_SIZE - 1) / BLOCK_SIZE;
    384368        }
    385         else if (isa<MatchStar>(stmt) || isa<ScanThru>(stmt)) {
     369        else if (MatchStar * m = dyn_cast<MatchStar>(stmt)) {
     370            m->setLocalCarryIndex(localCarries);
    386371            ++localCarries;
    387372        }
    388     }
    389     mCarryQueueSize += localCarries;
    390     mAdvanceQueueSize += localAdvances;
     373        else if (ScanThru * s = dyn_cast<ScanThru>(stmt)) {
     374            s->setLocalCarryIndex(localCarries);
     375            ++localCarries;
     376        }
     377    }
     378    carryDataIndex += localCarries + localAdvances;
    391379    for (Statement * stmt : blk) {
    392380        if (Call * call = dyn_cast<Call>(stmt)) {
     
    394382        }
    395383        else if (If * ifStatement = dyn_cast<If>(stmt)) {
    396             const auto preIfCarryCount = mCarryQueueSize;
    397             const auto preIfAdvanceCount = mAdvanceQueueSize;
    398             Examine(ifStatement->getBody());
    399             int ifCarryCount = mCarryQueueSize - preIfCarryCount;
    400             int ifAdvanceCount = mAdvanceQueueSize - preIfAdvanceCount;
    401             if ((ifCarryCount + ifAdvanceCount) > 1) {
    402                 ++mAdvanceQueueSize;
    403                 ++ifAdvanceCount;
    404             }
    405             ifStatement->setInclusiveCarryCount(ifCarryCount);
    406             ifStatement->setInclusiveAdvanceCount(ifAdvanceCount);
     384            const auto ifCarryDataSize = Examine(ifStatement->getBody(), carryDataIndex);
     385            nestedCarryDataSize += ifCarryDataSize;
     386            carryDataIndex += ifCarryDataSize;
    407387        }
    408388        else if (While * whileStatement = dyn_cast<While>(stmt)) {
    409             const auto preWhileCarryCount = mCarryQueueSize;
    410             const auto preWhileAdvanceCount = mAdvanceQueueSize;
    411389            mMaxNestingDepth = std::max(mMaxNestingDepth, ++mNestingDepth);
    412             Examine(whileStatement->getBody());
     390            const auto whileCarryDataSize = Examine(whileStatement->getBody(), carryDataIndex);
    413391            --mNestingDepth;
    414             whileStatement->setInclusiveCarryCount(mCarryQueueSize - preWhileCarryCount);
    415             whileStatement->setInclusiveAdvanceCount(mAdvanceQueueSize - preWhileAdvanceCount);
    416         }
    417     }
     392            nestedCarryDataSize += whileCarryDataSize;
     393            carryDataIndex += whileCarryDataSize;
     394        }
     395    }
     396    blk.setCarryIndexBase(carryDataIndexIn);
     397    blk.setLocalCarryCount(localCarries);
     398    blk.setLocalAdvanceCount(localAdvances);
     399    unsigned totalCarryDataSize = localCarries + localAdvances + nestedCarryDataSize;
     400    if (totalCarryDataSize > 1) {
     401        // Need extra space for the summary variable, always the last
     402        // entry within the block.
     403        totalCarryDataSize += 1;
     404    }
     405    blk.setTotalCarryDataSize(totalCarryDataSize);
     406    return totalCarryDataSize;
    418407}
    419408
     
    445434}
    446435
    447 void PabloCompiler::compileIf(const If * ifStatement) {
     436
     437
     438
     439void PabloCompiler::compileIf(const If * ifStatement) {       
    448440        //
    449441        //  The If-ElseZero stmt:
     
    467459        BasicBlock * ifEndBlock = BasicBlock::Create(mMod->getContext(), "if.end", mFunction, 0);
    468460       
    469         const auto baseCarryQueueIdx = mCarryQueueIdx;
    470         const auto baseAdvanceQueueIdx = mAdvanceQueueIdx;
    471        
    472         int ifCarryCount = ifStatement->getInclusiveCarryCount();
    473         int ifAdvanceCount = ifStatement->getInclusiveAdvanceCount();
    474         //  Carry/Advance queue strategy.   
    475         //  If there are any carries or advances at any nesting level within the
    476         //  if statement, then the statement must be executed.   A "summary"
    477         //  carryover variable is determined for this purpose, consisting of the
    478         //  or of all of the carry and advance variables within the if.
    479         //  This variable is determined as follows.
    480         //  (a)  If the CarryCount and AdvanceCount are both 0, there is no summary variable.
    481         //  (b)  If the CarryCount is 1 and the AdvanceCount is 0, then the summary
    482         //       carryover variable is just the single carry queue entry.
    483         //  (c)  If the CarryCount is 0 and the AdvanceCount is 1, then the summary
    484         //       carryover variable is just the advance carry queue entry.
    485         //  (d)  Otherwise, an additional advance queue entry is created for the
    486         //       summary variable.
    487         //  Note that the test for cases (c) and (d) may be combined: the summary carryover
    488         //  variable is just last advance queue entry.
    489         //
    490        
    491461        IRBuilder<> b_entry(ifEntryBlock);
    492462        mBasicBlock = ifEntryBlock;
     463   
     464        const unsigned baseCarryDataIdx = ifStatement->getBody().getCarryIndexBase();
     465        const unsigned carryDataSize = ifStatement->getBody().getTotalCarryDataSize();
     466        const unsigned carrySummaryIndex = baseCarryDataIdx + carryDataSize - 1;
     467       
    493468        Value* if_test_value = compileExpression(ifStatement->getCondition());
    494        
    495         if ((ifCarryCount == 1) && (ifAdvanceCount == 0)) {
    496             Value* last_if_pending_carries = genCarryInLoad(baseCarryQueueIdx);
    497             if_test_value = b_entry.CreateOr(if_test_value, last_if_pending_carries);
    498         }
    499         else if ((ifCarryCount > 0) || (ifAdvanceCount > 0)) {
    500             Value* last_if_pending_advances = genAdvanceInLoad(baseAdvanceQueueIdx + ifAdvanceCount - 1);
    501             if_test_value = b_entry.CreateOr(if_test_value, last_if_pending_advances);
     469        if (carryDataSize > 0) {
     470            // load the summary variable
     471            Value* last_if_pending_data = genCarryDataLoad(carrySummaryIndex);
     472            if_test_value = b_entry.CreateOr(if_test_value, last_if_pending_data);
    502473        }
    503474        b_entry.CreateCondBr(genBitBlockAny(if_test_value), ifEndBlock, ifBodyBlock);
     
    505476        // Entry processing is complete, now handle the body of the if.
    506477        mBasicBlock = ifBodyBlock;
    507         compileBlock(ifStatement->getBody());
     478        compileBlock(ifStatement -> getBody());
    508479
    509480        // If we compiled an If or a While statement, we won't be in the same basic block as before.
     
    513484        // carry over variable.
    514485       
    515         if ((ifCarryCount + ifAdvanceCount) > 1) {
    516             // A summary variable is needed.
    517 
     486        if (carryDataSize > 1) {
     487            // If there was only one carry entry, then it also serves as the summary variable.
     488            // Otherwise, we need to combine entries to compute the summary.
    518489            Value * carry_summary = mZeroInitializer;
    519             for (int c = baseCarryQueueIdx; c < baseCarryQueueIdx + ifCarryCount; c++) {
    520                 int s = mCarryQueueSummaryIdx[c];
     490            for (int c = baseCarryDataIdx; c < carrySummaryIndex; c++) {
     491                int s = mCarryDataSummaryIdx[c];
    521492                if (s == -1) {
    522                     Value* carryq_value = mCarryQueueVector[c];
     493                    Value* carryq_value = mCarryDataVector[c];
    523494                    if (carry_summary == mZeroInitializer) {
    524495                        carry_summary = carryq_value;
     
    527498                        carry_summary = bIfBody.CreateOr(carry_summary, carryq_value);
    528499                    }
    529                     mCarryQueueSummaryIdx[c] = mAdvanceQueueIdx;
     500                    mCarryDataSummaryIdx[c] = carrySummaryIndex;
    530501                }
    531502            }
    532             // Note that the limit in the following uses -1, because
    533             // last entry of the advance queue is for the summary variable.
    534             for (int c = baseAdvanceQueueIdx; c < baseAdvanceQueueIdx + ifAdvanceCount - 1; c++) {
    535                 int s = mAdvanceQueueSummaryIdx[c];
    536                 if (s == -1 ) {
    537                     Value* advance_q_value = mAdvanceQueueVector[c];
    538                     if (carry_summary == mZeroInitializer) {
    539                         carry_summary = advance_q_value;
    540                     }
    541                     else {
    542                         carry_summary = bIfBody.CreateOr(carry_summary, advance_q_value);
    543                     }
    544                     mAdvanceQueueSummaryIdx[c] = mAdvanceQueueIdx;
    545                 }
    546             }
    547             genAdvanceOutStore(carry_summary, mAdvanceQueueIdx++);
     503            genCarryDataStore(carry_summary, carrySummaryIndex);
    548504        }
    549505        bIfBody.CreateBr(ifEndBlock);
     
    560516        }
    561517        // Create the phi Node for the summary variable.
    562         if (ifAdvanceCount >= 1) {
    563             // final AdvanceQ entry is summary variable.
     518        if (carryDataSize > 0) {
    564519            PHINode * summary_phi = bEnd.CreatePHI(mBitBlockType, 2, "summary");
    565520            summary_phi->addIncoming(mZeroInitializer, ifEntryBlock);
    566             summary_phi->addIncoming(mAdvanceQueueVector[mAdvanceQueueIdx-1], mBasicBlock);
    567             mAdvanceQueueVector[mAdvanceQueueIdx-1] = summary_phi;
    568         }
    569         else if (ifCarryCount == 1) {
    570             PHINode * summary_phi = bEnd.CreatePHI(mBitBlockType, 2, "summary");
    571             summary_phi->addIncoming(mZeroInitializer, ifEntryBlock);
    572             summary_phi->addIncoming(mCarryQueueVector[baseCarryQueueIdx], mBasicBlock);
    573             mCarryQueueVector[baseCarryQueueIdx] = summary_phi;
     521            summary_phi->addIncoming(mCarryDataVector[carrySummaryIndex], mBasicBlock);
     522            mCarryDataVector[carrySummaryIndex] = summary_phi;
    574523        }
    575524       
     
    579528
    580529void PabloCompiler::compileWhile(const While * whileStatement) {
    581         const auto baseCarryQueueIdx = mCarryQueueIdx;
    582         const auto baseAdvanceQueueIdx = mAdvanceQueueIdx;
     530        const unsigned baseCarryDataIdx = whileStatement->getBody().getCarryIndexBase();
     531        const unsigned carryDataSize = whileStatement->getBody().getTotalCarryDataSize();
     532   
    583533        if (mNestingDepth == 0) {
    584             for (auto i = 0; i != whileStatement->getInclusiveCarryCount(); ++i) {
    585                 genCarryInLoad(baseCarryQueueIdx + i);
    586             }
    587             for (auto i = 0; i != whileStatement->getInclusiveAdvanceCount(); ++i) {
    588                 genAdvanceInLoad(baseAdvanceQueueIdx + i);
     534            for (auto i = 0; i < carryDataSize; ++i) {
     535                genCarryDataLoad(baseCarryDataIdx + i);
    589536            }
    590537        }
     
    597544        }
    598545
    599         // Compile the initial iteration statements; the calls to genCarryOutStore will update the
    600         // mCarryQueueVector with the appropriate values. Although we're not actually entering a new basic
    601         // block yet, increment the nesting depth so that any calls to genCarryInLoad or genCarryOutStore
     546        // Compile the initial iteration statements; the calls to genCarryDataStore will update the
     547        // mCarryDataVector with the appropriate values. Although we're not actually entering a new basic
     548        // block yet, increment the nesting depth so that any calls to genCarryDataLoad or genCarryDataStore
    602549        // will refer to the previous value.
    603550
     
    609556        // that compiling the while body twice will generate the equivalent IR. This is not necessarily true
    610557        // but works for now.
    611         mCarryQueueIdx = baseCarryQueueIdx;
    612         mAdvanceQueueIdx = baseAdvanceQueueIdx;
    613558
    614559        BasicBlock* whileCondBlock = BasicBlock::Create(mMod->getContext(), "while.cond", mFunction, 0);
     
    624569        IRBuilder<> bCond(whileCondBlock);
    625570        // generate phi nodes for any carry propogating instruction
    626         int whileCarryCount = whileStatement->getInclusiveCarryCount();
    627         int whileAdvanceCount = whileStatement->getInclusiveAdvanceCount();
    628         std::vector<PHINode*> phiNodes(whileCarryCount + whileAdvanceCount + nextNodes.size());
     571        std::vector<PHINode*> phiNodes(carryDataSize + nextNodes.size());
    629572        unsigned index = 0;
    630         for (index = 0; index != whileCarryCount; ++index) {
     573        for (index = 0; index < carryDataSize; ++index) {
    631574            PHINode * phi = bCond.CreatePHI(mBitBlockType, 2);
    632             phi->addIncoming(mCarryQueueVector[baseCarryQueueIdx + index], mBasicBlock);
    633             mCarryQueueVector[baseCarryQueueIdx + index] = mZeroInitializer; // (use phi for multi-carry mode.)
     575            phi->addIncoming(mCarryDataVector[baseCarryDataIdx + index], mBasicBlock);
     576            mCarryDataVector[baseCarryDataIdx + index] = mZeroInitializer; // (use phi for multi-carry mode.)
    634577            phiNodes[index] = phi;
    635         }
    636         for (int i = 0; i != whileAdvanceCount; ++i) {
    637             PHINode * phi = bCond.CreatePHI(mBitBlockType, 2);
    638             phi->addIncoming(mAdvanceQueueVector[baseAdvanceQueueIdx + i], mBasicBlock);
    639             mAdvanceQueueVector[baseAdvanceQueueIdx + i] = mZeroInitializer; // (use phi for multi-carry mode.)
    640             phiNodes[index++] = phi;
    641578        }
    642579        // and for any Next nodes in the loop body
     
    658595        // update phi nodes for any carry propogating instruction
    659596        IRBuilder<> bWhileBody(mBasicBlock);
    660         for (index = 0; index != whileStatement->getInclusiveCarryCount(); ++index) {
    661             Value * carryOut = bWhileBody.CreateOr(phiNodes[index], mCarryQueueVector[baseCarryQueueIdx + index]);
     597        for (index = 0; index < carryDataSize; ++index) {
     598            Value * carryOut = bWhileBody.CreateOr(phiNodes[index], mCarryDataVector[baseCarryDataIdx + index]);
    662599            PHINode * phi = phiNodes[index];
    663600            phi->addIncoming(carryOut, mBasicBlock);
    664             mCarryQueueVector[baseCarryQueueIdx + index] = phi;
    665         }
    666         for (int i = 0; i != whileAdvanceCount; ++i) {
    667             Value * advOut = bWhileBody.CreateOr(phiNodes[index], mAdvanceQueueVector[baseAdvanceQueueIdx + i]);
    668             PHINode * phi = phiNodes[index++];
    669             phi->addIncoming(advOut, mBasicBlock);
    670             mAdvanceQueueVector[baseAdvanceQueueIdx + i] = phi;
     601            mCarryDataVector[baseCarryDataIdx + index] = phi;
    671602        }
    672603        // and for any Next nodes in the loop body
     
    684615        mBasicBlock = whileEndBlock;
    685616        if (--mNestingDepth == 0) {
    686             for (index = 0; index != whileCarryCount; ++index) {
    687                 genCarryOutStore(phiNodes[index], baseCarryQueueIdx + index);
    688             }
    689             for (index = 0; index != whileAdvanceCount; ++index) {
    690                 genAdvanceOutStore(phiNodes[whileCarryCount + index], baseAdvanceQueueIdx + index);
     617            for (index = 0; index < carryDataSize; ++index) {
     618                genCarryDataStore(phiNodes[index], baseCarryDataIdx + index);
    691619            }
    692620        }
     
    759687        Value* strm_value = compileExpression(adv->getExpr());
    760688        int shift = adv->getAdvanceAmount();
    761         Value * expr = genAdvanceWithCarry(strm_value, shift);
     689        unsigned advance_index = adv->getLocalAdvanceIndex();
     690        Value * expr = genAdvanceWithCarry(strm_value, shift, advance_index, stmt->getParent());
    762691        mMarkerMap[adv] = expr;
    763692        // return expr;
     
    768697        Value * cc = compileExpression(mstar->getCharClass());
    769698        Value * marker_and_cc = b.CreateAnd(marker, cc);
    770         Value * expr = b.CreateOr(b.CreateXor(genAddWithCarry(marker_and_cc, cc), cc), marker, "matchstar");
     699        unsigned carry_index = mstar->getLocalCarryIndex();
     700        Value * expr = b.CreateOr(b.CreateXor(genAddWithCarry(marker_and_cc, cc, carry_index, stmt->getParent()), cc), marker, "matchstar");
    771701        mMarkerMap[mstar] = expr;
    772702        // return expr;
     
    776706        Value * marker_expr = compileExpression(sthru->getScanFrom());
    777707        Value * cc_expr = compileExpression(sthru->getScanThru());
    778         Value * expr = b.CreateAnd(genAddWithCarry(marker_expr, cc_expr), genNot(cc_expr), "scanthru");
     708        unsigned carry_index = sthru->getLocalCarryIndex();
     709        Value * expr = b.CreateAnd(genAddWithCarry(marker_expr, cc_expr, carry_index, stmt->getParent()), genNot(cc_expr), "scanthru");
    779710        mMarkerMap[sthru] = expr;
    780711        // return expr;
     
    860791
    861792
    862 Value* PabloCompiler::genAddWithCarry(Value* e1, Value* e2) {
     793Value* PabloCompiler::genAddWithCarry(Value* e1, Value* e2, unsigned localIndex, const PabloBlock * blk) {
    863794    IRBuilder<> b(mBasicBlock);
    864795
    865796    //CarryQ - carry in.
    866     const int carryIdx = mCarryQueueIdx++;
    867     Value* carryq_value = genCarryInLoad(carryIdx);
     797    const int carryIdx = blk->getCarryIndexBase() + localIndex;
     798    Value* carryq_value = genCarryDataLoad(carryIdx);
    868799#ifdef USE_TWO_UADD_OVERFLOW
    869800    //This is the ideal implementation, which uses two uadd.with.overflow
     
    922853#endif //USE_TWO_UADD_OVERFLOW
    923854
    924     genCarryOutStore(carry_out, carryIdx);
     855    genCarryDataStore(carry_out, carryIdx);
    925856    return sum;
    926857}
    927858
    928 Value* PabloCompiler::genCarryInLoad(const unsigned index) {
    929     assert (index < mCarryQueueVector.size());
     859Value* PabloCompiler::genCarryDataLoad(const unsigned index) {
     860    assert (index < mCarryDataVector.size());
    930861    if (mNestingDepth == 0) {
    931862        IRBuilder<> b(mBasicBlock);
    932         mCarryQueueVector[index] = b.CreateAlignedLoad(b.CreateGEP(mCarryDataPtr, b.getInt64(index)), BLOCK_SIZE/8, false);
    933     }
    934     return mCarryQueueVector[index];
    935 }
    936 
    937 void PabloCompiler::genCarryOutStore(Value* carryOut, const unsigned index ) {
     863        mCarryDataVector[index] = b.CreateAlignedLoad(b.CreateGEP(mCarryDataPtr, b.getInt64(index)), BLOCK_SIZE/8, false);
     864    }
     865    return mCarryDataVector[index];
     866}
     867
     868void PabloCompiler::genCarryDataStore(Value* carryOut, const unsigned index ) {
    938869    assert (carryOut);
    939     assert (index < mCarryQueueVector.size());
     870    assert (index < mCarryDataVector.size());
    940871    if (mNestingDepth == 0) {
    941872        IRBuilder<> b(mBasicBlock);
    942873        b.CreateAlignedStore(carryOut, b.CreateGEP(mCarryDataPtr, b.getInt64(index)), BLOCK_SIZE/8, false);
    943874    }
    944     mCarryQueueSummaryIdx[index] = -1;
    945     mCarryQueueVector[index] = carryOut;
    946 }
    947 
    948 Value* PabloCompiler::genAdvanceInLoad(const unsigned index) {
    949     assert (index < mAdvanceQueueVector.size());
    950     if (mNestingDepth == 0) {
    951         IRBuilder<> b(mBasicBlock);
    952         mAdvanceQueueVector[index] = b.CreateAlignedLoad(b.CreateGEP(mCarryDataPtr, b.getInt64(mCarryQueueSize + index)), BLOCK_SIZE/8, false);
    953     }
    954     return mAdvanceQueueVector[index];
    955 }
    956 
    957 void PabloCompiler::genAdvanceOutStore(Value* advanceOut, const unsigned index ) {
    958     assert (advanceOut);
    959     assert (index < mAdvanceQueueVector.size());
    960     if (mNestingDepth == 0) {
    961         IRBuilder<> b(mBasicBlock);
    962         b.CreateAlignedStore(advanceOut, b.CreateGEP(mCarryDataPtr, b.getInt64(mCarryQueueSize + index)), BLOCK_SIZE/8, false);
    963     }
    964     mAdvanceQueueSummaryIdx[index] = -1;
    965     mAdvanceQueueVector[index] = advanceOut;
     875    mCarryDataSummaryIdx[index] = -1;
     876    mCarryDataVector[index] = carryOut;
    966877}
    967878
     
    988899    return b.CreateXor(expr, mOneInitializer, "not");
    989900}
    990 Value* PabloCompiler::genAdvanceWithCarry(Value* strm_value, int shift_amount) {
     901Value* PabloCompiler::genAdvanceWithCarry(Value* strm_value, int shift_amount, unsigned localIndex, const PabloBlock * blk) {
    991902    IRBuilder<> b(mBasicBlock);
    992903    int advEntries = (shift_amount - 1) / BLOCK_SIZE + 1;
    993904    int block_shift = shift_amount % BLOCK_SIZE;
    994     const auto storeIdx = mAdvanceQueueIdx;
    995     const auto loadIdx = mAdvanceQueueIdx + advEntries - 1;
    996     mAdvanceQueueIdx += advEntries;
     905    const auto advanceIndex = blk->getCarryIndexBase() + blk->getLocalCarryCount() + localIndex;
     906    const auto storeIdx = advanceIndex;
     907    const auto loadIdx = advanceIndex + advEntries - 1;
    997908    Value* result_value;
    998909   
    999910    if (advEntries == 1) {
    1000911        if (block_shift == 0) { 
    1001             result_value = genAdvanceInLoad(loadIdx);
     912            result_value = genCarryDataLoad(loadIdx);
    1002913            //b.CreateCall(mFunc_print_register, result_value);
    1003914        }
    1004915#if (BLOCK_SIZE == 128) && !defined(USE_LONG_INTEGER_SHIFT)
    1005916        if (block_shift == 1) {
    1006             Value* advanceq_value = genShiftHighbitToLow(genAdvanceInLoad(loadIdx));
     917            Value* advanceq_value = genShiftHighbitToLow(genCarryDataLoad(loadIdx));
    1007918            Value* srli_1_value = b.CreateLShr(strm_value, 63);
    1008919            Value* packed_shuffle;
     
    1020931            // This is the preferred logic, but is too slow for the general case.
    1021932            // We need to speed up our custom LLVM for this code.
    1022             Value* advanceq_longint = b.CreateBitCast(genAdvanceInLoad(loadIdx), IntegerType::get(mMod->getContext(), BLOCK_SIZE));
     933            Value* advanceq_longint = b.CreateBitCast(genCarryDataLoad(loadIdx), IntegerType::get(mMod->getContext(), BLOCK_SIZE));
    1023934            Value* strm_longint = b.CreateBitCast(strm_value, IntegerType::get(mMod->getContext(), BLOCK_SIZE));
    1024935            Value* adv_longint = b.CreateOr(b.CreateShl(strm_longint, block_shift), b.CreateLShr(advanceq_longint, BLOCK_SIZE - block_shift), "advance");
     
    1026937        }
    1027938#else
    1028         Value* advanceq_longint = b.CreateBitCast(genAdvanceInLoad(loadIdx), IntegerType::get(mMod->getContext(), BLOCK_SIZE));
     939        Value* advanceq_longint = b.CreateBitCast(genCarryDataLoad(loadIdx), IntegerType::get(mMod->getContext(), BLOCK_SIZE));
    1029940        Value* strm_longint = b.CreateBitCast(strm_value, IntegerType::get(mMod->getContext(), BLOCK_SIZE));
    1030941        Value* adv_longint = b.CreateOr(b.CreateShl(strm_longint, block_shift), b.CreateLShr(advanceq_longint, BLOCK_SIZE - block_shift), "advance");
     
    1035946    else {
    1036947        if (block_shift == 0) {
    1037             result_value = genAdvanceInLoad(loadIdx);
     948            result_value = genCarryDataLoad(loadIdx);
    1038949        }
    1039950        else {
    1040951            // The advance is based on the two oldest bit blocks in the advance queue.
    1041             Value* advanceq_longint = b.CreateBitCast(genAdvanceInLoad(loadIdx), IntegerType::get(mMod->getContext(), BLOCK_SIZE));
    1042             Value* strm_longint = b.CreateBitCast(genAdvanceInLoad(loadIdx-1), IntegerType::get(mMod->getContext(), BLOCK_SIZE));
     952            Value* advanceq_longint = b.CreateBitCast(genCarryDataLoad(loadIdx), IntegerType::get(mMod->getContext(), BLOCK_SIZE));
     953            Value* strm_longint = b.CreateBitCast(genCarryDataLoad(loadIdx-1), IntegerType::get(mMod->getContext(), BLOCK_SIZE));
    1043954            Value* adv_longint = b.CreateOr(b.CreateShl(strm_longint, block_shift), b.CreateLShr(advanceq_longint, BLOCK_SIZE - block_shift), "longadvance");
    1044955            result_value = b.CreateBitCast(adv_longint, mBitBlockType);
    1045             //b.CreateCall(mFunc_print_register, genAdvanceInLoad(loadIdx));
    1046             //b.CreateCall(mFunc_print_register, genAdvanceInLoad(loadIdx-1));
     956            //b.CreateCall(mFunc_print_register, genCarryDataLoad(loadIdx));
     957            //b.CreateCall(mFunc_print_register, genCarryDataLoad(loadIdx-1));
    1047958            //b.CreateCall(mFunc_print_register, result_value);
    1048959        }
    1049960        // copy entries from previous blocks forward
    1050961        for (int i = loadIdx; i > storeIdx; i--) {
    1051             genAdvanceOutStore(genAdvanceInLoad(i-1), i);
    1052         }
    1053     }
    1054     genAdvanceOutStore(strm_value, storeIdx);
     962            genCarryDataStore(genCarryDataLoad(i-1), i);
     963        }
     964    }
     965    genCarryDataStore(strm_value, storeIdx);
    1055966    return result_value;
    1056967}
Note: See TracChangeset for help on using the changeset viewer.