source: icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp @ 4697

Last change on this file since 4697 was 4697, checked in by cameron, 4 years ago

Further progress on Carry Manager with bit packing

File size: 21.0 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <include/simd-lib/bitblock.hpp>
9#include <stdexcept>
10#include <pablo/carry_data.h>
11#include <pablo/codegenstate.h>
12#include <pablo/carry_manager.h>
13#include <pablo/pabloAST.h>
14#include <iostream>
15
16namespace pablo {
17 
18    unsigned doScopeCount(PabloBlock * pb) {
19        unsigned count = 1;
20       
21        for (Statement * stmt : *pb) {
22            if (If * ifStatement = dyn_cast<If>(stmt)) {
23                count += doScopeCount(&ifStatement->getBody());
24            }
25            else if (While * whileStatement = dyn_cast<While>(stmt)) {
26                count += doScopeCount(&whileStatement->getBody());
27            }
28        }
29        return count;
30       
31    }
32
33unsigned CarryManager::initialize(PabloBlock * pb, Value * carryPtr) {
34    mPabloRoot = pb;
35    mCarryDataPtr = carryPtr;
36    unsigned scopeCount = doScopeCount(pb);
37    mCarryInfoVector.resize(scopeCount);
38   
39    unsigned totalCarryDataBits = enumerate(pb, 0, 0);
40   
41    mTotalCarryDataBitBlocks = (totalCarryDataBits + BLOCK_SIZE - 1)/BLOCK_SIZE; 
42    // Carry Data area will have one extra bit block to store the block number.
43    mBlockNoPtr = mBuilder->CreateBitCast(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(mTotalCarryDataBitBlocks)), Type::getInt64PtrTy(mBuilder->getContext()));
44    mBlockNo = mBuilder->CreateLoad(mBlockNoPtr);
45    mCarryPackPtr.resize(mTotalCarryDataBitBlocks);
46    mCarryInPack.resize(mTotalCarryDataBitBlocks);
47    mCarryOutPack.resize(mTotalCarryDataBitBlocks);
48    for (auto i = 0; i < mTotalCarryDataBitBlocks; i++) mCarryInPack[i]=nullptr;
49   
50    /*  Set the current scope to PabloRoot */
51    mCurrentScope = mPabloRoot;
52    mCurrentFrameIndex = 0;
53    mCarryInfo = mCarryInfoVector[0];
54
55    return mTotalCarryDataBitBlocks + 1; // One extra element for the block no.
56}
57   
58void CarryManager::generateBlockNoIncrement() {
59    mBuilder->CreateStore(mBuilder->CreateAdd(mBlockNo, mBuilder->getInt64(1)), mBlockNoPtr);
60}
61
62Value * CarryManager::getBlockNoPtr() {
63    return mBlockNoPtr;
64}
65
66
67unsigned CarryManager::enumerate(PabloBlock * blk, unsigned ifDepth, unsigned whileDepth) {
68    llvm::raw_os_ostream cerr(std::cerr);
69    unsigned idx = blk->getScopeIndex();
70    PabloBlockCarryData * cd = new PabloBlockCarryData(blk);
71    mCarryInfoVector[idx] = cd;
72
73    cd->setIfDepth(ifDepth);
74    cd->setWhileDepth(whileDepth);
75    unsigned nestedOffset = cd->nested.frameOffsetinBits;
76 
77    for (Statement * stmt : *blk) {
78        if (If * ifStatement = dyn_cast<If>(stmt)) {
79            const unsigned ifCarryDataBits = enumerate(&ifStatement->getBody(), ifDepth+1, whileDepth);
80            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[ifStatement->getBody().getScopeIndex()];
81#ifdef PACKING
82            EnsurePackHasSpace(nestedOffset, ifCarryDataBits);
83#endif
84            nestedBlockData->setFramePosition(nestedOffset);
85
86            nestedOffset += ifCarryDataBits;
87            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
88            cd->nested.entries++;
89#ifndef NDEBUG
90            nestedBlockData->dumpCarryData(cerr);
91#endif
92        }
93        else if (While * whileStatement = dyn_cast<While>(stmt)) {
94            const unsigned whileCarryDataBits = enumerate(&whileStatement->getBody(), ifDepth, whileDepth+1);
95            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[whileStatement->getBody().getScopeIndex()];
96            //if (whileStatement->isMultiCarry()) whileCarryDataBits *= whileStatement->getMaxIterations();
97#ifdef PACKING
98            EnsurePackHasSpace(nestedOffset, whileCarryDataBits);
99#endif
100            nestedBlockData->setFramePosition(nestedOffset);
101            nestedOffset += whileCarryDataBits;
102            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
103            cd->nested.entries++;
104#ifndef NDEBUG
105            nestedBlockData->dumpCarryData(cerr);
106#endif
107        }
108    }
109   
110    cd->scopeCarryDataBits = nestedOffset;
111   
112    if (cd->explicitSummaryRequired()) {
113        // Need extra space for the summary variable, always the last
114        // entry within an if block.
115        cd->scopeCarryDataBits = alignCeiling(cd->scopeCarryDataBits, PACK_SIZE);
116        cd->summary.frameOffsetinBits = cd->scopeCarryDataBits;
117        cd->summary.allocatedBits = PACK_SIZE;
118        cd->scopeCarryDataBits += PACK_SIZE;
119    }
120    else {
121        cd->summary.frameOffsetinBits = 0;
122        cd->summary.allocatedBits = cd->scopeCarryDataBits;
123    }
124    return cd->scopeCarryDataBits;
125}
126
127
128/* Entering and leaving blocks. */
129
130void CarryManager::enterScope(PabloBlock * blk) {
131   
132    mCurrentScope = blk;
133    mCarryInfo = mCarryInfoVector[blk->getScopeIndex()];
134    mCurrentFrameIndex += mCarryInfo->getFrameIndex();
135    //std::cerr << "enterScope:  mCurrentFrameIndex = " << mCurrentFrameIndex << std::endl;
136}
137
138void CarryManager::leaveScope() {
139    mCurrentFrameIndex -= mCarryInfo->getFrameIndex();
140    mCurrentScope = mCurrentScope->getParent();
141    mCarryInfo = mCarryInfoVector[mCurrentScope->getScopeIndex()];
142    //std::cerr << "leaveScope:  mCurrentFrameIndex = " << mCurrentFrameIndex << std::endl;
143}
144
145
146/* Helper routines */
147
148unsigned CarryManager::absPosition(unsigned frameOffsetinBits, unsigned relPos) {
149#ifdef PACKING
150    return mCurrentFrameIndex + frameOffsetinBits + relPos;
151#else
152    return mCurrentFrameIndex + frameOffsetinBits/BLOCK_SIZE + relPos;
153#endif
154}
155
156
157unsigned CarryManager::carryOpPosition(unsigned localIndex) {
158    return absPosition(mCarryInfo->addWithCarry.frameOffsetinBits, localIndex);
159}
160
161unsigned CarryManager::advance1Position(unsigned localIndex) {
162    return absPosition(mCarryInfo->advance1.frameOffsetinBits, localIndex);
163}
164
165unsigned CarryManager::shortAdvancePosition(unsigned localIndex) {
166    return absPosition(mCarryInfo->shortAdvance.frameOffsetinBits, localIndex);
167}
168
169unsigned CarryManager::longAdvancePosition(unsigned localIndex) {
170    return absPosition(mCarryInfo->longAdvance.frameOffsetinBits, localIndex);
171}
172
173unsigned CarryManager::summaryPosition() {
174    return absPosition(mCarryInfo->summary.frameOffsetinBits, 0);
175}
176
177unsigned CarryManager::summaryBits() {
178    return mCarryInfo->summary.allocatedBits;
179}
180
181
182
183Value * CarryManager::getCarryPack(unsigned packIndex) {
184    if (mCarryInPack[packIndex] == nullptr) {
185        Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(packIndex));
186        mCarryPackPtr[packIndex] = packPtr;
187        mCarryInPack[packIndex] = mBuilder->CreateAlignedLoad(packPtr, PACK_SIZE/8);
188    }
189    return mCarryInPack[packIndex];
190}
191
192void CarryManager::storeCarryPack(unsigned packIndex) {
193    mBuilder->CreateAlignedStore(mCarryOutPack[packIndex], mCarryPackPtr[packIndex], PACK_SIZE/8);
194}
195
196Value * CarryManager::getCarryRange(unsigned carryBit_lo, unsigned carryRangeSize) {
197
198    unsigned packIndex = carryBit_lo / PACK_SIZE;
199    unsigned carryOffset = carryBit_lo % PACK_SIZE;
200    unsigned hiOffset = carryBit_lo + carryRangeSize - 1;
201   
202    Value * carryItem = getCarryPack(packIndex);
203    if (carryRangeSize < PACK_SIZE) {
204       carryItem = mBuilder->CreateAnd(carryItem, mBuilder->getInt64((1 << hiOffset) - 1));
205    }
206    if (carryOffset > 0) {
207       carryItem = mBuilder->CreateLShr(carryItem, mBuilder->getInt64(carryOffset));
208    }
209    return carryItem;
210}
211   
212Value * CarryManager::getCarryBit(unsigned carryBitPos) {
213    return getCarryRange(carryBitPos, 1);
214}
215   
216void CarryManager::setCarryBits(unsigned carryBit_lo, Value * bits) {
217   
218    unsigned packIndex = carryBit_lo / PACK_SIZE;
219    unsigned carryOffset = carryBit_lo % PACK_SIZE;
220    if (carryOffset > 0) {
221        bits = mBuilder->CreateLShr(bits, mBuilder->getInt64(carryOffset));
222    }
223    if (mCarryOutPack[packIndex] == nullptr) {
224        mCarryOutPack[packIndex] = bits;
225    }
226    else {
227        mCarryOutPack[packIndex] = mBuilder->CreateOr(mCarryOutPack[packIndex], bits);
228    }
229}
230   
231   
232/* Methods for getting and setting individual carry values. */
233   
234Value * CarryManager::getCarryOpCarryIn(int localIndex) {
235    unsigned posn = carryOpPosition(localIndex);
236#ifdef PACKING
237    return getCarryBit(posn);
238#else
239    return getCarryPack(posn);
240#endif
241}
242   
243   
244void CarryManager::setCarryOpCarryOut(unsigned localIndex, Value * carry_out) {
245    unsigned posn = carryOpPosition(localIndex);
246#ifdef PACKING
247    setCarryBits(posn, carry_out);
248#else
249    mCarryOutPack[posn] = carry_out;
250    if (mCarryInfo->getWhileDepth() == 0) {
251        storeCarryPack(posn);
252    }
253#endif
254}
255
256   
257Value * CarryManager::advanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
258    if (shift_amount == 1) {
259        return unitAdvanceCarryInCarryOut(localIndex, strm);
260    }
261    else if (shift_amount < LongAdvanceBase) {
262        return shortAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
263    }
264    else {
265        return longAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
266    }
267}
268
269Value * CarryManager::unitAdvanceCarryInCarryOut(int localIndex, Value * strm) {
270    unsigned posn = advance1Position(localIndex);
271#ifdef PACKING
272    unsigned offset = posn % PACK_SIZE;
273    unsigned rshift = PACK_SIZE - offset - 1;
274    Value * field = iBuilder->mvmd_extract(PACK_SIZE, strm, BLOCK_SIZE/PACK_SIZE - 1);
275    if (rshift != 0) {
276        field = mBuilder->CreateLShr(field, mBuilder->getInt64(rshift));
277    }
278    if (offset != 0) {
279        field = mBuilder->CreateAnd(field, mBuilder->getInt64(1<<offset));
280    }
281    setCarryBits(posn - offset, field);
282    Value* carry_longint = mBuilder->CreateZExt(getCarryBit(posn), mBuilder->getIntNTy(BLOCK_SIZE));
283    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
284    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), carry_longint);
285    result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
286    return result_value;
287#else
288    mCarryOutPack[posn] = strm; 
289    Value * carry_in = getCarryPack(posn);
290    if (mCarryInfo->getWhileDepth() == 0) {
291        storeCarryPack(posn);
292    }
293    Value* result_value;
294   
295#if (BLOCK_SIZE == 128) && !defined(USE_LONG_INTEGER_SHIFT)
296    Value * ahead64 = iBuilder->mvmd_dslli(64, carry_in, strm, 1);
297    result_value = mBuilder->CreateOr(iBuilder->simd_srli(64, ahead64, 63), iBuilder->simd_slli(64, strm, 1));
298#else
299    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
300    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
301    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - 1), "advance");
302    result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
303   
304#endif
305    return result_value;
306#endif
307}
308
309Value * CarryManager::shortAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
310    unsigned posn = shortAdvancePosition(localIndex);
311#ifdef PACKING
312    unsigned offset = posn % PACK_SIZE;
313    unsigned rshift = PACK_SIZE - offset - shift_amount;
314    Value * field = iBuilder->mvmd_extract(PACK_SIZE, strm, BLOCK_SIZE/PACK_SIZE - 1);
315    if (rshift != 0) {
316        field = mBuilder->CreateLShr(field, mBuilder->getInt64(rshift));
317    }
318    if (offset != 0) {
319        field = mBuilder->CreateAnd(field, mBuilder->getInt64(((1<<shift_amount) - 1) << offset));
320    }   
321    setCarryBits(posn - offset, field);
322    Value* carry_longint = mBuilder->CreateZExt(getCarryRange(posn, shift_amount), mBuilder->getIntNTy(BLOCK_SIZE));
323    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
324    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), carry_longint);
325    result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
326    return result_value;
327#else
328    mCarryOutPack[posn] = strm; 
329    Value * carry_in = getCarryPack(posn);
330    if (mCarryInfo->getWhileDepth() == 0) {
331        storeCarryPack(posn);
332    }
333    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
334    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
335    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, shift_amount), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - shift_amount), "advance");
336    return mBuilder->CreateBitCast(adv_longint, mBitBlockType);
337#endif
338}
339   
340
341/*  currently defined in carry_data.h
342 
343 static unsigned power2ceil (unsigned v) {
344 unsigned ceil = 1;
345 while (ceil < v) ceil *= 2;
346 return ceil;
347 }
348 
349 unsigned longAdvanceEntries(unsigned shift_amount) const {
350 return (shift_amount + BLOCK_SIZE - 1)/BLOCK_SIZE;
351 }
352 
353 unsigned longAdvanceBufferSize(unsigned shift_amount)  const {
354 return power2ceil(longAdvanceEntries(shift_amount));
355 }
356 */
357
358   
359Value * CarryManager::longAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * carry_out) {
360    unsigned carryDataIndex = longAdvancePosition(localIndex);
361    Value * advBaseIndex = mBuilder->getInt64(carryDataIndex);
362    if (shift_amount <= BLOCK_SIZE) {
363        // special case using a single buffer entry and the carry_out value.
364        Value * advanceDataPtr = mBuilder->CreateGEP(mCarryDataPtr, advBaseIndex);
365        Value * carry_block0 = mBuilder->CreateAlignedLoad(advanceDataPtr, BLOCK_SIZE/8);
366        mBuilder->CreateAlignedStore(carry_out, advanceDataPtr, BLOCK_SIZE/8);
367        /* Very special case - no combine */
368        if (shift_amount == BLOCK_SIZE) return carry_block0;
369        Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - shift_amount);
370        Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_out, mBuilder->getIntNTy(BLOCK_SIZE)), shift_amount);
371        return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
372    }
373    // We need a buffer of at least two elements for storing the advance data.
374    const unsigned block_shift = shift_amount % BLOCK_SIZE;
375    const unsigned advanceEntries = mCarryInfo->longAdvanceEntries(shift_amount);
376    const unsigned bufsize = mCarryInfo->longAdvanceBufferSize(shift_amount);
377    Value * indexMask = mBuilder->getInt64(bufsize - 1);  // A mask to implement circular buffer indexing
378    Value * loadIndex0 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries)), indexMask), advBaseIndex);
379    Value * storeIndex = mBuilder->CreateAdd(mBuilder->CreateAnd(mBlockNo, indexMask), advBaseIndex);
380    Value * carry_block0 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex0), BLOCK_SIZE/8);
381    // If the long advance is an exact multiple of BLOCK_SIZE, we simply return the oldest
382    // block in the long advance carry data area. 
383    if (block_shift == 0) {
384        mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
385        return carry_block0;
386    }
387    // Otherwise we need to combine data from the two oldest blocks.
388    Value * loadIndex1 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries-1)), indexMask), advBaseIndex);
389    Value * carry_block1 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex1), BLOCK_SIZE/8);
390    Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - block_shift);
391    Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_block1, mBuilder->getIntNTy(BLOCK_SIZE)), block_shift);
392    mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
393    return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
394}
395   
396
397/* Methods for getting and setting carry summary values */
398   
399bool CarryManager::blockHasCarries(){
400    return mCarryInfo->blockHasCarries();
401} 
402
403
404Value * CarryManager::getCarrySummaryExpr() {
405    unsigned summary_posn = summaryPosition();
406#ifdef PACKING
407    return getCarryRange(summary_posn, getSummaryBits());
408#else
409    return getCarryPack(summary_posn);
410#endif
411}
412
413void CarryManager::addSummaryPhiIfNeeded(BasicBlock * ifEntryBlock, BasicBlock * ifBodyFinalBlock) {
414    if ((mCarryInfo->getIfDepth() <= 1) || !mCarryInfo->blockHasCarries()){
415        // For ifDepth == 1, the parent does not need a summary as it is not itself within an if.
416        // Therefore, it doesn't need access to this block's summary in building its own.
417        return;
418    }
419    const unsigned carrySummaryIndex = summaryPosition();
420    PHINode * summary_phi = mBuilder->CreatePHI(mBitBlockType, 2, "summary");
421    summary_phi->addIncoming(mZeroInitializer, ifEntryBlock);
422    summary_phi->addIncoming(mCarryOutPack[carrySummaryIndex], ifBodyFinalBlock);
423    mCarryOutPack[carrySummaryIndex] = summary_phi;
424}
425
426void CarryManager::generateCarryOutSummaryCodeIfNeeded() {
427   
428    if (!mCarryInfo->explicitSummaryRequired()) {
429        // An explicit summary may not be required, if there is a single carry
430        // operation within the block, or the carries are packed and all carry
431        // bits fit within a single pack.
432        return;
433    }
434   
435    const unsigned carrySummaryIndex = summaryPosition();
436   
437    Value * carry_summary = mZeroInitializer;
438   
439    if (mCarryInfo->blockHasLongAdvances()) { // Force if entry
440        carry_summary = mOneInitializer;
441    }
442    else {
443        auto localCarryIndex = mCurrentFrameIndex + mCarryInfo->getLocalCarryPackIndex();
444        auto localCarryPacks = mCarryInfo->getLocalCarryPackCount();
445        if (localCarryPacks > 0) {
446            carry_summary = mCarryOutPack[localCarryIndex];
447            for (auto i = 1; i < localCarryPacks; i++) {
448                //carry_summary = mBuilder->CreateOr(carry_summary, mPabloBlock->mCarryOutPack[i]);           
449                carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutPack[localCarryIndex+i]);
450            }
451        }
452        for (Statement * stmt : *mCurrentScope) {
453            if (If * innerIf = dyn_cast<If>(stmt)) {
454                PabloBlock * inner_blk = & innerIf->getBody();
455                enterScope(inner_blk);
456                if (blockHasCarries()) {
457                  carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutPack[summaryPosition()]);
458                }
459                leaveScope();
460            }
461            else if (While * innerWhile = dyn_cast<While>(stmt)) {
462                PabloBlock * inner_blk = & innerWhile->getBody();
463                enterScope(inner_blk);
464                if (blockHasCarries()) {
465                    carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutPack[summaryPosition()]);
466                }
467                leaveScope();
468            }
469        }
470    }
471    // Calculation of the carry out summary is complete.   Store it and make it
472    // available in case it must included by parent blocks.
473    mCarryOutPack[carrySummaryIndex] = carry_summary;
474    storeCarryPack(carrySummaryIndex);
475}
476
477
478
479void CarryManager::ensureCarriesLoadedRecursive() {
480    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
481    if (mCarryInfo->getWhileDepth() == 1) {
482        for (auto i = mCurrentFrameIndex; i < mCurrentFrameIndex + scopeCarryPacks; ++i) {
483            getCarryPack(i);
484        }
485    }
486}
487
488
489void CarryManager::initializeCarryDataPhisAtWhileEntry(BasicBlock * whileEntryBlock) {
490    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
491    mCarryOutAccumPhis.resize(scopeCarryPacks);
492#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
493    mCarryInPhis.resize(scopeCarryPacks);
494#endif
495    for (unsigned index = 0; index < scopeCarryPacks; ++index) {
496#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
497        PHINode * phi_in = mBuilder->CreatePHI(mBitBlockType, 2);
498        phi_in->addIncoming(mCarryInPack[mCurrentFrameIndex+index], whileEntryBlock);
499        mCarryInPhis[index] = phi_in;
500#endif
501        PHINode * phi_out = mBuilder->CreatePHI(mBitBlockType, 2);
502        phi_out->addIncoming(mZeroInitializer, whileEntryBlock);
503        mCarryOutAccumPhis[index] = phi_out;
504    }
505}
506
507
508void CarryManager::extendCarryDataPhisAtWhileBodyFinalBlock(BasicBlock * whileBodyFinalBlock) {
509    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
510    for (unsigned index = 0; index < scopeCarryPacks; ++index) {
511#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
512        mCarryInPhis[index]->addIncoming(mZeroInitializer, whileBodyFinalBlock);
513#endif
514        PHINode * phi = mCarryOutAccumPhis[index];
515        Value * carryOut = mBuilder->CreateOr(phi, mCarryOutPack[mCurrentFrameIndex+index]);
516        phi->addIncoming(carryOut, whileBodyFinalBlock);
517        mCarryOutPack[mCurrentFrameIndex+index] = carryOut;
518    }
519}
520
521void CarryManager::ensureCarriesStoredRecursive() {
522    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
523    if (mCarryInfo->getWhileDepth() == 1) {
524        for (auto i = mCurrentFrameIndex; i < mCurrentFrameIndex + scopeCarryPacks; ++i) {
525            storeCarryPack(i);
526        }
527    }
528}
529
530}
531
Note: See TracBrowser for help on using the repository browser.