source: icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp @ 4694

Last change on this file since 4694 was 4694, checked in by cameron, 4 years ago

Carry manager refactoring, progress towards bit packing

File size: 19.8 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <include/simd-lib/bitblock.hpp>
9#include <stdexcept>
10#include <pablo/carry_data.h>
11#include <pablo/codegenstate.h>
12#include <pablo/carry_manager.h>
13#include <pablo/pabloAST.h>
14#include <iostream>
15
16namespace pablo {
17 
18    unsigned doScopeCount(PabloBlock * pb) {
19        unsigned count = 1;
20       
21        for (Statement * stmt : *pb) {
22            if (If * ifStatement = dyn_cast<If>(stmt)) {
23                count += doScopeCount(&ifStatement->getBody());
24            }
25            else if (While * whileStatement = dyn_cast<While>(stmt)) {
26                count += doScopeCount(&whileStatement->getBody());
27            }
28        }
29        return count;
30       
31    }
32
33unsigned CarryManager::initialize(PabloBlock * pb, Value * carryPtr) {
34    mPabloRoot = pb;
35    mCarryDataPtr = carryPtr;
36    unsigned scopeCount = doScopeCount(pb);
37    mCarryInfoVector.resize(scopeCount);
38   
39    unsigned totalCarryDataBits = enumerate(pb, 0, 0);
40   
41    mTotalCarryDataBitBlocks = (totalCarryDataBits + BLOCK_SIZE - 1)/BLOCK_SIZE; 
42    // Carry Data area will have one extra bit block to store the block number.
43    mBlockNoPtr = mBuilder->CreateBitCast(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(mTotalCarryDataBitBlocks)), Type::getInt64PtrTy(mBuilder->getContext()));
44    mBlockNo = mBuilder->CreateLoad(mBlockNoPtr);
45    mCarryPackPtr.resize(mTotalCarryDataBitBlocks);
46    mCarryInPack.resize(mTotalCarryDataBitBlocks);
47    mCarryInPhis.resize(mTotalCarryDataBitBlocks);
48    mCarryOutAccumPhis.resize(mTotalCarryDataBitBlocks);
49    mCarryOutVector.resize(mTotalCarryDataBitBlocks);
50    for (auto i = 0; i < mTotalCarryDataBitBlocks; i++) mCarryInPack[i]=nullptr;
51   
52    /*  Set the current scope to PabloRoot */
53    mCurrentScope = mPabloRoot;
54    mCurrentFrameIndex = 0;
55    mCarryInfo = mCarryInfoVector[0];
56
57    return mTotalCarryDataBitBlocks + 1; // One extra element for the block no.
58}
59   
60void CarryManager::generateBlockNoIncrement() {
61    mBuilder->CreateStore(mBuilder->CreateAdd(mBlockNo, mBuilder->getInt64(1)), mBlockNoPtr);
62}
63
64Value * CarryManager::getBlockNoPtr() {
65    return mBlockNoPtr;
66}
67
68
69unsigned CarryManager::enumerate(PabloBlock * blk, unsigned ifDepth, unsigned whileDepth) {
70    llvm::raw_os_ostream cerr(std::cerr);
71    unsigned idx = blk->getScopeIndex();
72    PabloBlockCarryData * cd = new PabloBlockCarryData(blk);
73    mCarryInfoVector[idx] = cd;
74
75    cd->setIfDepth(ifDepth);
76    cd->setWhileDepth(whileDepth);
77    unsigned nestedOffset = cd->nested.frameOffsetinBits;
78 
79    for (Statement * stmt : *blk) {
80        if (If * ifStatement = dyn_cast<If>(stmt)) {
81            const unsigned ifCarryDataBits = enumerate(&ifStatement->getBody(), ifDepth+1, whileDepth);
82            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[ifStatement->getBody().getScopeIndex()];
83#ifdef PACKING
84            EnsurePackHasSpace(nestedOffset, ifCarryDataBits);
85#endif
86            nestedBlockData->setFramePosition(nestedOffset);
87
88            nestedOffset += ifCarryDataBits;
89            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
90            cd->nested.entries++;
91        }
92        else if (While * whileStatement = dyn_cast<While>(stmt)) {
93            const unsigned whileCarryDataBits = enumerate(&whileStatement->getBody(), ifDepth, whileDepth+1);
94            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[whileStatement->getBody().getScopeIndex()];
95            //if (whileStatement->isMultiCarry()) whileCarryDataBits *= whileStatement->getMaxIterations();
96#ifdef PACKING
97            EnsurePackHasSpace(nestedOffset, whileCarryDataBits);
98#endif
99            nestedBlockData->setFramePosition(nestedOffset);
100            nestedOffset += whileCarryDataBits;
101            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
102            cd->nested.entries++;
103        }
104    }
105   
106    cd->scopeCarryDataBits = nestedOffset;
107   
108    if (cd->explicitSummaryRequired()) {
109        // Need extra space for the summary variable, always the last
110        // entry within an if block.
111        cd->scopeCarryDataBits = alignCeiling(cd->scopeCarryDataBits, PACK_SIZE);
112        cd->summary.frameOffsetinBits = cd->scopeCarryDataBits;
113        cd->summary.allocatedBits = PACK_SIZE;
114        cd->scopeCarryDataBits += PACK_SIZE;
115    }
116    else {
117        cd->summary.frameOffsetinBits = 0;
118        cd->summary.allocatedBits = cd->scopeCarryDataBits;
119    }
120#ifndef NDEBUG
121    cd->dumpCarryData(cerr);
122#endif
123    return cd->scopeCarryDataBits;
124}
125
126
127/* Entering and leaving blocks. */
128
129void CarryManager::enterScope(PabloBlock * blk) {
130   
131    mCurrentScope = blk;
132    mCarryInfo = mCarryInfoVector[blk->getScopeIndex()];
133    mCurrentFrameIndex += mCarryInfo->getFrameIndex();
134    //std::cerr << "enterScope:  mCurrentFrameIndex = " << mCurrentFrameIndex << std::endl;
135}
136
137void CarryManager::leaveScope() {
138    mCurrentFrameIndex -= mCarryInfo->getFrameIndex();
139    mCurrentScope = mCurrentScope->getParent();
140    mCarryInfo = mCarryInfoVector[mCurrentScope->getScopeIndex()];
141    //std::cerr << "leaveScope:  mCurrentFrameIndex = " << mCurrentFrameIndex << std::endl;
142}
143
144
145/* Helper routines */
146
147
148Value * CarryManager::getCarryPack(unsigned packIndex) {
149    if (mCarryInPack[packIndex] == nullptr) {
150        Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(packIndex));
151        mCarryPackPtr[packIndex] = packPtr;
152        mCarryInPack[packIndex] = mBuilder->CreateAlignedLoad(packPtr, PACK_SIZE/8);
153    }
154    return mCarryInPack[packIndex];
155}
156
157void CarryManager::CarryPackStore(unsigned packIndex) {
158    mBuilder->CreateAlignedStore(mCarryOutVector[packIndex], mCarryPackPtr[packIndex], PACK_SIZE/8);
159}
160
161Value * CarryManager::genCarryInRange(unsigned carryBit_lo, unsigned carryRangeSize) {
162
163    unsigned packIndex = carryBit_lo / PACK_SIZE;
164    unsigned carryOffset = carryBit_lo % PACK_SIZE;
165    unsigned hiOffset = carryBit_lo + carryRangeSize - 1;
166   
167    Value * carryItem = getCarryPack(packIndex);
168    if (carryRangeSize < PACK_SIZE) {
169       carryItem = mBuilder->CreateAnd(carryItem, mBuilder->getInt64((1 << hiOffset) - 1));
170    }
171    if (carryOffset > 0) {
172       carryItem = mBuilder->CreateLShr(carryItem, mBuilder->getInt64(carryOffset));
173    }
174    return carryItem;
175}
176   
177 
178Value * CarryManager::genCarryInBit(unsigned carryBitPos) {
179    return genCarryInRange(carryBitPos, 1);
180}
181    /* Methods for getting and setting individual carry values. */
182   
183//#define LOAD_STORE_ON_BLOCK_ENTRY_EXIT   
184Value * CarryManager::getCarryOpCarryIn(int localIndex) {
185    unsigned cd_index = mCurrentFrameIndex + mCarryInfo->carryOpCarryDataOffset(localIndex);
186    return getCarryPack(cd_index);
187}
188
189void CarryManager::setCarryOpCarryOut(unsigned localIndex, Value * carry_out) {
190    unsigned cd_index = mCurrentFrameIndex + mCarryInfo->carryOpCarryDataOffset(localIndex);
191    mCarryOutVector[cd_index] = carry_out;
192#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
193    if (mCarryInfo->getWhileDepth() == 0) {
194        CarryPackStore(cd_index);
195    }
196#endif
197}
198
199   
200Value * CarryManager::advanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
201    if (shift_amount == 1) {
202        return unitAdvanceCarryInCarryOut(localIndex, strm);
203    }
204    else if (shift_amount < LongAdvanceBase) {
205        return shortAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
206    }
207    else {
208        return longAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
209    }
210}
211
212Value * CarryManager::unitAdvanceCarryInCarryOut(int localIndex, Value * strm) {
213   
214    unsigned carryDataIndex = mCurrentFrameIndex + mCarryInfo->unitAdvanceCarryDataOffset(localIndex);
215    mCarryOutVector[carryDataIndex] = strm; 
216    Value * carry_in = getCarryPack(carryDataIndex);
217#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
218    if (mCarryInfo->getWhileDepth() == 0) {
219        CarryPackStore(carryDataIndex);
220    }
221#endif
222    Value* result_value;
223   
224#if (BLOCK_SIZE == 128) && !defined(USE_LONG_INTEGER_SHIFT)
225    Value * ahead64 = iBuilder->mvmd_dslli(64, carry_in, strm, 1);
226    result_value = mBuilder->CreateOr(iBuilder->simd_srli(64, ahead64, 63), iBuilder->simd_slli(64, strm, 1));
227#else
228    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
229    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
230    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - 1), "advance");
231    result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
232   
233#endif
234    return result_value;
235}
236
237Value * CarryManager::shortAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
238    unsigned carryDataIndex = mCurrentFrameIndex + mCarryInfo->shortAdvanceCarryDataOffset(localIndex);
239    mCarryOutVector[carryDataIndex] = strm; 
240    Value * carry_in = getCarryPack(carryDataIndex);
241#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
242    if (mCarryInfo->getWhileDepth() == 0) {
243        CarryPackStore(carryDataIndex);
244    }
245#endif
246    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
247    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
248    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, shift_amount), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - shift_amount), "advance");
249    return mBuilder->CreateBitCast(adv_longint, mBitBlockType);
250}
251   
252
253/*  currently defined in carry_data.h
254 
255 static unsigned power2ceil (unsigned v) {
256 unsigned ceil = 1;
257 while (ceil < v) ceil *= 2;
258 return ceil;
259 }
260 
261 unsigned longAdvanceEntries(unsigned shift_amount) const {
262 return (shift_amount + BLOCK_SIZE - 1)/BLOCK_SIZE;
263 }
264 
265 unsigned longAdvanceBufferSize(unsigned shift_amount)  const {
266 return power2ceil(longAdvanceEntries(shift_amount));
267 }
268 */
269
270   
271Value * CarryManager::longAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * carry_out) {
272    unsigned carryDataIndex = mCurrentFrameIndex + mCarryInfo->longAdvanceCarryDataOffset(localIndex);
273    Value * advBaseIndex = mBuilder->getInt64(carryDataIndex);
274    if (shift_amount <= BLOCK_SIZE) {
275        // special case using a single buffer entry and the carry_out value.
276        Value * advanceDataPtr = mBuilder->CreateGEP(mCarryDataPtr, advBaseIndex);
277        Value * carry_block0 = mBuilder->CreateAlignedLoad(advanceDataPtr, BLOCK_SIZE/8);
278        mBuilder->CreateAlignedStore(carry_out, advanceDataPtr, BLOCK_SIZE/8);
279        /* Very special case - no combine */
280        if (shift_amount == BLOCK_SIZE) return carry_block0;
281        Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - shift_amount);
282        Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_out, mBuilder->getIntNTy(BLOCK_SIZE)), shift_amount);
283        return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
284    }
285    // We need a buffer of at least two elements for storing the advance data.
286    const unsigned block_shift = shift_amount % BLOCK_SIZE;
287    const unsigned advanceEntries = mCarryInfo->longAdvanceEntries(shift_amount);
288    const unsigned bufsize = mCarryInfo->longAdvanceBufferSize(shift_amount);
289    Value * indexMask = mBuilder->getInt64(bufsize - 1);  // A mask to implement circular buffer indexing
290    Value * loadIndex0 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries)), indexMask), advBaseIndex);
291    Value * storeIndex = mBuilder->CreateAdd(mBuilder->CreateAnd(mBlockNo, indexMask), advBaseIndex);
292    Value * carry_block0 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex0), BLOCK_SIZE/8);
293    // If the long advance is an exact multiple of BLOCK_SIZE, we simply return the oldest
294    // block in the long advance carry data area. 
295    if (block_shift == 0) {
296        mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
297        return carry_block0;
298    }
299    // Otherwise we need to combine data from the two oldest blocks.
300    Value * loadIndex1 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries-1)), indexMask), advBaseIndex);
301    Value * carry_block1 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex1), BLOCK_SIZE/8);
302    Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - block_shift);
303    Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_block1, mBuilder->getIntNTy(BLOCK_SIZE)), block_shift);
304    mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
305    return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
306}
307   
308
309/* Methods for getting and setting carry summary values */
310   
311bool CarryManager::blockHasCarries(){
312    return mCarryInfo->blockHasCarries();
313} 
314
315
316Value * CarryManager::getCarrySummaryExpr() {
317    unsigned summary_idx = mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex();
318    return getCarryPack(summary_idx);
319}
320
321void CarryManager::addSummaryPhiIfNeeded(BasicBlock * ifEntryBlock, BasicBlock * ifBodyFinalBlock) {
322    if ((mCarryInfo->getIfDepth() <= 1) || !mCarryInfo->blockHasCarries()){
323        // For ifDepth == 1, the parent does not need a summary as it is not itself within an if.
324        // Therefore, it doesn't need access to this block's summary in building its own.
325        return;
326    }
327    const unsigned carrySummaryIndex = mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex();
328    PHINode * summary_phi = mBuilder->CreatePHI(mBitBlockType, 2, "summary");
329    summary_phi->addIncoming(mZeroInitializer, ifEntryBlock);
330    summary_phi->addIncoming(mCarryOutVector[carrySummaryIndex], ifBodyFinalBlock);
331    mCarryOutVector[carrySummaryIndex] = summary_phi;
332}
333
334void CarryManager::generateCarryOutSummaryCodeIfNeeded() {
335   
336    if (!mCarryInfo->explicitSummaryRequired()) {
337        // An explicit summary may not be required, if there is a single carry
338        // operation within the block, or the carries are packed and all carry
339        // bits fit within a single pack.
340        return;
341    }
342   
343    const unsigned carrySummaryIndex = mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex();
344   
345    Value * carry_summary = mZeroInitializer;
346   
347    if (mCarryInfo->blockHasLongAdvances()) { // Force if entry
348        carry_summary = mOneInitializer;
349    }
350    else {
351        auto localCarryIndex = mCurrentFrameIndex + mCarryInfo->getLocalCarryPackIndex();
352        auto localCarryPacks = mCarryInfo->getLocalCarryDataSize();
353        if (localCarryPacks > 0) {
354            carry_summary = mCarryOutVector[localCarryIndex];
355            for (auto i = 1; i < localCarryPacks; i++) {
356                //carry_summary = mBuilder->CreateOr(carry_summary, mPabloBlock->mCarryOutPack[i]);           
357                carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[localCarryIndex+i]);
358            }
359        }
360        for (Statement * stmt : *mCurrentScope) {
361            if (If * innerIf = dyn_cast<If>(stmt)) {
362                PabloBlock * inner_blk = & innerIf->getBody();
363                enterScope(inner_blk);
364                if (blockHasCarries()) {
365                  carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex()]);
366                }
367                leaveScope();
368            }
369            else if (While * innerWhile = dyn_cast<While>(stmt)) {
370                PabloBlock * inner_blk = & innerWhile->getBody();
371                enterScope(inner_blk);
372                if (blockHasCarries()) {
373                    carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex()]);
374                }
375                leaveScope();
376            }
377        }
378    }
379    // Calculation of the carry out summary is complete.   Store it and make it
380    // available in case it must included by parent blocks.
381    mCarryOutVector[carrySummaryIndex] = carry_summary;
382    CarryPackStore(carrySummaryIndex);
383}
384
385
386void CarryManager::ensureCarriesLoadedLocal() {
387#ifdef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
388    if ((mCarryInfo->getScopeCarryDataSize() == 0 ) || (mCarryInfo->getWhileDepth() > 0)) return;
389    if ((mCarryInfo->getIfDepth() == 0) || mCarryInfo->explicitSummaryRequired()) {
390        auto localCarryIndex = mCurrentFrameIndex + mCarryInfo->getLocalCarryPackIndex();
391        auto localCarryPacks = mCarryInfo->getLocalCarryDataSize();
392        //std::cerr << "ensureCarriesLoadedLocal: localCarryIndex =  " << localCarryIndex << "localCarryPacks =  " << localCarryPacks << std::endl;
393        for (auto i = localCarryIndex; i < localCarryIndex + localCarryPacks; i++) {       
394            mCarryInPack[i] = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i)), BLOCK_SIZE/8, false);
395        }
396    }
397#endif
398}
399
400void CarryManager::ensureCarriesStoredLocal() {
401#ifdef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
402    if ((mCarryInfo->getScopeCarryDataSize() == 0 ) || (mCarryInfo->getWhileDepth() > 0)) return;
403    auto localCarryIndex = mCurrentFrameIndex + mCarryInfo->getLocalCarryPackIndex();
404    auto localCarryPacks = mCarryInfo->getLocalCarryDataSize();
405    //std::cerr << "ensureCarriesStoredLocal: localCarryIndex =  " << localCarryIndex << "localCarryPacks =  " << localCarryPacks << std::endl;
406    for (auto i = localCarryIndex; i < localCarryIndex + localCarryPacks; i++) {       
407        Value * storePtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i));
408        mBuilder->CreateAlignedStore(mCarryOutVector[i], storePtr, BLOCK_SIZE/8, false);
409    }
410#endif
411}
412
413
414
415void CarryManager::ensureCarriesLoadedRecursive() {
416    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
417    if (mCarryInfo->getWhileDepth() == 1) {
418        for (auto i = mCurrentFrameIndex; i < mCurrentFrameIndex + scopeCarryDataSize; ++i) {
419            getCarryPack(i);
420        }
421    }
422}
423
424
425void CarryManager::initializeCarryDataPhisAtWhileEntry(BasicBlock * whileEntryBlock) {
426    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
427    for (unsigned index = mCurrentFrameIndex; index < mCurrentFrameIndex + scopeCarryDataSize; ++index) {
428#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
429        PHINode * phi_in = mBuilder->CreatePHI(mBitBlockType, 2);
430        phi_in->addIncoming(mCarryInPack[index], whileEntryBlock);
431        mCarryInPhis[index] = phi_in;
432#endif
433        PHINode * phi_out = mBuilder->CreatePHI(mBitBlockType, 2);
434        phi_out->addIncoming(mZeroInitializer, whileEntryBlock);
435        mCarryOutAccumPhis[index] = phi_out;
436    }
437}
438
439
440void CarryManager::extendCarryDataPhisAtWhileBodyFinalBlock(BasicBlock * whileBodyFinalBlock) {
441    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
442    for (unsigned index = mCurrentFrameIndex; index < mCurrentFrameIndex + scopeCarryDataSize; ++index) {
443#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
444        mCarryInPhis[index]->addIncoming(mZeroInitializer, whileBodyFinalBlock);
445#endif
446        PHINode * phi = mCarryOutAccumPhis[index];
447        assert (phi);
448        assert (mCarryOutVector[index]);
449        Value * carryOut = mBuilder->CreateOr(phi, mCarryOutVector[index]);
450        phi->addIncoming(carryOut, whileBodyFinalBlock);
451        mCarryOutVector[index] = carryOut;
452    }
453}
454
455void CarryManager::ensureCarriesStoredRecursive() {
456    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
457    if (mCarryInfo->getWhileDepth() == 1) {
458        for (auto i = mCurrentFrameIndex; i < mCurrentFrameIndex + scopeCarryDataSize; ++i) {
459            CarryPackStore(i);
460        }
461    }
462}
463
464}
465
Note: See TracBrowser for help on using the repository browser.