source: icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp @ 4689

Last change on this file since 4689 was 4689, checked in by cameron, 4 years ago

Ensure frame positions of nested blocks can be adjusted based on carry data size for the block.

File size: 19.7 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <include/simd-lib/bitblock.hpp>
9#include <stdexcept>
10#include <pablo/carry_data.h>
11#include <pablo/codegenstate.h>
12#include <pablo/carry_manager.h>
13#include <pablo/pabloAST.h>
14#include <iostream>
15
16namespace pablo {
17 
18    unsigned doScopeCount(PabloBlock * pb) {
19        unsigned count = 1;
20       
21        for (Statement * stmt : *pb) {
22            if (If * ifStatement = dyn_cast<If>(stmt)) {
23                count += doScopeCount(&ifStatement->getBody());
24            }
25            else if (While * whileStatement = dyn_cast<While>(stmt)) {
26                count += doScopeCount(&whileStatement->getBody());
27            }
28        }
29        return count;
30       
31    }
32
33unsigned CarryManager::initialize(PabloBlock * pb, Value * carryPtr) {
34    mPabloRoot = pb;
35    mCurrentScope = pb;
36    mCurrentScopeIndex = 0;
37   
38    mCarryDataPtr = carryPtr;
39    unsigned scopeCount = doScopeCount(pb);
40    mCarryInfoVector.resize(scopeCount);
41   
42    unsigned totalCarryDataBits = enumerate(pb, 0, 0);
43   
44    mTotalCarryDataBitBlocks = (totalCarryDataBits + BLOCK_SIZE - 1)/BLOCK_SIZE + 1; // One extra element for the block no.
45    mBlockNoPtr = mBuilder->CreateBitCast(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(mTotalCarryDataBitBlocks - 1)), Type::getInt64PtrTy(mBuilder->getContext()));
46    mBlockNo = mBuilder->CreateLoad(mBlockNoPtr);
47    mCarryInVector.resize(mTotalCarryDataBitBlocks);
48    mCarryInPhis.resize(mTotalCarryDataBitBlocks);
49    mCarryOutAccumPhis.resize(mTotalCarryDataBitBlocks);
50    mCarryOutVector.resize(mTotalCarryDataBitBlocks);
51   
52    return mTotalCarryDataBitBlocks;
53}
54   
55void CarryManager::generateBlockNoIncrement() {
56    mBuilder->CreateStore(mBuilder->CreateAdd(mBlockNo, mBuilder->getInt64(1)), mBlockNoPtr);
57}
58
59Value * CarryManager::getBlockNoPtr() {
60    return mBlockNoPtr;
61}
62
63unsigned CarryManager::enumerate(PabloBlock * blk, unsigned ifDepth, unsigned whileDepth) {
64    llvm::raw_os_ostream cerr(std::cerr);
65    unsigned idx = blk->getScopeIndex();
66    PabloBlockCarryData * cd = new PabloBlockCarryData(blk);
67    mCarryInfoVector[idx] = cd;
68
69    cd->setIfDepth(ifDepth);
70    cd->setWhileDepth(whileDepth);
71    unsigned nestedOffset = cd->nested.frameOffsetinBits;
72 
73    for (Statement * stmt : *blk) {
74        if (If * ifStatement = dyn_cast<If>(stmt)) {
75            const unsigned ifCarryDataBits = enumerate(&ifStatement->getBody(), ifDepth+1, whileDepth);
76            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[ifStatement->getBody().getScopeIndex()];
77#ifdef PACKING
78            EnsurePackHasSpace(nestedOffset, ifCarryDataBits);
79#endif
80            nestedBlockData->setFramePosition(nestedOffset);
81
82            nestedOffset += ifCarryDataBits;
83            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
84            cd->nested.entries++;
85        }
86        else if (While * whileStatement = dyn_cast<While>(stmt)) {
87            const unsigned whileCarryDataBits = enumerate(&whileStatement->getBody(), ifDepth, whileDepth+1);
88            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[whileStatement->getBody().getScopeIndex()];
89            //if (whileStatement->isMultiCarry()) whileCarryDataBits *= whileStatement->getMaxIterations();
90#ifdef PACKING
91            EnsurePackHasSpace(nestedOffset, whileCarryDataBits);
92#endif
93            nestedBlockData->setFramePosition(nestedOffset);
94            nestedOffset += whileCarryDataBits;
95            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
96            cd->nested.entries++;
97        }
98    }
99   
100    cd->scopeCarryDataBits = nestedOffset;
101   
102    if (cd->explicitSummaryRequired()) {
103        // Need extra space for the summary variable, always the last
104        // entry within an if block.
105        cd->scopeCarryDataBits = alignCeiling(cd->scopeCarryDataBits, PACK_SIZE);
106        cd->summary.frameOffsetinBits = cd->scopeCarryDataBits;
107        cd->summary.allocatedBits = PACK_SIZE;
108        cd->scopeCarryDataBits += PACK_SIZE;
109    }
110    else {
111        cd->summary.frameOffsetinBits = 0;
112        cd->summary.allocatedBits = cd->scopeCarryDataBits;
113    }
114#ifndef NDEBUG
115    cd->dumpCarryData(cerr);
116#endif
117    return cd->scopeCarryDataBits;
118}
119
120
121/* Entering and leaving blocks. */
122
123void CarryManager::enterScope(PabloBlock * blk) {
124   
125    mCurrentScope = blk;
126    mCarryInfo = mCarryInfoVector[blk->getScopeIndex()];
127    mCurrentScopeIndex += mCarryInfo->getBlockCarryDataIndex();
128    //std::cerr << "enterScope:  mCurrentScopeIndex = " << mCurrentScopeIndex << std::endl;
129}
130
131void CarryManager::leaveScope() {
132    mCurrentScopeIndex -= mCarryInfo->getBlockCarryDataIndex();
133    mCurrentScope = mCurrentScope->getParent();
134    mCarryInfo = mCarryInfoVector[mCurrentScope->getScopeIndex()];
135    //std::cerr << "leaveScope:  mCurrentScopeIndex = " << mCurrentScopeIndex << std::endl;
136}
137
138    /* Methods for getting and setting individual carry values. */
139   
140//#define LOAD_STORE_ON_BLOCK_ENTRY_EXIT   
141Value * CarryManager::getCarryOpCarryIn(int localIndex) {
142    unsigned cd_index = mCurrentScopeIndex + mCarryInfo->carryOpCarryDataOffset(localIndex);
143#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
144    if (mCarryInfo->getWhileDepth() == 0) {
145       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd_index));
146       mCarryInVector[cd_index] = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
147    }
148#endif
149    return mCarryInVector[cd_index];
150}
151
152void CarryManager::setCarryOpCarryOut(unsigned localIndex, Value * carry_out) {
153    unsigned cd_index = mCurrentScopeIndex + mCarryInfo->carryOpCarryDataOffset(localIndex);
154    mCarryOutVector[cd_index] = carry_out;
155#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
156    if (mCarryInfo->getWhileDepth() == 0) {
157       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd_index));
158       mBuilder->CreateAlignedStore(carry_out, packPtr, BLOCK_SIZE/8);
159    }
160#endif
161}
162
163   
164Value * CarryManager::advanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
165    if (shift_amount == 1) {
166        return unitAdvanceCarryInCarryOut(localIndex, strm);
167    }
168    else if (shift_amount < LongAdvanceBase) {
169        return shortAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
170    }
171    else {
172        return longAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
173    }
174}
175
176Value * CarryManager::unitAdvanceCarryInCarryOut(int localIndex, Value * strm) {
177    unsigned carryDataIndex = mCurrentScopeIndex + mCarryInfo->unitAdvanceCarryDataOffset(localIndex);
178    mCarryOutVector[carryDataIndex] = strm; 
179#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
180    if (mCarryInfo->getWhileDepth() == 0) {
181        Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(carryDataIndex));
182        mCarryInVector[carryDataIndex] = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
183        mBuilder->CreateAlignedStore(strm, packPtr, BLOCK_SIZE/8);
184    }
185#endif
186    Value * carry_in = mCarryInVector[carryDataIndex];
187    Value* result_value;
188   
189#if (BLOCK_SIZE == 128) && !defined(USE_LONG_INTEGER_SHIFT)
190    Value * ahead64 = iBuilder->mvmd_dslli(64, carry_in, strm, 1);
191    result_value = mBuilder->CreateOr(iBuilder->simd_srli(64, ahead64, 63), iBuilder->simd_slli(64, strm, 1));
192#else
193    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
194    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
195    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - 1), "advance");
196    result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
197   
198#endif
199    return result_value;
200}
201
202Value * CarryManager::shortAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
203    unsigned carryDataIndex = mCurrentScopeIndex + mCarryInfo->shortAdvanceCarryDataOffset(localIndex);
204    mCarryOutVector[carryDataIndex] = strm; 
205#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
206    if (mCarryInfo->getWhileDepth() == 0) {
207        Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(carryDataIndex));
208        mCarryInVector[carryDataIndex] = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
209        mBuilder->CreateAlignedStore(strm, packPtr, BLOCK_SIZE/8);
210    }
211#endif
212    Value * carry_in = mCarryInVector[carryDataIndex];
213    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
214    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
215    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, shift_amount), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - shift_amount), "advance");
216    return mBuilder->CreateBitCast(adv_longint, mBitBlockType);
217}
218   
219
220/*  currently defined in carry_data.h
221 
222 static unsigned power2ceil (unsigned v) {
223 unsigned ceil = 1;
224 while (ceil < v) ceil *= 2;
225 return ceil;
226 }
227 
228 unsigned longAdvanceEntries(unsigned shift_amount) const {
229 return (shift_amount + BLOCK_SIZE - 1)/BLOCK_SIZE;
230 }
231 
232 unsigned longAdvanceBufferSize(unsigned shift_amount)  const {
233 return power2ceil(longAdvanceEntries(shift_amount));
234 }
235 */
236
237   
238Value * CarryManager::longAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * carry_out) {
239    unsigned carryDataIndex = mCurrentScopeIndex + mCarryInfo->longAdvanceCarryDataOffset(localIndex);
240    Value * advBaseIndex = mBuilder->getInt64(carryDataIndex);
241    if (shift_amount <= BLOCK_SIZE) {
242        // special case using a single buffer entry and the carry_out value.
243        Value * advanceDataPtr = mBuilder->CreateGEP(mCarryDataPtr, advBaseIndex);
244        Value * carry_block0 = mBuilder->CreateAlignedLoad(advanceDataPtr, BLOCK_SIZE/8);
245        mBuilder->CreateAlignedStore(carry_out, advanceDataPtr, BLOCK_SIZE/8);
246        /* Very special case - no combine */
247        if (shift_amount == BLOCK_SIZE) return carry_block0;
248        Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - shift_amount);
249        Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_out, mBuilder->getIntNTy(BLOCK_SIZE)), shift_amount);
250        return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
251    }
252    // We need a buffer of at least two elements for storing the advance data.
253    const unsigned block_shift = shift_amount % BLOCK_SIZE;
254    const unsigned advanceEntries = mCarryInfo->longAdvanceEntries(shift_amount);
255    const unsigned bufsize = mCarryInfo->longAdvanceBufferSize(shift_amount);
256    Value * indexMask = mBuilder->getInt64(bufsize - 1);  // A mask to implement circular buffer indexing
257    Value * loadIndex0 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries)), indexMask), advBaseIndex);
258    Value * storeIndex = mBuilder->CreateAdd(mBuilder->CreateAnd(mBlockNo, indexMask), advBaseIndex);
259    Value * carry_block0 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex0), BLOCK_SIZE/8);
260    // If the long advance is an exact multiple of BLOCK_SIZE, we simply return the oldest
261    // block in the long advance carry data area. 
262    if (block_shift == 0) {
263        mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
264        return carry_block0;
265    }
266    // Otherwise we need to combine data from the two oldest blocks.
267    Value * loadIndex1 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries-1)), indexMask), advBaseIndex);
268    Value * carry_block1 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex1), BLOCK_SIZE/8);
269    Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - block_shift);
270    Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_block1, mBuilder->getIntNTy(BLOCK_SIZE)), block_shift);
271    mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
272    return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
273}
274   
275
276/* Methods for getting and setting carry summary values */
277   
278bool CarryManager::blockHasCarries(){
279    return mCarryInfo->blockHasCarries();
280} 
281
282
283Value * CarryManager::getCarrySummaryExpr() {
284    unsigned summary_idx = mCurrentScopeIndex + mCarryInfo->summaryCarryDataIndex();
285    Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(summary_idx));
286    Value * summary_expr = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
287    // If the scopeCarryDataSize is 1, then the carry summary is also the pack expr.
288    mCarryInVector[summary_idx] = summary_expr;
289    return summary_expr;
290}
291
292void CarryManager::addSummaryPhiIfNeeded(BasicBlock * ifEntryBlock, BasicBlock * ifBodyFinalBlock) {
293    if ((mCarryInfo->getIfDepth() <= 1) || !mCarryInfo->blockHasCarries()){
294        // For ifDepth == 1, the parent does not need a summary as it is not itself within an if.
295        // Therefore, it doesn't need access to this block's summary in building its own.
296        return;
297    }
298    const unsigned carrySummaryIndex = mCurrentScopeIndex + mCarryInfo->summaryCarryDataIndex();
299    PHINode * summary_phi = mBuilder->CreatePHI(mBitBlockType, 2, "summary");
300    summary_phi->addIncoming(mZeroInitializer, ifEntryBlock);
301    summary_phi->addIncoming(mCarryOutVector[carrySummaryIndex], ifBodyFinalBlock);
302    mCarryOutVector[carrySummaryIndex] = summary_phi;
303}
304
305void CarryManager::generateCarryOutSummaryCodeIfNeeded() {
306   
307    if (!mCarryInfo->explicitSummaryRequired()) {
308        // An explicit summary may not be required, if there is a single carry
309        // operation within the block, or the carries are packed and all carry
310        // bits fit within a single pack.
311        return;
312    }
313   
314    const unsigned carrySummaryIndex = mCurrentScopeIndex + mCarryInfo->summaryCarryDataIndex();
315   
316    Value * carry_summary = mZeroInitializer;
317   
318    if (mCarryInfo->blockHasLongAdvances()) { // Force if entry
319        carry_summary = mOneInitializer;
320    }
321    else {
322        auto localCarryIndex = mCurrentScopeIndex + mCarryInfo->getLocalCarryPackIndex();
323        auto localCarryPacks = mCarryInfo->getLocalCarryDataSize();
324        if (localCarryPacks > 0) {
325            carry_summary = mCarryOutVector[localCarryIndex];
326            for (auto i = 1; i < localCarryPacks; i++) {
327                //carry_summary = mBuilder->CreateOr(carry_summary, mPabloBlock->mCarryOutPack[i]);           
328                carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[localCarryIndex+i]);
329            }
330        }
331        for (Statement * stmt : *mCurrentScope) {
332            if (If * innerIf = dyn_cast<If>(stmt)) {
333                PabloBlock * inner_blk = & innerIf->getBody();
334                enterScope(inner_blk);
335                if (blockHasCarries()) {
336                  carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[mCurrentScopeIndex + mCarryInfo->summaryCarryDataIndex()]);
337                }
338                leaveScope();
339            }
340            else if (While * innerWhile = dyn_cast<While>(stmt)) {
341                PabloBlock * inner_blk = & innerWhile->getBody();
342                enterScope(inner_blk);
343                if (blockHasCarries()) {
344                    carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[mCurrentScopeIndex + mCarryInfo->summaryCarryDataIndex()]);
345                }
346                leaveScope();
347            }
348        }
349    }
350    // Calculation of the carry out summary is complete.   Store it and make it
351    // available in case it must included by parent blocks.
352    mCarryOutVector[carrySummaryIndex] = carry_summary;
353    Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(carrySummaryIndex));
354    mBuilder->CreateAlignedStore(carry_summary, packPtr, BLOCK_SIZE/8);
355}
356
357
358void CarryManager::ensureCarriesLoadedLocal() {
359#ifdef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
360    if ((mCarryInfo->getScopeCarryDataSize() == 0 ) || (mCarryInfo->getWhileDepth() > 0)) return;
361    if ((mCarryInfo->getIfDepth() == 0) || mCarryInfo->explicitSummaryRequired()) {
362        auto localCarryIndex = mCurrentScopeIndex + mCarryInfo->getLocalCarryPackIndex();
363        auto localCarryPacks = mCarryInfo->getLocalCarryDataSize();
364        //std::cerr << "ensureCarriesLoadedLocal: localCarryIndex =  " << localCarryIndex << "localCarryPacks =  " << localCarryPacks << std::endl;
365        for (auto i = localCarryIndex; i < localCarryIndex + localCarryPacks; i++) {       
366            mCarryInVector[i] = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i)), BLOCK_SIZE/8, false);
367        }
368    }
369#endif
370}
371
372void CarryManager::ensureCarriesStoredLocal() {
373#ifdef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
374    if ((mCarryInfo->getScopeCarryDataSize() == 0 ) || (mCarryInfo->getWhileDepth() > 0)) return;
375    auto localCarryIndex = mCurrentScopeIndex + mCarryInfo->getLocalCarryPackIndex();
376    auto localCarryPacks = mCarryInfo->getLocalCarryDataSize();
377    //std::cerr << "ensureCarriesStoredLocal: localCarryIndex =  " << localCarryIndex << "localCarryPacks =  " << localCarryPacks << std::endl;
378    for (auto i = localCarryIndex; i < localCarryIndex + localCarryPacks; i++) {       
379        Value * storePtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i));
380        mBuilder->CreateAlignedStore(mCarryOutVector[i], storePtr, BLOCK_SIZE/8, false);
381    }
382#endif
383}
384
385
386
387void CarryManager::ensureCarriesLoadedRecursive() {
388    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
389    if (mCarryInfo->getWhileDepth() == 1) {
390        for (auto i = mCurrentScopeIndex; i < mCurrentScopeIndex + scopeCarryDataSize; ++i) {
391            mCarryInVector[i] = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i)), BLOCK_SIZE/8, false);
392        }
393    }
394}
395
396
397void CarryManager::initializeCarryDataPhisAtWhileEntry(BasicBlock * whileEntryBlock) {
398    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
399    for (unsigned index = mCurrentScopeIndex; index < mCurrentScopeIndex + scopeCarryDataSize; ++index) {
400#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
401        PHINode * phi_in = mBuilder->CreatePHI(mBitBlockType, 2);
402        phi_in->addIncoming(mCarryInVector[index], whileEntryBlock);
403        mCarryInPhis[index] = phi_in;
404#endif
405        PHINode * phi_out = mBuilder->CreatePHI(mBitBlockType, 2);
406        phi_out->addIncoming(mZeroInitializer, whileEntryBlock);
407        mCarryOutAccumPhis[index] = phi_out;
408    }
409}
410
411
412void CarryManager::extendCarryDataPhisAtWhileBodyFinalBlock(BasicBlock * whileBodyFinalBlock) {
413    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
414    for (unsigned index = mCurrentScopeIndex; index < mCurrentScopeIndex + scopeCarryDataSize; ++index) {
415#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
416        mCarryInPhis[index]->addIncoming(mZeroInitializer, whileBodyFinalBlock);
417#endif
418        PHINode * phi = mCarryOutAccumPhis[index];
419        Value * carryOut = mBuilder->CreateOr(phi, mCarryOutVector[index]);
420        phi->addIncoming(carryOut, whileBodyFinalBlock);
421        mCarryOutVector[index] = carryOut;
422    }
423}
424
425void CarryManager::ensureCarriesStoredRecursive() {
426    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
427    if (mCarryInfo->getWhileDepth() == 1) {
428        for (auto i = mCurrentScopeIndex; i < mCurrentScopeIndex + scopeCarryDataSize; ++i) {
429            Value * storePtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i));
430            mBuilder->CreateAlignedStore(mCarryOutVector[i], storePtr, BLOCK_SIZE/8, false);
431        }
432    }
433}
434
435}
436
Note: See TracBrowser for help on using the repository browser.