source: icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp @ 4692

Last change on this file since 4692 was 4692, checked in by nmedfort, 4 years ago

Temporary check in.

File size: 19.9 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <include/simd-lib/bitblock.hpp>
9#include <stdexcept>
10#include <pablo/carry_data.h>
11#include <pablo/codegenstate.h>
12#include <pablo/carry_manager.h>
13#include <pablo/pabloAST.h>
14#include <iostream>
15
16namespace pablo {
17 
18    unsigned doScopeCount(PabloBlock * pb) {
19        unsigned count = 1;
20       
21        for (Statement * stmt : *pb) {
22            if (If * ifStatement = dyn_cast<If>(stmt)) {
23                count += doScopeCount(&ifStatement->getBody());
24            }
25            else if (While * whileStatement = dyn_cast<While>(stmt)) {
26                count += doScopeCount(&whileStatement->getBody());
27            }
28        }
29        return count;
30       
31    }
32
33unsigned CarryManager::initialize(PabloBlock * pb, Value * carryPtr) {
34    mPabloRoot = pb;
35    mCarryDataPtr = carryPtr;
36    unsigned scopeCount = doScopeCount(pb);
37    mCarryInfoVector.resize(scopeCount);
38   
39    unsigned totalCarryDataBits = enumerate(pb, 0, 0);
40   
41    mTotalCarryDataBitBlocks = (totalCarryDataBits + BLOCK_SIZE - 1)/BLOCK_SIZE + 1; // One extra element for the block no.
42    mBlockNoPtr = mBuilder->CreateBitCast(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(mTotalCarryDataBitBlocks - 1)), Type::getInt64PtrTy(mBuilder->getContext()));
43    mBlockNo = mBuilder->CreateLoad(mBlockNoPtr);
44    mCarryInVector.resize(mTotalCarryDataBitBlocks);
45    mCarryInPhis.resize(mTotalCarryDataBitBlocks);
46    mCarryOutAccumPhis.resize(mTotalCarryDataBitBlocks);
47    mCarryOutVector.resize(mTotalCarryDataBitBlocks);
48   
49    /*  Set the current scope to PabloRoot */
50    mCurrentScope = mPabloRoot;
51    mCurrentFrameIndex = 0;
52    mCarryInfo = mCarryInfoVector[0];
53
54    return mTotalCarryDataBitBlocks;
55}
56   
57void CarryManager::generateBlockNoIncrement() {
58    mBuilder->CreateStore(mBuilder->CreateAdd(mBlockNo, mBuilder->getInt64(1)), mBlockNoPtr);
59}
60
61Value * CarryManager::getBlockNoPtr() {
62    return mBlockNoPtr;
63}
64
65unsigned CarryManager::enumerate(PabloBlock * blk, unsigned ifDepth, unsigned whileDepth) {
66    llvm::raw_os_ostream cerr(std::cerr);
67    unsigned idx = blk->getScopeIndex();
68    PabloBlockCarryData * cd = new PabloBlockCarryData(blk);
69    mCarryInfoVector[idx] = cd;
70
71    cd->setIfDepth(ifDepth);
72    cd->setWhileDepth(whileDepth);
73    unsigned nestedOffset = cd->nested.frameOffsetinBits;
74 
75    for (Statement * stmt : *blk) {
76        if (If * ifStatement = dyn_cast<If>(stmt)) {
77            const unsigned ifCarryDataBits = enumerate(&ifStatement->getBody(), ifDepth+1, whileDepth);
78            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[ifStatement->getBody().getScopeIndex()];
79#ifdef PACKING
80            EnsurePackHasSpace(nestedOffset, ifCarryDataBits);
81#endif
82            nestedBlockData->setFramePosition(nestedOffset);
83
84            nestedOffset += ifCarryDataBits;
85            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
86            cd->nested.entries++;
87        }
88        else if (While * whileStatement = dyn_cast<While>(stmt)) {
89            const unsigned whileCarryDataBits = enumerate(&whileStatement->getBody(), ifDepth, whileDepth+1);
90            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[whileStatement->getBody().getScopeIndex()];
91            //if (whileStatement->isMultiCarry()) whileCarryDataBits *= whileStatement->getMaxIterations();
92#ifdef PACKING
93            EnsurePackHasSpace(nestedOffset, whileCarryDataBits);
94#endif
95            nestedBlockData->setFramePosition(nestedOffset);
96            nestedOffset += whileCarryDataBits;
97            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
98            cd->nested.entries++;
99        }
100    }
101   
102    cd->scopeCarryDataBits = nestedOffset;
103   
104    if (cd->explicitSummaryRequired()) {
105        // Need extra space for the summary variable, always the last
106        // entry within an if block.
107        cd->scopeCarryDataBits = alignCeiling(cd->scopeCarryDataBits, PACK_SIZE);
108        cd->summary.frameOffsetinBits = cd->scopeCarryDataBits;
109        cd->summary.allocatedBits = PACK_SIZE;
110        cd->scopeCarryDataBits += PACK_SIZE;
111    }
112    else {
113        cd->summary.frameOffsetinBits = 0;
114        cd->summary.allocatedBits = cd->scopeCarryDataBits;
115    }
116#ifndef NDEBUG
117    cd->dumpCarryData(cerr);
118#endif
119    return cd->scopeCarryDataBits;
120}
121
122
123/* Entering and leaving blocks. */
124
125void CarryManager::enterScope(PabloBlock * blk) {
126   
127    mCurrentScope = blk;
128    mCarryInfo = mCarryInfoVector[blk->getScopeIndex()];
129    mCurrentFrameIndex += mCarryInfo->getFrameIndex();
130    //std::cerr << "enterScope:  mCurrentFrameIndex = " << mCurrentFrameIndex << std::endl;
131}
132
133void CarryManager::leaveScope() {
134    mCurrentFrameIndex -= mCarryInfo->getFrameIndex();
135    mCurrentScope = mCurrentScope->getParent();
136    mCarryInfo = mCarryInfoVector[mCurrentScope->getScopeIndex()];
137    //std::cerr << "leaveScope:  mCurrentFrameIndex = " << mCurrentFrameIndex << std::endl;
138}
139
140    /* Methods for getting and setting individual carry values. */
141   
142//#define LOAD_STORE_ON_BLOCK_ENTRY_EXIT   
143Value * CarryManager::getCarryOpCarryIn(int localIndex) {
144    unsigned cd_index = mCurrentFrameIndex + mCarryInfo->carryOpCarryDataOffset(localIndex);
145#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
146    if (mCarryInfo->getWhileDepth() == 0) {
147       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd_index));
148       mCarryInVector[cd_index] = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
149    }
150#endif
151    return mCarryInVector[cd_index];
152}
153
154void CarryManager::setCarryOpCarryOut(unsigned localIndex, Value * carry_out) {
155    unsigned cd_index = mCurrentFrameIndex + mCarryInfo->carryOpCarryDataOffset(localIndex);
156    mCarryOutVector[cd_index] = carry_out;
157#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
158    if (mCarryInfo->getWhileDepth() == 0) {
159       Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(cd_index));
160       mBuilder->CreateAlignedStore(carry_out, packPtr, BLOCK_SIZE/8);
161    }
162#endif
163}
164
165   
166Value * CarryManager::advanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
167    if (shift_amount == 1) {
168        return unitAdvanceCarryInCarryOut(localIndex, strm);
169    }
170    else if (shift_amount < LongAdvanceBase) {
171        return shortAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
172    }
173    else {
174        return longAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
175    }
176}
177
178Value * CarryManager::unitAdvanceCarryInCarryOut(int localIndex, Value * strm) {
179   
180    unsigned carryDataIndex = mCurrentFrameIndex + mCarryInfo->unitAdvanceCarryDataOffset(localIndex);
181    mCarryOutVector[carryDataIndex] = strm; 
182#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
183    if (mCarryInfo->getWhileDepth() == 0) {
184        Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(carryDataIndex));
185        mCarryInVector[carryDataIndex] = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
186        mBuilder->CreateAlignedStore(strm, packPtr, BLOCK_SIZE/8);
187    }
188#endif
189    Value * carry_in = mCarryInVector[carryDataIndex];
190    Value* result_value;
191   
192#if (BLOCK_SIZE == 128) && !defined(USE_LONG_INTEGER_SHIFT)
193    Value * ahead64 = iBuilder->mvmd_dslli(64, carry_in, strm, 1);
194    result_value = mBuilder->CreateOr(iBuilder->simd_srli(64, ahead64, 63), iBuilder->simd_slli(64, strm, 1));
195#else
196    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
197    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
198    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - 1), "advance");
199    result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
200   
201#endif
202    return result_value;
203}
204
205Value * CarryManager::shortAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
206    unsigned carryDataIndex = mCurrentFrameIndex + mCarryInfo->shortAdvanceCarryDataOffset(localIndex);
207    mCarryOutVector[carryDataIndex] = strm; 
208#ifndef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
209    if (mCarryInfo->getWhileDepth() == 0) {
210        Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(carryDataIndex));
211        mCarryInVector[carryDataIndex] = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
212        mBuilder->CreateAlignedStore(strm, packPtr, BLOCK_SIZE/8);
213    }
214#endif
215    Value * carry_in = mCarryInVector[carryDataIndex];
216    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
217    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
218    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, shift_amount), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - shift_amount), "advance");
219    return mBuilder->CreateBitCast(adv_longint, mBitBlockType);
220}
221   
222
223/*  currently defined in carry_data.h
224 
225 static unsigned power2ceil (unsigned v) {
226 unsigned ceil = 1;
227 while (ceil < v) ceil *= 2;
228 return ceil;
229 }
230 
231 unsigned longAdvanceEntries(unsigned shift_amount) const {
232 return (shift_amount + BLOCK_SIZE - 1)/BLOCK_SIZE;
233 }
234 
235 unsigned longAdvanceBufferSize(unsigned shift_amount)  const {
236 return power2ceil(longAdvanceEntries(shift_amount));
237 }
238 */
239
240   
241Value * CarryManager::longAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * carry_out) {
242    unsigned carryDataIndex = mCurrentFrameIndex + mCarryInfo->longAdvanceCarryDataOffset(localIndex);
243    Value * advBaseIndex = mBuilder->getInt64(carryDataIndex);
244    if (shift_amount <= BLOCK_SIZE) {
245        // special case using a single buffer entry and the carry_out value.
246        Value * advanceDataPtr = mBuilder->CreateGEP(mCarryDataPtr, advBaseIndex);
247        Value * carry_block0 = mBuilder->CreateAlignedLoad(advanceDataPtr, BLOCK_SIZE/8);
248        mBuilder->CreateAlignedStore(carry_out, advanceDataPtr, BLOCK_SIZE/8);
249        /* Very special case - no combine */
250        if (shift_amount == BLOCK_SIZE) return carry_block0;
251        Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - shift_amount);
252        Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_out, mBuilder->getIntNTy(BLOCK_SIZE)), shift_amount);
253        return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
254    }
255    // We need a buffer of at least two elements for storing the advance data.
256    const unsigned block_shift = shift_amount % BLOCK_SIZE;
257    const unsigned advanceEntries = mCarryInfo->longAdvanceEntries(shift_amount);
258    const unsigned bufsize = mCarryInfo->longAdvanceBufferSize(shift_amount);
259    Value * indexMask = mBuilder->getInt64(bufsize - 1);  // A mask to implement circular buffer indexing
260    Value * loadIndex0 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries)), indexMask), advBaseIndex);
261    Value * storeIndex = mBuilder->CreateAdd(mBuilder->CreateAnd(mBlockNo, indexMask), advBaseIndex);
262    Value * carry_block0 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex0), BLOCK_SIZE/8);
263    // If the long advance is an exact multiple of BLOCK_SIZE, we simply return the oldest
264    // block in the long advance carry data area. 
265    if (block_shift == 0) {
266        mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
267        return carry_block0;
268    }
269    // Otherwise we need to combine data from the two oldest blocks.
270    Value * loadIndex1 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries-1)), indexMask), advBaseIndex);
271    Value * carry_block1 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, loadIndex1), BLOCK_SIZE/8);
272    Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - block_shift);
273    Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_block1, mBuilder->getIntNTy(BLOCK_SIZE)), block_shift);
274    mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryDataPtr, storeIndex), BLOCK_SIZE/8);
275    return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
276}
277   
278
279/* Methods for getting and setting carry summary values */
280   
281bool CarryManager::blockHasCarries(){
282    return mCarryInfo->blockHasCarries();
283} 
284
285
286Value * CarryManager::getCarrySummaryExpr() {
287    unsigned summary_idx = mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex();
288    Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(summary_idx));
289    Value * summary_expr = mBuilder->CreateAlignedLoad(packPtr, BLOCK_SIZE/8);
290    // If the scopeCarryDataSize is 1, then the carry summary is also the pack expr.
291    mCarryInVector[summary_idx] = summary_expr;
292    return summary_expr;
293}
294
295void CarryManager::addSummaryPhiIfNeeded(BasicBlock * ifEntryBlock, BasicBlock * ifBodyFinalBlock) {
296    if ((mCarryInfo->getIfDepth() <= 1) || !mCarryInfo->blockHasCarries()){
297        // For ifDepth == 1, the parent does not need a summary as it is not itself within an if.
298        // Therefore, it doesn't need access to this block's summary in building its own.
299        return;
300    }
301    const unsigned carrySummaryIndex = mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex();
302    PHINode * summary_phi = mBuilder->CreatePHI(mBitBlockType, 2, "summary");
303    summary_phi->addIncoming(mZeroInitializer, ifEntryBlock);
304    summary_phi->addIncoming(mCarryOutVector[carrySummaryIndex], ifBodyFinalBlock);
305    mCarryOutVector[carrySummaryIndex] = summary_phi;
306}
307
308void CarryManager::generateCarryOutSummaryCodeIfNeeded() {
309   
310    if (!mCarryInfo->explicitSummaryRequired()) {
311        // An explicit summary may not be required, if there is a single carry
312        // operation within the block, or the carries are packed and all carry
313        // bits fit within a single pack.
314        return;
315    }
316   
317    const unsigned carrySummaryIndex = mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex();
318   
319    Value * carry_summary = mZeroInitializer;
320   
321    if (mCarryInfo->blockHasLongAdvances()) { // Force if entry
322        carry_summary = mOneInitializer;
323    }
324    else {
325        auto localCarryIndex = mCurrentFrameIndex + mCarryInfo->getLocalCarryPackIndex();
326        auto localCarryPacks = mCarryInfo->getLocalCarryDataSize();
327        if (localCarryPacks > 0) {
328            carry_summary = mCarryOutVector[localCarryIndex];
329            for (auto i = 1; i < localCarryPacks; i++) {
330                //carry_summary = mBuilder->CreateOr(carry_summary, mPabloBlock->mCarryOutPack[i]);           
331                carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[localCarryIndex+i]);
332            }
333        }
334        for (Statement * stmt : *mCurrentScope) {
335            if (If * innerIf = dyn_cast<If>(stmt)) {
336                PabloBlock * inner_blk = & innerIf->getBody();
337                enterScope(inner_blk);
338                if (blockHasCarries()) {
339                  carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex()]);
340                }
341                leaveScope();
342            }
343            else if (While * innerWhile = dyn_cast<While>(stmt)) {
344                PabloBlock * inner_blk = & innerWhile->getBody();
345                enterScope(inner_blk);
346                if (blockHasCarries()) {
347                    carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutVector[mCurrentFrameIndex + mCarryInfo->summaryCarryDataIndex()]);
348                }
349                leaveScope();
350            }
351        }
352    }
353    // Calculation of the carry out summary is complete.   Store it and make it
354    // available in case it must included by parent blocks.
355    mCarryOutVector[carrySummaryIndex] = carry_summary;
356    Value * packPtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(carrySummaryIndex));
357    mBuilder->CreateAlignedStore(carry_summary, packPtr, BLOCK_SIZE/8);
358}
359
360
361void CarryManager::ensureCarriesLoadedLocal() {
362#ifdef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
363    if ((mCarryInfo->getScopeCarryDataSize() == 0 ) || (mCarryInfo->getWhileDepth() > 0)) return;
364    if ((mCarryInfo->getIfDepth() == 0) || mCarryInfo->explicitSummaryRequired()) {
365        auto localCarryIndex = mCurrentFrameIndex + mCarryInfo->getLocalCarryPackIndex();
366        auto localCarryPacks = mCarryInfo->getLocalCarryDataSize();
367        //std::cerr << "ensureCarriesLoadedLocal: localCarryIndex =  " << localCarryIndex << "localCarryPacks =  " << localCarryPacks << std::endl;
368        for (auto i = localCarryIndex; i < localCarryIndex + localCarryPacks; i++) {       
369            mCarryInVector[i] = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i)), BLOCK_SIZE/8, false);
370        }
371    }
372#endif
373}
374
375void CarryManager::ensureCarriesStoredLocal() {
376#ifdef LOAD_STORE_ON_BLOCK_ENTRY_EXIT
377    if ((mCarryInfo->getScopeCarryDataSize() == 0 ) || (mCarryInfo->getWhileDepth() > 0)) return;
378    auto localCarryIndex = mCurrentFrameIndex + mCarryInfo->getLocalCarryPackIndex();
379    auto localCarryPacks = mCarryInfo->getLocalCarryDataSize();
380    //std::cerr << "ensureCarriesStoredLocal: localCarryIndex =  " << localCarryIndex << "localCarryPacks =  " << localCarryPacks << std::endl;
381    for (auto i = localCarryIndex; i < localCarryIndex + localCarryPacks; i++) {       
382        Value * storePtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i));
383        mBuilder->CreateAlignedStore(mCarryOutVector[i], storePtr, BLOCK_SIZE/8, false);
384    }
385#endif
386}
387
388
389
390void CarryManager::ensureCarriesLoadedRecursive() {
391    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
392    if (mCarryInfo->getWhileDepth() == 1) {
393        for (auto i = mCurrentFrameIndex; i < mCurrentFrameIndex + scopeCarryDataSize; ++i) {
394            mCarryInVector[i] = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i)), BLOCK_SIZE/8, false);
395        }
396    }
397}
398
399
400void CarryManager::initializeCarryDataPhisAtWhileEntry(BasicBlock * whileEntryBlock) {
401    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
402    for (unsigned index = mCurrentFrameIndex; index < mCurrentFrameIndex + scopeCarryDataSize; ++index) {
403#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
404        PHINode * phi_in = mBuilder->CreatePHI(mBitBlockType, 2);
405        phi_in->addIncoming(mCarryInVector[index], whileEntryBlock);
406        mCarryInPhis[index] = phi_in;
407#endif
408        PHINode * phi_out = mBuilder->CreatePHI(mBitBlockType, 2);
409        phi_out->addIncoming(mZeroInitializer, whileEntryBlock);
410        mCarryOutAccumPhis[index] = phi_out;
411    }
412}
413
414
415void CarryManager::extendCarryDataPhisAtWhileBodyFinalBlock(BasicBlock * whileBodyFinalBlock) {
416    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
417    for (unsigned index = mCurrentFrameIndex; index < mCurrentFrameIndex + scopeCarryDataSize; ++index) {
418#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
419        mCarryInPhis[index]->addIncoming(mZeroInitializer, whileBodyFinalBlock);
420#endif
421        PHINode * phi = mCarryOutAccumPhis[index];
422        assert (phi);
423        assert (mCarryOutVector[index]);
424        Value * carryOut = mBuilder->CreateOr(phi, mCarryOutVector[index]);
425        phi->addIncoming(carryOut, whileBodyFinalBlock);
426        mCarryOutVector[index] = carryOut;
427    }
428}
429
430void CarryManager::ensureCarriesStoredRecursive() {
431    const unsigned scopeCarryDataSize = mCarryInfo->getScopeCarryDataSize();
432    if (mCarryInfo->getWhileDepth() == 1) {
433        for (auto i = mCurrentFrameIndex; i < mCurrentFrameIndex + scopeCarryDataSize; ++i) {
434            Value * storePtr = mBuilder->CreateGEP(mCarryDataPtr, mBuilder->getInt64(i));
435            mBuilder->CreateAlignedStore(mCarryOutVector[i], storePtr, BLOCK_SIZE/8, false);
436        }
437    }
438}
439
440}
441
Note: See TracBrowser for help on using the repository browser.