source: icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp @ 4700

Last change on this file since 4700 was 4700, checked in by cameron, 4 years ago

Bit pack carry manager progress

File size: 22.1 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <include/simd-lib/bitblock.hpp>
9#include <stdexcept>
10#include <pablo/carry_data.h>
11#include <pablo/codegenstate.h>
12#include <pablo/carry_manager.h>
13#include <pablo/pabloAST.h>
14#include <iostream>
15
16namespace pablo {
17 
18    unsigned doScopeCount(PabloBlock * pb) {
19        unsigned count = 1;
20       
21        for (Statement * stmt : *pb) {
22            if (If * ifStatement = dyn_cast<If>(stmt)) {
23                count += doScopeCount(&ifStatement->getBody());
24            }
25            else if (While * whileStatement = dyn_cast<While>(stmt)) {
26                count += doScopeCount(&whileStatement->getBody());
27            }
28        }
29        return count;
30       
31    }
32
33unsigned CarryManager::initialize(PabloBlock * pb, Value * carryPtr) {
34    mPabloRoot = pb;
35#ifdef PACKING
36    mCarryPackType = mBuilder->getIntNTy(PACK_SIZE);
37    mCarryPackBasePtr = mBuilder->CreateBitCast(carryPtr, Type::getInt64PtrTy(mBuilder->getContext()));
38    mCarryBitBlockPtr = carryPtr;
39    mZeroInitializer = mBuilder->getInt64(0);
40    mOneInitializer = mBuilder->getInt64(-1);
41#else
42#define mCarryPackType mBitBlockType
43    mCarryPackBasePtr = carryPtr;
44#define mCarryBitBlockPtr mCarryPackBasePtr
45#endif
46    unsigned scopeCount = doScopeCount(pb);
47    mCarryInfoVector.resize(scopeCount);
48   
49    unsigned totalCarryDataBits = enumerate(pb, 0, 0);
50   
51    mTotalCarryDataBitBlocks = (totalCarryDataBits + BLOCK_SIZE - 1)/BLOCK_SIZE; 
52    // Carry Data area will have one extra bit block to store the block number.
53    mBlockNoPtr = mBuilder->CreateBitCast(mBuilder->CreateGEP(carryPtr, mBuilder->getInt64(mTotalCarryDataBitBlocks)), Type::getInt64PtrTy(mBuilder->getContext()));
54    mBlockNo = mBuilder->CreateLoad(mBlockNoPtr);
55    mCarryPackPtr.resize(mTotalCarryDataBitBlocks);
56    mCarryInPack.resize(mTotalCarryDataBitBlocks);
57    mCarryOutPack.resize(mTotalCarryDataBitBlocks);
58    for (auto i = 0; i < mTotalCarryDataBitBlocks; i++) mCarryInPack[i]=nullptr;
59   
60    /*  Set the current scope to PabloRoot */
61    mCurrentScope = mPabloRoot;
62    mCurrentFrameIndex = 0;
63    mCarryInfo = mCarryInfoVector[0];
64
65    return mTotalCarryDataBitBlocks + 1; // One extra element for the block no.
66}
67   
68void CarryManager::generateBlockNoIncrement() {
69    mBuilder->CreateStore(mBuilder->CreateAdd(mBlockNo, mBuilder->getInt64(1)), mBlockNoPtr);
70}
71
72Value * CarryManager::getBlockNoPtr() {
73    return mBlockNoPtr;
74}
75
76
77unsigned CarryManager::enumerate(PabloBlock * blk, unsigned ifDepth, unsigned whileDepth) {
78    llvm::raw_os_ostream cerr(std::cerr);
79    unsigned idx = blk->getScopeIndex();
80    PabloBlockCarryData * cd = new PabloBlockCarryData(blk);
81    mCarryInfoVector[idx] = cd;
82
83    cd->setIfDepth(ifDepth);
84    cd->setWhileDepth(whileDepth);
85    unsigned nestedOffset = cd->nested.frameOffsetinBits;
86 
87    for (Statement * stmt : *blk) {
88        if (If * ifStatement = dyn_cast<If>(stmt)) {
89            const unsigned ifCarryDataBits = enumerate(&ifStatement->getBody(), ifDepth+1, whileDepth);
90            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[ifStatement->getBody().getScopeIndex()];
91#ifdef PACKING
92            EnsurePackHasSpace(nestedOffset, ifCarryDataBits);
93#endif
94            nestedBlockData->setFramePosition(nestedOffset);
95
96            nestedOffset += ifCarryDataBits;
97            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
98            cd->nested.entries++;
99#ifndef NDEBUG
100            nestedBlockData->dumpCarryData(cerr);
101#endif
102        }
103        else if (While * whileStatement = dyn_cast<While>(stmt)) {
104            const unsigned whileCarryDataBits = enumerate(&whileStatement->getBody(), ifDepth, whileDepth+1);
105            PabloBlockCarryData * nestedBlockData = mCarryInfoVector[whileStatement->getBody().getScopeIndex()];
106            //if (whileStatement->isMultiCarry()) whileCarryDataBits *= whileStatement->getMaxIterations();
107#ifdef PACKING
108            EnsurePackHasSpace(nestedOffset, whileCarryDataBits);
109#endif
110            nestedBlockData->setFramePosition(nestedOffset);
111            nestedOffset += whileCarryDataBits;
112            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
113            cd->nested.entries++;
114#ifndef NDEBUG
115            nestedBlockData->dumpCarryData(cerr);
116#endif
117        }
118    }
119   
120    cd->scopeCarryDataBits = nestedOffset;
121   
122    if (cd->explicitSummaryRequired()) {
123        // Need extra space for the summary variable, always the last
124        // entry within an if block.
125        cd->scopeCarryDataBits = alignCeiling(cd->scopeCarryDataBits, PACK_SIZE);
126        cd->summary.frameOffsetinBits = cd->scopeCarryDataBits;
127        cd->summary.allocatedBits = PACK_SIZE;
128        cd->scopeCarryDataBits += PACK_SIZE;
129    }
130    else {
131        cd->summary.frameOffsetinBits = 0;
132        cd->summary.allocatedBits = cd->scopeCarryDataBits;
133    }
134    return cd->scopeCarryDataBits;
135}
136
137
138/* Entering and leaving blocks. */
139
140void CarryManager::enterScope(PabloBlock * blk) {
141   
142    mCurrentScope = blk;
143    mCarryInfo = mCarryInfoVector[blk->getScopeIndex()];
144    mCurrentFrameIndex += mCarryInfo->getFrameIndex();
145    //std::cerr << "enterScope:  mCurrentFrameIndex = " << mCurrentFrameIndex << std::endl;
146}
147
148void CarryManager::leaveScope() {
149    mCurrentFrameIndex -= mCarryInfo->getFrameIndex();
150    mCurrentScope = mCurrentScope->getParent();
151    mCarryInfo = mCarryInfoVector[mCurrentScope->getScopeIndex()];
152    //std::cerr << "leaveScope:  mCurrentFrameIndex = " << mCurrentFrameIndex << std::endl;
153}
154
155
156/* Helper routines */
157
158unsigned CarryManager::absPosition(unsigned frameOffsetinBits, unsigned relPos) {
159#ifdef PACKING
160    return mCurrentFrameIndex + frameOffsetinBits + relPos;
161#else
162    return mCurrentFrameIndex + frameOffsetinBits/BLOCK_SIZE + relPos;
163#endif
164}
165
166
167unsigned CarryManager::carryOpPosition(unsigned localIndex) {
168    return absPosition(mCarryInfo->addWithCarry.frameOffsetinBits, localIndex);
169}
170
171unsigned CarryManager::advance1Position(unsigned localIndex) {
172    return absPosition(mCarryInfo->advance1.frameOffsetinBits, localIndex);
173}
174
175unsigned CarryManager::shortAdvancePosition(unsigned localIndex) {
176    return absPosition(mCarryInfo->shortAdvance.frameOffsetinBits, localIndex);
177}
178
179unsigned CarryManager::longAdvanceBitBlockPosition(unsigned localIndex) {
180#ifdef PACKING
181    return (mCurrentFrameIndex + mCarryInfo->longAdvance.frameOffsetinBits) / BLOCK_SIZE + localIndex;
182#else
183    return mCurrentFrameIndex + (mCarryInfo->longAdvance.frameOffsetinBits / BLOCK_SIZE) + localIndex;
184#endif
185}
186   
187unsigned CarryManager::localBasePack() {
188#ifdef PACKING
189    return (mCurrentFrameIndex + mCarryInfo->shortAdvance.frameOffsetinBits) / PACK_SIZE;
190#else
191    return mCurrentFrameIndex + (mCarryInfo->shortAdvance.frameOffsetinBits / PACK_SIZE);
192#endif
193}
194   
195unsigned CarryManager::scopeBasePack() {
196#ifdef PACKING
197    return mCurrentFrameIndex / PACK_SIZE;
198#else
199    return mCurrentFrameIndex;
200#endif
201}
202   
203
204
205unsigned CarryManager::summaryPosition() {
206    return absPosition(mCarryInfo->summary.frameOffsetinBits, 0);
207}
208
209unsigned CarryManager::summaryBits() {
210    return mCarryInfo->summary.allocatedBits;
211}
212
213
214
215Value * CarryManager::getCarryPack(unsigned packIndex) {
216    if (mCarryInPack[packIndex] == nullptr) {
217        Value * packPtr = mBuilder->CreateGEP(mCarryPackBasePtr, mBuilder->getInt64(packIndex));
218        mCarryPackPtr[packIndex] = packPtr;
219        mCarryInPack[packIndex] = mBuilder->CreateAlignedLoad(packPtr, PACK_SIZE/8);
220    }
221    return mCarryInPack[packIndex];
222}
223
224void CarryManager::storeCarryPack(unsigned packIndex) {
225    mBuilder->CreateAlignedStore(mCarryOutPack[packIndex], mCarryPackPtr[packIndex], PACK_SIZE/8);
226}
227
228Value * CarryManager::getCarryRange(unsigned carryBit_lo, unsigned carryRangeSize) {
229
230    unsigned packIndex = carryBit_lo / PACK_SIZE;
231    unsigned carryOffset = carryBit_lo % PACK_SIZE;
232    unsigned hiOffset = carryBit_lo + carryRangeSize - 1;
233   
234    Value * carryItem = getCarryPack(packIndex);
235    if (carryRangeSize < PACK_SIZE) {
236       carryItem = mBuilder->CreateAnd(carryItem, mBuilder->getInt64((1 << hiOffset) - 1));
237    }
238    if (carryOffset > 0) {
239       carryItem = mBuilder->CreateLShr(carryItem, mBuilder->getInt64(carryOffset));
240    }
241    return carryItem;
242}
243   
244Value * CarryManager::getCarryBit(unsigned carryBitPos) {
245    return getCarryRange(carryBitPos, 1);
246}
247   
248void CarryManager::setCarryBits(unsigned carryBit_lo, Value * bits) {
249   
250    unsigned packIndex = carryBit_lo / PACK_SIZE;
251    unsigned carryOffset = carryBit_lo % PACK_SIZE;
252    if (carryOffset > 0) {
253        bits = mBuilder->CreateLShr(bits, mBuilder->getInt64(carryOffset));
254    }
255    if (mCarryOutPack[packIndex] == nullptr) {
256        mCarryOutPack[packIndex] = bits;
257    }
258    else {
259        mCarryOutPack[packIndex] = mBuilder->CreateOr(mCarryOutPack[packIndex], bits);
260    }
261}
262   
263   
264/* Methods for getting and setting individual carry values. */
265   
266Value * CarryManager::getCarryOpCarryIn(int localIndex) {
267    unsigned posn = carryOpPosition(localIndex);
268#ifdef PACKING
269    return getCarryBit(posn);
270#else
271    return getCarryPack(posn);
272#endif
273}
274   
275   
276void CarryManager::setCarryOpCarryOut(unsigned localIndex, Value * carry_out) {
277    unsigned posn = carryOpPosition(localIndex);
278#ifdef PACKING
279    setCarryBits(posn, carry_out);
280#else
281    mCarryOutPack[posn] = carry_out;
282    if (mCarryInfo->getWhileDepth() == 0) {
283        storeCarryPack(posn);
284    }
285#endif
286}
287
288   
289Value * CarryManager::advanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
290    if (shift_amount == 1) {
291        return unitAdvanceCarryInCarryOut(localIndex, strm);
292    }
293    else if (shift_amount < LongAdvanceBase) {
294        return shortAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
295    }
296    else {
297        return longAdvanceCarryInCarryOut(localIndex, shift_amount, strm);
298    }
299}
300
301Value * CarryManager::unitAdvanceCarryInCarryOut(int localIndex, Value * strm) {
302    unsigned posn = advance1Position(localIndex);
303#ifdef PACKING
304    unsigned offset = posn % PACK_SIZE;
305    unsigned rshift = PACK_SIZE - offset - 1;
306    Value * field = iBuilder->mvmd_extract(PACK_SIZE, strm, BLOCK_SIZE/PACK_SIZE - 1);
307    if (rshift != 0) {
308        field = mBuilder->CreateLShr(field, mBuilder->getInt64(rshift));
309    }
310    if (offset != 0) {
311        field = mBuilder->CreateAnd(field, mBuilder->getInt64(1<<offset));
312    }
313    setCarryBits(posn - offset, field);
314    Value* carry_longint = mBuilder->CreateZExt(getCarryBit(posn), mBuilder->getIntNTy(BLOCK_SIZE));
315    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
316    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), carry_longint);
317    Value* result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
318    return result_value;
319#else
320    mCarryOutPack[posn] = strm; 
321    Value * carry_in = getCarryPack(posn);
322    if (mCarryInfo->getWhileDepth() == 0) {
323        storeCarryPack(posn);
324    }
325    Value* result_value;
326   
327#if (BLOCK_SIZE == 128) && !defined(USE_LONG_INTEGER_SHIFT)
328    Value * ahead64 = iBuilder->mvmd_dslli(64, carry_in, strm, 1);
329    result_value = mBuilder->CreateOr(iBuilder->simd_srli(64, ahead64, 63), iBuilder->simd_slli(64, strm, 1));
330#else
331    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
332    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
333    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - 1), "advance");
334    result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
335   
336#endif
337    return result_value;
338#endif
339}
340
341Value * CarryManager::shortAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * strm) {
342    unsigned posn = shortAdvancePosition(localIndex);
343#ifdef PACKING
344    unsigned offset = posn % PACK_SIZE;
345    unsigned rshift = PACK_SIZE - offset - shift_amount;
346    Value * field = iBuilder->mvmd_extract(PACK_SIZE, strm, BLOCK_SIZE/PACK_SIZE - 1);
347    if (rshift != 0) {
348        field = mBuilder->CreateLShr(field, mBuilder->getInt64(rshift));
349    }
350    if (offset != 0) {
351        field = mBuilder->CreateAnd(field, mBuilder->getInt64(((1<<shift_amount) - 1) << offset));
352    }   
353    setCarryBits(posn - offset, field);
354    Value* carry_longint = mBuilder->CreateZExt(getCarryRange(posn, shift_amount), mBuilder->getIntNTy(BLOCK_SIZE));
355    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
356    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, 1), carry_longint);
357    Value* result_value = mBuilder->CreateBitCast(adv_longint, mBitBlockType);
358    return result_value;
359#else
360    mCarryOutPack[posn] = strm; 
361    Value * carry_in = getCarryPack(posn);
362    if (mCarryInfo->getWhileDepth() == 0) {
363        storeCarryPack(posn);
364    }
365    Value* advanceq_longint = mBuilder->CreateBitCast(carry_in, mBuilder->getIntNTy(BLOCK_SIZE));
366    Value* strm_longint = mBuilder->CreateBitCast(strm, mBuilder->getIntNTy(BLOCK_SIZE));
367    Value* adv_longint = mBuilder->CreateOr(mBuilder->CreateShl(strm_longint, shift_amount), mBuilder->CreateLShr(advanceq_longint, BLOCK_SIZE - shift_amount), "advance");
368    return mBuilder->CreateBitCast(adv_longint, mBitBlockType);
369#endif
370}
371   
372
373/*  currently defined in carry_data.h
374 
375 static unsigned power2ceil (unsigned v) {
376 unsigned ceil = 1;
377 while (ceil < v) ceil *= 2;
378 return ceil;
379 }
380 
381 unsigned longAdvanceEntries(unsigned shift_amount) const {
382 return (shift_amount + BLOCK_SIZE - 1)/BLOCK_SIZE;
383 }
384 
385 unsigned longAdvanceBufferSize(unsigned shift_amount)  const {
386 return power2ceil(longAdvanceEntries(shift_amount));
387 }
388 */
389
390   
391Value * CarryManager::longAdvanceCarryInCarryOut(int localIndex, int shift_amount, Value * carry_out) {
392    unsigned carryDataIndex = longAdvanceBitBlockPosition(localIndex);
393    Value * advBaseIndex = mBuilder->getInt64(carryDataIndex);
394    if (shift_amount <= BLOCK_SIZE) {
395        // special case using a single buffer entry and the carry_out value.
396        Value * advanceDataPtr = mBuilder->CreateGEP(mCarryBitBlockPtr, advBaseIndex);
397        Value * carry_block0 = mBuilder->CreateAlignedLoad(advanceDataPtr, BLOCK_SIZE/8);
398        mBuilder->CreateAlignedStore(carry_out, advanceDataPtr, BLOCK_SIZE/8);
399        /* Very special case - no combine */
400        if (shift_amount == BLOCK_SIZE) return carry_block0;
401        Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - shift_amount);
402        Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_out, mBuilder->getIntNTy(BLOCK_SIZE)), shift_amount);
403        return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
404    }
405    // We need a buffer of at least two elements for storing the advance data.
406    const unsigned block_shift = shift_amount % BLOCK_SIZE;
407    const unsigned advanceEntries = mCarryInfo->longAdvanceEntries(shift_amount);
408    const unsigned bufsize = mCarryInfo->longAdvanceBufferSize(shift_amount);
409    Value * indexMask = mBuilder->getInt64(bufsize - 1);  // A mask to implement circular buffer indexing
410    Value * loadIndex0 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries)), indexMask), advBaseIndex);
411    Value * storeIndex = mBuilder->CreateAdd(mBuilder->CreateAnd(mBlockNo, indexMask), advBaseIndex);
412    Value * carry_block0 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryBitBlockPtr, loadIndex0), BLOCK_SIZE/8);
413    // If the long advance is an exact multiple of BLOCK_SIZE, we simply return the oldest
414    // block in the long advance carry data area. 
415    if (block_shift == 0) {
416        mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryBitBlockPtr, storeIndex), BLOCK_SIZE/8);
417        return carry_block0;
418    }
419    // Otherwise we need to combine data from the two oldest blocks.
420    Value * loadIndex1 = mBuilder->CreateAdd(mBuilder->CreateAnd(mBuilder->CreateSub(mBlockNo, mBuilder->getInt64(advanceEntries-1)), indexMask), advBaseIndex);
421    Value * carry_block1 = mBuilder->CreateAlignedLoad(mBuilder->CreateGEP(mCarryBitBlockPtr, loadIndex1), BLOCK_SIZE/8);
422    Value* block0_shr = mBuilder->CreateLShr(mBuilder->CreateBitCast(carry_block0, mBuilder->getIntNTy(BLOCK_SIZE)), BLOCK_SIZE - block_shift);
423    Value* block1_shl = mBuilder->CreateShl(mBuilder->CreateBitCast(carry_block1, mBuilder->getIntNTy(BLOCK_SIZE)), block_shift);
424    mBuilder->CreateAlignedStore(carry_out, mBuilder->CreateGEP(mCarryBitBlockPtr, storeIndex), BLOCK_SIZE/8);
425    return mBuilder->CreateBitCast(mBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
426}
427   
428
429/* Methods for getting and setting carry summary values */
430   
431bool CarryManager::blockHasCarries(){
432    return mCarryInfo->blockHasCarries();
433} 
434
435
436Value * CarryManager::getCarrySummaryExpr() {
437    unsigned summary_posn = summaryPosition();
438#ifdef PACKING
439    return getCarryRange(summary_posn, summaryBits());
440#else
441    return getCarryPack(summary_posn);
442#endif
443}
444
445void CarryManager::addSummaryPhiIfNeeded(BasicBlock * ifEntryBlock, BasicBlock * ifBodyFinalBlock) {
446    if ((mCarryInfo->getIfDepth() <= 1) || !mCarryInfo->blockHasCarries()){
447        // For ifDepth == 1, the parent does not need a summary as it is not itself within an if.
448        // Therefore, it doesn't need access to this block's summary in building its own.
449        return;
450    }
451    const unsigned carrySummaryIndex = summaryPosition();
452    PHINode * summary_phi = mBuilder->CreatePHI(mCarryPackType, 2, "summary");
453    summary_phi->addIncoming(mZeroInitializer, ifEntryBlock);
454    summary_phi->addIncoming(mCarryOutPack[carrySummaryIndex], ifBodyFinalBlock);
455    mCarryOutPack[carrySummaryIndex] = summary_phi;
456}
457
458void CarryManager::generateCarryOutSummaryCodeIfNeeded() {
459   
460    if (!mCarryInfo->explicitSummaryRequired()) {
461        // An explicit summary may not be required, if there is a single carry
462        // operation within the block, or the carries are packed and all carry
463        // bits fit within a single pack.
464        return;
465    }
466   
467    const unsigned carrySummaryIndex = summaryPosition();
468   
469    Value * carry_summary = mZeroInitializer;
470    if (mCarryInfo->blockHasLongAdvances()) { // Force if entry
471        carry_summary = mOneInitializer;
472    }
473    else {
474        auto localCarryIndex = localBasePack();
475        auto localCarryPacks = mCarryInfo->getLocalCarryPackCount();
476        if (localCarryPacks > 0) {
477            carry_summary = mCarryOutPack[localCarryIndex];
478            for (auto i = 1; i < localCarryPacks; i++) {
479                //carry_summary = mBuilder->CreateOr(carry_summary, mPabloBlock->mCarryOutPack[i]);           
480                carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutPack[localCarryIndex+i]);
481            }
482        }
483        for (Statement * stmt : *mCurrentScope) {
484            if (If * innerIf = dyn_cast<If>(stmt)) {
485                PabloBlock * inner_blk = & innerIf->getBody();
486                enterScope(inner_blk);
487                if (blockHasCarries()) {
488                  carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutPack[summaryPosition()]);
489                }
490                leaveScope();
491            }
492            else if (While * innerWhile = dyn_cast<While>(stmt)) {
493                PabloBlock * inner_blk = & innerWhile->getBody();
494                enterScope(inner_blk);
495                if (blockHasCarries()) {
496                    carry_summary = mBuilder->CreateOr(carry_summary, mCarryOutPack[summaryPosition()]);
497                }
498                leaveScope();
499            }
500        }
501    }
502    // Calculation of the carry out summary is complete.   Store it and make it
503    // available in case it must included by parent blocks.
504    mCarryOutPack[carrySummaryIndex] = carry_summary;
505    storeCarryPack(carrySummaryIndex);
506}
507
508
509
510void CarryManager::ensureCarriesLoadedRecursive() {
511    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
512    const unsigned currentScopeBase = scopeBasePack();
513    if (mCarryInfo->getWhileDepth() == 1) {
514        for (auto i = currentScopeBase; i < currentScopeBase + scopeCarryPacks; ++i) {
515            getCarryPack(i);
516        }
517    }
518}
519
520
521void CarryManager::initializeCarryDataPhisAtWhileEntry(BasicBlock * whileEntryBlock) {
522    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
523    mCarryOutAccumPhis.resize(scopeCarryPacks);
524#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
525    const unsigned currentScopeBase = scopeBasePack();
526    mCarryInPhis.resize(scopeCarryPacks);
527#endif
528    for (unsigned index = 0; index < scopeCarryPacks; ++index) {
529#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
530        PHINode * phi_in = mBuilder->CreatePHI(mCarryPackType, 2);
531        phi_in->addIncoming(mCarryInPack[currentScopeBase+index], whileEntryBlock);
532        mCarryInPhis[index] = phi_in;
533#endif
534        PHINode * phi_out = mBuilder->CreatePHI(mCarryPackType, 2);
535        phi_out->addIncoming(mZeroInitializer, whileEntryBlock);
536        mCarryOutAccumPhis[index] = phi_out;
537    }
538}
539
540
541void CarryManager::extendCarryDataPhisAtWhileBodyFinalBlock(BasicBlock * whileBodyFinalBlock) {
542    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
543    const unsigned currentScopeBase = scopeBasePack();
544    for (unsigned index = 0; index < scopeCarryPacks; ++index) {
545#ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
546        mCarryInPhis[index]->addIncoming(mZeroInitializer, whileBodyFinalBlock);
547#endif
548        PHINode * phi = mCarryOutAccumPhis[index];
549        Value * carryOut = mBuilder->CreateOr(phi, mCarryOutPack[currentScopeBase+index]);
550        phi->addIncoming(carryOut, whileBodyFinalBlock);
551        mCarryOutPack[currentScopeBase+index] = carryOut;
552    }
553}
554
555void CarryManager::ensureCarriesStoredRecursive() {
556    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
557    const unsigned currentScopeBase = scopeBasePack();
558    if (mCarryInfo->getWhileDepth() == 1) {
559        for (auto i = currentScopeBase; i < currentScopeBase + scopeCarryPacks; ++i) {
560            storeCarryPack(i);
561        }
562    }
563}
564
565}
566
Note: See TracBrowser for help on using the repository browser.