source: icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp @ 5106

Last change on this file since 5106 was 5106, checked in by cameron, 3 years ago

Use dynamic size_t determination

File size: 27.9 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <stdexcept>
8#include <pablo/carry_data.h>
9#include <pablo/codegenstate.h>
10#include <pablo/carry_manager.h>
11#include <pablo/pabloAST.h>
12#include <llvm/Support/CommandLine.h>
13#include <llvm/IR/BasicBlock.h>
14#include <llvm/IR/CallingConv.h>
15#include <llvm/IR/Function.h>
16
17#define DSSLI_FIELDWIDTH 64
18
19namespace pablo {
20
21/** ------------------------------------------------------------------------------------------------------------- *
22 * @brief initializeCarryData
23 ** ------------------------------------------------------------------------------------------------------------- */
24Type * CarryManager::initializeCarryData(PabloFunction * const function) {
25    mRootScope = function->getEntryBlock();
26    mCarryInfoVector.resize(mRootScope->enumerateScopes(0) + 1);
27    mCarryPackType = mBitBlockType;
28    const unsigned totalCarryDataSize = enumerate(mRootScope, 0, 0);
29
30    mCarryPackPtr.resize(totalCarryDataSize + 1, nullptr);
31    mCarryInPack.resize(totalCarryDataSize + 1, nullptr);
32    mCarryOutPack.resize(totalCarryDataSize + 1, nullptr);
33
34    mTotalCarryDataBitBlocks = totalCarryDataSize;
35    ArrayType* cdArrayTy = ArrayType::get(mBitBlockType, mTotalCarryDataBitBlocks);
36    return cdArrayTy;
37}
38
39/** ------------------------------------------------------------------------------------------------------------- *
40 * @brief initializeCodeGen
41 ** ------------------------------------------------------------------------------------------------------------- */
42void CarryManager::initializeCodeGen(PabloKernel * const kBuilder, Value * selfPtr) {
43    mKernelBuilder = kBuilder;
44    mSelf = selfPtr;
45   
46    Value * cdArrayPtr = iBuilder->CreateGEP(mSelf, {ConstantInt::get(iBuilder->getSizeTy(), 0), mKernelBuilder->getScalarIndex("carries")});
47#ifndef NDEBUG
48    iBuilder->CallPrintInt("cdArrayPtr", iBuilder->CreatePtrToInt(cdArrayPtr, iBuilder->getSizeTy()));
49#endif
50    mCarryPackBasePtr = iBuilder->CreateBitCast(cdArrayPtr, PointerType::get(mCarryPackType, 0));
51    mCarryBitBlockPtr = iBuilder->CreateBitCast(cdArrayPtr, PointerType::get(mBitBlockType, 0));
52    mCurrentScope = mRootScope;
53    mCurrentFrameIndex = 0;
54    assert(mCarryInfoVector.size() > 0);
55    mCarryInfo = mCarryInfoVector[0];
56    assert(summaryPack() < mCarryOutPack.size());
57    mCarryOutPack[summaryPack()] = Constant::getNullValue(mCarryPackType);
58    assert (mCarrySummary.empty());
59}
60
61/** ------------------------------------------------------------------------------------------------------------- *
62 * @brief enterScope
63 ** ------------------------------------------------------------------------------------------------------------- */
64void CarryManager::enterScope(PabloBlock * const scope) {
65    assert(summaryPack() < mCarryOutPack.size());
66    Value * summaryCarry = mCarryOutPack[summaryPack()];
67    mCarrySummary.push_back(summaryCarry);
68    mCurrentScope = scope;
69    mCarryInfo = mCarryInfoVector[scope->getScopeIndex()];
70    mCurrentFrameIndex += mCarryInfo->getFrameIndex();
71    assert(summaryPack() < mCarryOutPack.size());
72    mCarryOutPack[summaryPack()] = Constant::getNullValue(mCarryPackType);
73}
74
75/** ------------------------------------------------------------------------------------------------------------- *
76 * @brief leaveScope
77 ** ------------------------------------------------------------------------------------------------------------- */
78void CarryManager::leaveScope() {
79    assert(summaryPack() < mCarryOutPack.size());
80    Value * summaryCarry = mCarryOutPack[summaryPack()];
81    assert (mCurrentScope != mRootScope);
82    mCurrentFrameIndex -= mCarryInfo->getFrameIndex();
83    mCurrentScope = mCurrentScope->getParent();
84    mCarryInfo = mCarryInfoVector[mCurrentScope->getScopeIndex()];
85    assert(summaryPack() < mCarryOutPack.size());
86    mCarryOutPack[summaryPack()] = summaryCarry;
87    mCarrySummary.pop_back();
88}
89
90/** ------------------------------------------------------------------------------------------------------------- *
91 * @brief addCarryInCarryOut
92 ** ------------------------------------------------------------------------------------------------------------- */
93Value * CarryManager::addCarryInCarryOut(const unsigned localIndex, Value * const e1, Value * const e2) {
94    Value * sum = nullptr;
95    if (mBitBlockWidth == 128) {
96        Value * carryq_value = getCarryIn(localIndex);
97        //calculate carry through logical ops
98        Value * carrygen = iBuilder->simd_and(e1, e2);
99        Value * carryprop = iBuilder->simd_or(e1, e2);
100        Value * digitsum = iBuilder->simd_add(64, e1, e2);
101        Value * partial = iBuilder->simd_add(64, digitsum, carryq_value);
102        Value * digitcarry = iBuilder->simd_or(carrygen, iBuilder->simd_and(carryprop, iBuilder->CreateNot(partial)));
103        Value * mid_carry_in = iBuilder->simd_slli(128, iBuilder->CreateLShr(digitcarry, 63), 64);
104        sum = iBuilder->simd_add(64, partial, iBuilder->CreateBitCast(mid_carry_in, mBitBlockType));
105        Value * carry_out_strm = iBuilder->simd_or(carrygen, iBuilder->simd_and(carryprop, iBuilder->CreateNot(sum)));
106        setCarryOut(localIndex, carry_out_strm);
107    } else if (mBitBlockWidth >= 256) {
108        // using LONG_ADD
109        Value * carryq_value = getCarryIn(localIndex);
110        Value * carryin = iBuilder->mvmd_extract(32, carryq_value, 0);
111        Value * carrygen = iBuilder->simd_and(e1, e2);
112        Value * carryprop = iBuilder->simd_or(e1, e2);
113        Value * digitsum = iBuilder->simd_add(64, e1, e2);
114        Value * digitcarry = iBuilder->simd_or(carrygen, iBuilder->simd_and(carryprop, iBuilder->CreateNot(digitsum)));
115        Value * carryMask = iBuilder->hsimd_signmask(64, digitcarry);
116        Value * carryMask2 = iBuilder->CreateOr(iBuilder->CreateAdd(carryMask, carryMask), carryin);
117        Value * bubble = iBuilder->simd_eq(64, digitsum, iBuilder->allOnes());
118        Value * bubbleMask = iBuilder->hsimd_signmask(64, bubble);
119        Value * incrementMask = iBuilder->CreateXor(iBuilder->CreateAdd(bubbleMask, carryMask2), bubbleMask);
120        Value * increments = iBuilder->esimd_bitspread(64,incrementMask);
121        sum = iBuilder->simd_add(64, digitsum, increments);
122        Value * carry_out_strm = iBuilder->CreateZExt(iBuilder->CreateLShr(incrementMask, mBitBlockWidth / 64), iBuilder->getIntNTy(mBitBlockWidth));
123        setCarryOut(localIndex, iBuilder->bitCast(carry_out_strm));
124    }
125    else {
126        Value * carryq_value = getCarryIn(localIndex);
127        Value * carrygen = iBuilder->simd_and(e1, e2);
128        Value * carryprop = iBuilder->simd_or(e1, e2);
129        sum = iBuilder->simd_add(mBitBlockWidth, iBuilder->simd_add(mBitBlockWidth, e1, e2), carryq_value);
130        Value * carry_out_strm = iBuilder->simd_or(carrygen, iBuilder->simd_and(carryprop, iBuilder->CreateNot(sum)));
131        setCarryOut(localIndex, carry_out_strm);
132    }
133    return sum;
134}
135
136/** ------------------------------------------------------------------------------------------------------------- *
137 * @brief advanceCarryInCarryOut
138 ** ------------------------------------------------------------------------------------------------------------- */
139Value * CarryManager::advanceCarryInCarryOut(const unsigned localIndex, const unsigned shiftAmount, Value * const value) {
140    if (LLVM_LIKELY(shiftAmount == 1)) {
141        return shortAdvanceCarryInCarryOut(unitAdvancePosition(localIndex), shiftAmount, value);
142    } else if (shiftAmount < LongAdvanceBase) {
143        return shortAdvanceCarryInCarryOut(shortAdvancePosition(localIndex), shiftAmount, value);
144    } else {
145        return longAdvanceCarryInCarryOut(longAdvancePosition(localIndex), shiftAmount, value);
146    }
147}
148
149/** ------------------------------------------------------------------------------------------------------------- *
150 * @brief shortAdvanceCarryInCarryOut
151 ** ------------------------------------------------------------------------------------------------------------- */
152Value * CarryManager::shortAdvanceCarryInCarryOut(const unsigned index, const unsigned shiftAmount, Value * const value) {
153    Value * result = nullptr;
154    Value * const carryIn = getCarryPack(index);
155    assert (index < mCarryOutPack.size());
156    mCarryOutPack[index] = value;
157    if (mCarryInfo->getWhileDepth() == 0) {
158        storeCarryOut(index);
159    }
160    if (LLVM_UNLIKELY((shiftAmount % 8) == 0)) { // Use a single whole-byte shift, if possible.
161        result = iBuilder->mvmd_dslli(8, value, carryIn, (iBuilder->getBitBlockWidth() / 8) - (shiftAmount / 8));
162    } else if (LLVM_LIKELY(shiftAmount < DSSLI_FIELDWIDTH)) {
163        Value * ahead = iBuilder->mvmd_dslli(DSSLI_FIELDWIDTH, value, carryIn, iBuilder->getBitBlockWidth() / DSSLI_FIELDWIDTH - 1);
164        result = iBuilder->simd_or(iBuilder->simd_srli(DSSLI_FIELDWIDTH, ahead, DSSLI_FIELDWIDTH - shiftAmount), iBuilder->simd_slli(DSSLI_FIELDWIDTH, value, shiftAmount));
165    } else {
166        Value* advanceq_longint = iBuilder->CreateBitCast(carryIn, iBuilder->getIntNTy(mBitBlockWidth));
167        Value* strm_longint = iBuilder->CreateBitCast(value, iBuilder->getIntNTy(mBitBlockWidth));
168        Value* adv_longint = iBuilder->CreateOr(iBuilder->CreateShl(strm_longint, shiftAmount), iBuilder->CreateLShr(advanceq_longint, mBitBlockWidth - shiftAmount), "advance");
169        result = iBuilder->CreateBitCast(adv_longint, mBitBlockType);
170    }
171    if (LLVM_LIKELY(hasSummary())) {
172        addToSummary(value);
173    }
174    return result;
175}
176
177/** ------------------------------------------------------------------------------------------------------------- *
178 * @brief longAdvanceCarryInCarryOut
179 ** ------------------------------------------------------------------------------------------------------------- */
180Value * CarryManager::longAdvanceCarryInCarryOut(const unsigned index, const unsigned shiftAmount, Value * const value) {
181    Value * advBaseIndex = ConstantInt::get(iBuilder->getSizeTy(), index);
182    if (shiftAmount <= mBitBlockWidth) {
183        // special case using a single buffer entry and the carry_out value.
184        Value * advanceDataPtr = iBuilder->CreateGEP(mCarryBitBlockPtr, advBaseIndex);
185        Value * carry_block0 = iBuilder->CreateBlockAlignedLoad(advanceDataPtr);
186        iBuilder->CreateBlockAlignedStore(value, advanceDataPtr);
187        /* Very special case - no combine */
188        if (shiftAmount == mBitBlockWidth) {
189            return carry_block0;
190        }
191        Value* block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carry_block0, iBuilder->getIntNTy(mBitBlockWidth)), mBitBlockWidth - shiftAmount);
192        Value* block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(value, iBuilder->getIntNTy(mBitBlockWidth)), shiftAmount);
193        return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
194    }
195    // We need a buffer of at least two elements for storing the advance data.
196    const unsigned block_shift = shiftAmount % mBitBlockWidth;
197    const unsigned advanceEntries = mCarryInfo->longAdvanceEntries(shiftAmount);
198    const unsigned bufsize = mCarryInfo->longAdvanceBufferSize(shiftAmount);
199    Value * indexMask = ConstantInt::get(iBuilder->getSizeTy(), bufsize - 1);  // A mask to implement circular buffer indexing
200    Value * blockIndex = mKernelBuilder->getScalarField(mSelf, blockNoScalar);
201    Value * loadIndex0 = iBuilder->CreateAdd(iBuilder->CreateAnd(iBuilder->CreateSub(blockIndex, ConstantInt::get(iBuilder->getSizeTy(), advanceEntries)), indexMask), advBaseIndex);
202    Value * storeIndex = iBuilder->CreateAdd(iBuilder->CreateAnd(blockIndex, indexMask), advBaseIndex);
203    Value * carry_block0 = iBuilder->CreateBlockAlignedLoad(iBuilder->CreateGEP(mCarryBitBlockPtr, loadIndex0));
204    // If the long advance is an exact multiple of mBITBLOCK_WIDTH, we simply return the oldest
205    // block in the long advance carry data area. 
206    if (block_shift == 0) {
207        iBuilder->CreateBlockAlignedStore(value, iBuilder->CreateGEP(mCarryBitBlockPtr, storeIndex));
208        return carry_block0;
209    }
210    // Otherwise we need to combine data from the two oldest blocks.
211    Value * loadIndex1 = iBuilder->CreateAdd(iBuilder->CreateAnd(iBuilder->CreateSub(blockIndex, ConstantInt::get(iBuilder->getSizeTy(), advanceEntries-1)), indexMask), advBaseIndex);
212    Value * carry_block1 = iBuilder->CreateBlockAlignedLoad(iBuilder->CreateGEP(mCarryBitBlockPtr, loadIndex1));
213    Value* block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carry_block0, iBuilder->getIntNTy(mBitBlockWidth)), mBitBlockWidth - block_shift);
214    Value* block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(carry_block1, iBuilder->getIntNTy(mBitBlockWidth)), block_shift);
215    iBuilder->CreateBlockAlignedStore(value, iBuilder->CreateGEP(mCarryBitBlockPtr, storeIndex));
216    return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
217}
218
219/** ------------------------------------------------------------------------------------------------------------- *
220 * @brief generateSummaryTest
221 ** ------------------------------------------------------------------------------------------------------------- */
222Value * CarryManager::generateSummaryTest(Value * condition) {
223    if (mCarryInfo->hasCarries()) {
224        Value * summary_pack = getCarryPack(summaryPack());
225        condition = iBuilder->simd_or(condition, summary_pack);
226    }
227    return iBuilder->bitblock_any(condition);
228}
229
230/** ------------------------------------------------------------------------------------------------------------- *
231 * @brief storeCarryOutSummary
232 ** ------------------------------------------------------------------------------------------------------------- */
233void CarryManager::storeCarryOutSummary() {
234    if (LLVM_LIKELY(mCarryInfo->explicitSummaryRequired())) {
235        const unsigned carrySummaryIndex = summaryPack();
236        if (LLVM_UNLIKELY(mCarryInfo->hasLongAdvances())) { // Force if entry
237            assert (carrySummaryIndex < mCarryOutPack.size());
238            mCarryOutPack[carrySummaryIndex] = Constant::getAllOnesValue(mCarryPackType);
239        }
240        storeCarryOut(carrySummaryIndex);
241    }
242}
243
244/** ------------------------------------------------------------------------------------------------------------- *
245 * @brief addOuterSummaryToNestedSummary
246 ** ------------------------------------------------------------------------------------------------------------- */
247void CarryManager::addOuterSummaryToNestedSummary() {
248    if (LLVM_LIKELY(mCarrySummary.size() > 0)) {
249        addToSummary(mCarrySummary.back());
250    }
251}
252
253/** ------------------------------------------------------------------------------------------------------------- *
254 * @brief buildCarryDataPhisAfterIfBody
255 ** ------------------------------------------------------------------------------------------------------------- */
256void CarryManager::buildCarryDataPhisAfterIfBody(BasicBlock * const entry, BasicBlock * const end) {
257    if (mCarryInfo->getWhileDepth() > 0) {
258        // We need to phi out everything for the while carry accumulation process.
259        const unsigned scopeBaseOffset = scopeBasePack();
260        const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
261        for (unsigned i = scopeBaseOffset; i < scopeBaseOffset + scopeCarryPacks; ++i) {
262            assert (i < mCarryOutPack.size());
263            Type * const type = mCarryOutPack[i]->getType();
264            PHINode * phi = iBuilder->CreatePHI(type, 2);
265            phi->addIncoming(Constant::getNullValue(type), entry);
266            phi->addIncoming(mCarryOutPack[i], end);
267            mCarryOutPack[i] = phi;
268        }
269    }
270    if (LLVM_LIKELY(mCarrySummary.size() > 0)) {
271        const unsigned summaryIndex = summaryPack();
272        assert (summaryIndex < mCarryOutPack.size());
273        Value * carrySummary = mCarryOutPack[summaryIndex];
274        if (mCarrySummary.back() != carrySummary) {
275            Value * outerCarrySummary = mCarrySummary.back();
276            Value * nestedCarrySummary = mCarryOutPack[summaryIndex];
277            assert (outerCarrySummary->getType() == nestedCarrySummary->getType());
278            PHINode * const phi = iBuilder->CreatePHI(outerCarrySummary->getType(), 2, "summary");
279            phi->addIncoming(outerCarrySummary, entry);
280            phi->addIncoming(nestedCarrySummary, end);
281            mCarryOutPack[summaryIndex] = phi;
282        }
283    }
284}
285
286/** ------------------------------------------------------------------------------------------------------------- *
287 * @brief initializeWhileEntryCarryDataPhis
288 ** ------------------------------------------------------------------------------------------------------------- */
289void CarryManager::initializeWhileEntryCarryDataPhis(BasicBlock * const end) {
290    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
291    mCarryOutAccumPhis.resize(scopeCarryPacks);
292    #ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
293    const unsigned currentScopeBase = scopeBasePack();
294    mCarryInPhis.resize(scopeCarryPacks);
295    #endif
296    for (unsigned index = 0; index < scopeCarryPacks; ++index) {
297        #ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
298        PHINode * phi_in = iBuilder->CreatePHI(mCarryPackType, 2);
299        phi_in->addIncoming(mCarryInPack[currentScopeBase+index], whileEntryBlock);
300        mCarryInPhis[index] = phi_in;
301        #endif
302        PHINode * phi_out = iBuilder->CreatePHI(mCarryPackType, 2);
303        phi_out->addIncoming(Constant::getNullValue(mCarryPackType), end);
304        assert (index < mCarryOutAccumPhis.size());
305        mCarryOutAccumPhis[index] = phi_out;
306    }
307}
308
309/** ------------------------------------------------------------------------------------------------------------- *
310 * @brief finalizeWhileBlockCarryDataPhis
311 ** ------------------------------------------------------------------------------------------------------------- */
312void CarryManager::finalizeWhileBlockCarryDataPhis(BasicBlock * const end) {
313    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
314    const unsigned currentScopeBase = scopeBasePack();
315    for (unsigned index = 0; index < scopeCarryPacks; ++index) {
316        #ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
317        mCarryInPhis[index]->addIncoming(Constant::getNullValue(mCarryPackType), whileBodyFinalBlock);
318        #endif
319        assert (index < mCarryOutAccumPhis.size());
320        PHINode * phi = mCarryOutAccumPhis[index];
321        Value * carryOut = iBuilder->CreateOr(phi, mCarryOutPack[currentScopeBase + index]);
322        phi->addIncoming(carryOut, end);
323        mCarryOutPack[currentScopeBase + index] = carryOut;
324    }
325}
326
327/** ------------------------------------------------------------------------------------------------------------- *
328 * @brief ensureCarriesLoadedRecursive
329 ** ------------------------------------------------------------------------------------------------------------- */
330void CarryManager::ensureCarriesLoadedRecursive() {
331    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
332    const unsigned currentScopeBase = scopeBasePack();
333    if (mCarryInfo->getWhileDepth() == 1) {
334        for (auto i = currentScopeBase; i < currentScopeBase + scopeCarryPacks; ++i) {
335            getCarryPack(i);
336        }
337    }
338}
339
340/** ------------------------------------------------------------------------------------------------------------- *
341 * @brief ensureCarriesStoredRecursive
342 ** ------------------------------------------------------------------------------------------------------------- */
343void CarryManager::ensureCarriesStoredRecursive() {
344    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
345    const unsigned currentScopeBase = scopeBasePack();
346    if (mCarryInfo->getWhileDepth() == 1) {
347        for (auto i = currentScopeBase; i < currentScopeBase + scopeCarryPacks; ++i) {
348            storeCarryOut(i);
349        }
350    }
351}
352
353/** ------------------------------------------------------------------------------------------------------------- *
354 * @brief getCarryIn
355 ** ------------------------------------------------------------------------------------------------------------- */
356Value * CarryManager::getCarryIn(const unsigned localIndex) {
357    return getCarryPack(addPosition(localIndex));
358}
359
360/** ------------------------------------------------------------------------------------------------------------- *
361 * @brief setCarryOut
362 ** ------------------------------------------------------------------------------------------------------------- */
363void CarryManager::setCarryOut(const unsigned localIndex, Value * carryOut) {
364    const unsigned index = addPosition(localIndex);
365    if (mBitBlockWidth < 256) { // #ifndef USING_LONG_ADD
366        Value * carry_bit = iBuilder->CreateLShr(iBuilder->CreateBitCast(carryOut, iBuilder->getIntNTy(mBitBlockWidth)), mBitBlockWidth-1);
367        carryOut = iBuilder->CreateBitCast(carry_bit, mBitBlockType);
368    }
369    assert (index < mCarryOutPack.size());
370    mCarryOutPack[index] = carryOut;
371    if (LLVM_LIKELY(hasSummary())) {
372        addToSummary(carryOut);
373    }
374    if (mCarryInfo->getWhileDepth() == 0) {
375        storeCarryOut(index);
376    }
377}
378
379/** ------------------------------------------------------------------------------------------------------------- *
380 * @brief enumerate
381 ** ------------------------------------------------------------------------------------------------------------- */
382unsigned CarryManager::enumerate(PabloBlock * blk, unsigned ifDepth, unsigned whileDepth) {
383    unsigned idx = blk->getScopeIndex();
384    CarryData * cd = new CarryData(blk, mBitBlockWidth, 1, mBitBlockWidth);
385    mCarryInfoVector[idx] = cd;
386
387    cd->setIfDepth(ifDepth);
388    cd->setWhileDepth(whileDepth);
389    unsigned nestedOffset = cd->nested.frameOffset;
390
391    for (Statement * stmt : *blk) {
392        if (If * ifStatement = dyn_cast<If>(stmt)) {
393            const unsigned ifCarryDataBits = enumerate(ifStatement->getBody(), ifDepth + 1, whileDepth);
394            CarryData * nestedBlockData = mCarryInfoVector[ifStatement->getBody()->getScopeIndex()];
395            if (1 == mBitBlockWidth) {  // PACKING
396                if (cd->roomInFinalPack(nestedOffset) < ifCarryDataBits) {
397                    nestedOffset = alignCeiling(nestedOffset, mBitBlockWidth);
398                }
399            }
400            nestedBlockData->setFramePosition(nestedOffset);
401            nestedOffset += ifCarryDataBits;
402            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) {
403                cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
404            }
405            cd->nested.entries++;
406        } else if (While * whileStatement = dyn_cast<While>(stmt)) {
407            const unsigned whileCarryDataBits = enumerate(whileStatement->getBody(), ifDepth, whileDepth + 1);
408            CarryData * const nestedBlockData = mCarryInfoVector[whileStatement->getBody()->getScopeIndex()];
409            if (1 == mBitBlockWidth) {  // PACKING
410                if (cd->roomInFinalPack(nestedOffset) < whileCarryDataBits) {
411                    nestedOffset = alignCeiling(nestedOffset, mBitBlockWidth);
412                }
413            }
414            nestedBlockData->setFramePosition(nestedOffset);
415            nestedOffset += whileCarryDataBits;
416            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) {
417                cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
418            }
419            cd->nested.entries++;
420        }
421    }
422
423    cd->scopeCarryDataSize = nestedOffset;
424
425    if (cd->explicitSummaryRequired()) {
426        // Need extra space for the summary variable, always the last
427        // entry within an if block.
428        if (1 == mBitBlockWidth) {  // PACKING
429            cd->scopeCarryDataSize = alignCeiling(cd->scopeCarryDataSize, mBitBlockWidth);
430        }
431        cd->summary.frameOffset = cd->scopeCarryDataSize;
432        cd->scopeCarryDataSize += 1;  //  computed summary is a full pack.
433    } else {
434        cd->summary.frameOffset = 0;
435    }
436
437    return cd->scopeCarryDataSize;
438}
439
440/** ------------------------------------------------------------------------------------------------------------- *
441 * @brief addToSummary
442 ** ------------------------------------------------------------------------------------------------------------- */
443inline Value * CarryManager::addToSummary(Value * const value) {
444    const unsigned summaryIndex = summaryPack();
445    assert (summaryIndex < mCarryInPack.size());
446    Value * summary = mCarryOutPack[summaryIndex];
447    assert (summary);
448    assert (value);
449    if (LLVM_UNLIKELY(isa<Constant>(summary))) {
450        if (LLVM_LIKELY(cast<Constant>(summary)->isZeroValue())) {
451            summary = value;
452            goto return_result;
453        } else if (cast<Constant>(summary)->isAllOnesValue()) {
454            goto return_result;
455        }
456    }
457    if (LLVM_UNLIKELY(isa<Constant>(value))) {
458        if (LLVM_LIKELY(cast<Constant>(value)->isZeroValue())) {
459            goto return_result;
460        } else if (cast<Constant>(summary)->isAllOnesValue()) {
461            summary = value;
462            goto return_result;
463        }
464    }
465    if (LLVM_LIKELY(summary != value)) {
466        summary = iBuilder->CreateOr(summary, value, "summary");
467    }
468return_result:
469    mCarryOutPack[summaryIndex] = summary;
470    return summary;
471}
472
473/** ------------------------------------------------------------------------------------------------------------- *
474 * @brief getCarryPack
475 ** ------------------------------------------------------------------------------------------------------------- */
476Value * CarryManager::getCarryPack(const unsigned packIndex) {
477    assert (packIndex < mCarryInPack.size());
478    if (mCarryInPack[packIndex] == nullptr) {
479        Value * const packPtr = iBuilder->CreateGEP(mCarryPackBasePtr, ConstantInt::get(iBuilder->getSizeTy(), packIndex));
480        assert (packIndex < mCarryPackPtr.size());
481        mCarryPackPtr[packIndex] = packPtr;
482        mCarryInPack[packIndex] = iBuilder->CreateBlockAlignedLoad(packPtr);
483    }
484    return mCarryInPack[packIndex];
485}
486
487/** ------------------------------------------------------------------------------------------------------------- *
488 * @brief storeCarryOut
489 ** ------------------------------------------------------------------------------------------------------------- */
490void CarryManager::storeCarryOut(const unsigned packIndex) {
491    assert (packIndex < mCarryOutPack.size());
492    assert (mCarryOutPack[packIndex]);
493    assert (packIndex < mCarryPackPtr.size());
494    Value * const ptr = mCarryPackPtr[packIndex];
495    assert (ptr);
496    assert (cast<PointerType>(ptr->getType())->getElementType() == mBitBlockType);
497    Value * const value = iBuilder->CreateBitCast(mCarryOutPack[packIndex], mBitBlockType);
498    iBuilder->CreateBlockAlignedStore(value, ptr);
499}
500
501/* Helper routines */
502
503inline unsigned CarryManager::relativeFrameOffset(const unsigned frameOffset, const unsigned index) const {
504    return mCurrentFrameIndex + frameOffset + index;
505}
506
507inline unsigned CarryManager::addPosition(const unsigned localIndex) const {
508    assert (mCarryInfo);
509    return relativeFrameOffset(mCarryInfo->addWithCarry.frameOffset, localIndex);
510}
511
512inline unsigned CarryManager::unitAdvancePosition(const unsigned localIndex) const {
513    assert (mCarryInfo);
514    return relativeFrameOffset(mCarryInfo->unitAdvance.frameOffset, localIndex);
515}
516
517inline unsigned CarryManager::shortAdvancePosition(const unsigned localIndex) const {
518    assert (mCarryInfo);
519    return relativeFrameOffset(mCarryInfo->shortAdvance.frameOffset, localIndex);
520}
521
522inline unsigned CarryManager::longAdvancePosition(const unsigned localIndex) const {
523    assert (mCarryInfo);
524    return (mCurrentFrameIndex + mCarryInfo->longAdvance.frameOffset) + localIndex;
525}
526
527inline unsigned CarryManager::localBasePack() const {
528    assert (mCarryInfo);
529    return (mCurrentFrameIndex + mCarryInfo->shortAdvance.frameOffset);
530}
531
532inline unsigned CarryManager::scopeBasePack() const {
533    return mCurrentFrameIndex;
534}
535
536inline unsigned CarryManager::summaryPack() const {
537    assert (mCarryInfo);
538    return relativeFrameOffset(mCarryInfo->summary.frameOffset, 0);
539}
540
541inline bool CarryManager::hasSummary() const {
542    assert (mCarryInfo);
543    return mCarryInfo->explicitSummaryRequired() && !(mCarryInfo->hasLongAdvances());
544}
545
546CarryManager::~CarryManager() {
547    for (auto * cd : mCarryInfoVector) {
548        delete cd;
549    }
550}
551
552}
553
Note: See TracBrowser for help on using the repository browser.