source: icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp @ 5115

Last change on this file since 5115 was 5115, checked in by cameron, 3 years ago

Use bitblock_add_with_carry in carry_manager; add AVX2 implementation

File size: 25.8 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <stdexcept>
8#include <pablo/carry_data.h>
9#include <pablo/codegenstate.h>
10#include <pablo/carry_manager.h>
11#include <pablo/pabloAST.h>
12#include <llvm/Support/CommandLine.h>
13#include <llvm/IR/BasicBlock.h>
14#include <llvm/IR/CallingConv.h>
15#include <llvm/IR/Function.h>
16
17#define DSSLI_FIELDWIDTH 64
18
19namespace pablo {
20
21/** ------------------------------------------------------------------------------------------------------------- *
22 * @brief initializeCarryData
23 ** ------------------------------------------------------------------------------------------------------------- */
24Type * CarryManager::initializeCarryData(PabloFunction * const function) {
25    mRootScope = function->getEntryBlock();
26    mCarryInfoVector.resize(mRootScope->enumerateScopes(0) + 1);
27    mCarryPackType = mBitBlockType;
28    const unsigned totalCarryDataSize = enumerate(mRootScope, 0, 0);
29
30    mCarryPackPtr.resize(totalCarryDataSize + 1, nullptr);
31    mCarryInPack.resize(totalCarryDataSize + 1, nullptr);
32    mCarryOutPack.resize(totalCarryDataSize + 1, nullptr);
33
34    mTotalCarryDataBitBlocks = totalCarryDataSize;
35    ArrayType* cdArrayTy = ArrayType::get(mBitBlockType, mTotalCarryDataBitBlocks);
36    return cdArrayTy;
37}
38
39/** ------------------------------------------------------------------------------------------------------------- *
40 * @brief initializeCodeGen
41 ** ------------------------------------------------------------------------------------------------------------- */
42void CarryManager::initializeCodeGen(PabloKernel * const kBuilder, Value * selfPtr) {
43    mKernelBuilder = kBuilder;
44    mSelf = selfPtr;
45   
46    Value * cdArrayPtr = iBuilder->CreateGEP(mSelf, {ConstantInt::get(iBuilder->getSizeTy(), 0), mKernelBuilder->getScalarIndex("carries")});
47#ifndef NDEBUG
48    iBuilder->CallPrintInt("cdArrayPtr", iBuilder->CreatePtrToInt(cdArrayPtr, iBuilder->getSizeTy()));
49#endif
50    mCarryPackBasePtr = iBuilder->CreateBitCast(cdArrayPtr, PointerType::get(mCarryPackType, 0));
51    mCarryBitBlockPtr = iBuilder->CreateBitCast(cdArrayPtr, PointerType::get(mBitBlockType, 0));
52    mCurrentScope = mRootScope;
53    mCurrentFrameIndex = 0;
54    assert(mCarryInfoVector.size() > 0);
55    mCarryInfo = mCarryInfoVector[0];
56    assert(summaryPack() < mCarryOutPack.size());
57    mCarryOutPack[summaryPack()] = Constant::getNullValue(mCarryPackType);
58    assert (mCarrySummary.empty());
59}
60
61/** ------------------------------------------------------------------------------------------------------------- *
62 * @brief enterScope
63 ** ------------------------------------------------------------------------------------------------------------- */
64void CarryManager::enterScope(PabloBlock * const scope) {
65    assert(summaryPack() < mCarryOutPack.size());
66    Value * summaryCarry = mCarryOutPack[summaryPack()];
67    mCarrySummary.push_back(summaryCarry);
68    mCurrentScope = scope;
69    mCarryInfo = mCarryInfoVector[scope->getScopeIndex()];
70    mCurrentFrameIndex += mCarryInfo->getFrameIndex();
71    assert(summaryPack() < mCarryOutPack.size());
72    mCarryOutPack[summaryPack()] = Constant::getNullValue(mCarryPackType);
73}
74
75/** ------------------------------------------------------------------------------------------------------------- *
76 * @brief leaveScope
77 ** ------------------------------------------------------------------------------------------------------------- */
78void CarryManager::leaveScope() {
79    assert(summaryPack() < mCarryOutPack.size());
80    Value * summaryCarry = mCarryOutPack[summaryPack()];
81    assert (mCurrentScope != mRootScope);
82    mCurrentFrameIndex -= mCarryInfo->getFrameIndex();
83    mCurrentScope = mCurrentScope->getParent();
84    mCarryInfo = mCarryInfoVector[mCurrentScope->getScopeIndex()];
85    assert(summaryPack() < mCarryOutPack.size());
86    mCarryOutPack[summaryPack()] = summaryCarry;
87    mCarrySummary.pop_back();
88}
89
90/** ------------------------------------------------------------------------------------------------------------- *
91 * @brief addCarryInCarryOut
92 ** ------------------------------------------------------------------------------------------------------------- */
93Value * CarryManager::addCarryInCarryOut(const unsigned localIndex, Value * const e1, Value * const e2) {
94    std::pair<Value *, Value *> fullAdd = iBuilder->bitblock_add_with_carry(e1, e2, getCarryIn(localIndex));
95    setCarryOut(localIndex, std::get<0>(fullAdd));
96    return std::get<1>(fullAdd);
97}
98
99/** ------------------------------------------------------------------------------------------------------------- *
100 * @brief advanceCarryInCarryOut
101 ** ------------------------------------------------------------------------------------------------------------- */
102Value * CarryManager::advanceCarryInCarryOut(const unsigned localIndex, const unsigned shiftAmount, Value * const value) {
103    if (LLVM_LIKELY(shiftAmount == 1)) {
104        return shortAdvanceCarryInCarryOut(unitAdvancePosition(localIndex), shiftAmount, value);
105    } else if (shiftAmount < LongAdvanceBase) {
106        return shortAdvanceCarryInCarryOut(shortAdvancePosition(localIndex), shiftAmount, value);
107    } else {
108        return longAdvanceCarryInCarryOut(longAdvancePosition(localIndex), shiftAmount, value);
109    }
110}
111
112/** ------------------------------------------------------------------------------------------------------------- *
113 * @brief shortAdvanceCarryInCarryOut
114 ** ------------------------------------------------------------------------------------------------------------- */
115Value * CarryManager::shortAdvanceCarryInCarryOut(const unsigned index, const unsigned shiftAmount, Value * const value) {
116    Value * result = nullptr;
117    Value * const carryIn = getCarryPack(index);
118    assert (index < mCarryOutPack.size());
119    mCarryOutPack[index] = value;
120    if (mCarryInfo->getWhileDepth() == 0) {
121        storeCarryOut(index);
122    }
123    if (LLVM_UNLIKELY((shiftAmount % 8) == 0)) { // Use a single whole-byte shift, if possible.
124        result = iBuilder->mvmd_dslli(8, value, carryIn, (iBuilder->getBitBlockWidth() / 8) - (shiftAmount / 8));
125    } else if (LLVM_LIKELY(shiftAmount < DSSLI_FIELDWIDTH)) {
126        Value * ahead = iBuilder->mvmd_dslli(DSSLI_FIELDWIDTH, value, carryIn, iBuilder->getBitBlockWidth() / DSSLI_FIELDWIDTH - 1);
127        result = iBuilder->simd_or(iBuilder->simd_srli(DSSLI_FIELDWIDTH, ahead, DSSLI_FIELDWIDTH - shiftAmount), iBuilder->simd_slli(DSSLI_FIELDWIDTH, value, shiftAmount));
128    } else {
129        Value* advanceq_longint = iBuilder->CreateBitCast(carryIn, iBuilder->getIntNTy(mBitBlockWidth));
130        Value* strm_longint = iBuilder->CreateBitCast(value, iBuilder->getIntNTy(mBitBlockWidth));
131        Value* adv_longint = iBuilder->CreateOr(iBuilder->CreateShl(strm_longint, shiftAmount), iBuilder->CreateLShr(advanceq_longint, mBitBlockWidth - shiftAmount), "advance");
132        result = iBuilder->CreateBitCast(adv_longint, mBitBlockType);
133    }
134    if (LLVM_LIKELY(hasSummary())) {
135        addToSummary(value);
136    }
137    return result;
138}
139
140/** ------------------------------------------------------------------------------------------------------------- *
141 * @brief longAdvanceCarryInCarryOut
142 ** ------------------------------------------------------------------------------------------------------------- */
143Value * CarryManager::longAdvanceCarryInCarryOut(const unsigned index, const unsigned shiftAmount, Value * const value) {
144    Value * advBaseIndex = ConstantInt::get(iBuilder->getSizeTy(), index);
145    if (shiftAmount <= mBitBlockWidth) {
146        // special case using a single buffer entry and the carry_out value.
147        Value * advanceDataPtr = iBuilder->CreateGEP(mCarryBitBlockPtr, advBaseIndex);
148        Value * carry_block0 = iBuilder->CreateBlockAlignedLoad(advanceDataPtr);
149        iBuilder->CreateBlockAlignedStore(value, advanceDataPtr);
150        /* Very special case - no combine */
151        if (shiftAmount == mBitBlockWidth) {
152            return carry_block0;
153        }
154        Value* block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carry_block0, iBuilder->getIntNTy(mBitBlockWidth)), mBitBlockWidth - shiftAmount);
155        Value* block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(value, iBuilder->getIntNTy(mBitBlockWidth)), shiftAmount);
156        return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
157    }
158    // We need a buffer of at least two elements for storing the advance data.
159    const unsigned block_shift = shiftAmount % mBitBlockWidth;
160    const unsigned advanceEntries = mCarryInfo->longAdvanceEntries(shiftAmount);
161    const unsigned bufsize = mCarryInfo->longAdvanceBufferSize(shiftAmount);
162    Value * indexMask = ConstantInt::get(iBuilder->getSizeTy(), bufsize - 1);  // A mask to implement circular buffer indexing
163    Value * blockIndex = mKernelBuilder->getScalarField(mSelf, blockNoScalar);
164    Value * loadIndex0 = iBuilder->CreateAdd(iBuilder->CreateAnd(iBuilder->CreateSub(blockIndex, ConstantInt::get(iBuilder->getSizeTy(), advanceEntries)), indexMask), advBaseIndex);
165    Value * storeIndex = iBuilder->CreateAdd(iBuilder->CreateAnd(blockIndex, indexMask), advBaseIndex);
166    Value * carry_block0 = iBuilder->CreateBlockAlignedLoad(iBuilder->CreateGEP(mCarryBitBlockPtr, loadIndex0));
167    // If the long advance is an exact multiple of mBITBLOCK_WIDTH, we simply return the oldest
168    // block in the long advance carry data area. 
169    if (block_shift == 0) {
170        iBuilder->CreateBlockAlignedStore(value, iBuilder->CreateGEP(mCarryBitBlockPtr, storeIndex));
171        return carry_block0;
172    }
173    // Otherwise we need to combine data from the two oldest blocks.
174    Value * loadIndex1 = iBuilder->CreateAdd(iBuilder->CreateAnd(iBuilder->CreateSub(blockIndex, ConstantInt::get(iBuilder->getSizeTy(), advanceEntries-1)), indexMask), advBaseIndex);
175    Value * carry_block1 = iBuilder->CreateBlockAlignedLoad(iBuilder->CreateGEP(mCarryBitBlockPtr, loadIndex1));
176    Value* block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carry_block0, iBuilder->getIntNTy(mBitBlockWidth)), mBitBlockWidth - block_shift);
177    Value* block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(carry_block1, iBuilder->getIntNTy(mBitBlockWidth)), block_shift);
178    iBuilder->CreateBlockAlignedStore(value, iBuilder->CreateGEP(mCarryBitBlockPtr, storeIndex));
179    return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), mBitBlockType);
180}
181
182/** ------------------------------------------------------------------------------------------------------------- *
183 * @brief generateSummaryTest
184 ** ------------------------------------------------------------------------------------------------------------- */
185Value * CarryManager::generateSummaryTest(Value * condition) {
186    if (mCarryInfo->hasCarries()) {
187        Value * summary_pack = getCarryPack(summaryPack());
188        condition = iBuilder->simd_or(condition, summary_pack);
189    }
190    return iBuilder->bitblock_any(condition);
191}
192
193/** ------------------------------------------------------------------------------------------------------------- *
194 * @brief storeCarryOutSummary
195 ** ------------------------------------------------------------------------------------------------------------- */
196void CarryManager::storeCarryOutSummary() {
197    if (LLVM_LIKELY(mCarryInfo->explicitSummaryRequired())) {
198        const unsigned carrySummaryIndex = summaryPack();
199        if (LLVM_UNLIKELY(mCarryInfo->hasLongAdvances())) { // Force if entry
200            assert (carrySummaryIndex < mCarryOutPack.size());
201            mCarryOutPack[carrySummaryIndex] = Constant::getAllOnesValue(mCarryPackType);
202        }
203        storeCarryOut(carrySummaryIndex);
204    }
205}
206
207/** ------------------------------------------------------------------------------------------------------------- *
208 * @brief addOuterSummaryToNestedSummary
209 ** ------------------------------------------------------------------------------------------------------------- */
210void CarryManager::addOuterSummaryToNestedSummary() {
211    if (LLVM_LIKELY(mCarrySummary.size() > 0)) {
212        addToSummary(mCarrySummary.back());
213    }
214}
215
216/** ------------------------------------------------------------------------------------------------------------- *
217 * @brief buildCarryDataPhisAfterIfBody
218 ** ------------------------------------------------------------------------------------------------------------- */
219void CarryManager::buildCarryDataPhisAfterIfBody(BasicBlock * const entry, BasicBlock * const end) {
220    if (mCarryInfo->getWhileDepth() > 0) {
221        // We need to phi out everything for the while carry accumulation process.
222        const unsigned scopeBaseOffset = scopeBasePack();
223        const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
224        for (unsigned i = scopeBaseOffset; i < scopeBaseOffset + scopeCarryPacks; ++i) {
225            assert (i < mCarryOutPack.size());
226            Type * const type = mCarryOutPack[i]->getType();
227            PHINode * phi = iBuilder->CreatePHI(type, 2);
228            phi->addIncoming(Constant::getNullValue(type), entry);
229            phi->addIncoming(mCarryOutPack[i], end);
230            mCarryOutPack[i] = phi;
231        }
232    }
233    if (LLVM_LIKELY(mCarrySummary.size() > 0)) {
234        const unsigned summaryIndex = summaryPack();
235        assert (summaryIndex < mCarryOutPack.size());
236        Value * carrySummary = mCarryOutPack[summaryIndex];
237        if (mCarrySummary.back() != carrySummary) {
238            Value * outerCarrySummary = mCarrySummary.back();
239            Value * nestedCarrySummary = mCarryOutPack[summaryIndex];
240            assert (outerCarrySummary->getType() == nestedCarrySummary->getType());
241            PHINode * const phi = iBuilder->CreatePHI(outerCarrySummary->getType(), 2, "summary");
242            phi->addIncoming(outerCarrySummary, entry);
243            phi->addIncoming(nestedCarrySummary, end);
244            mCarryOutPack[summaryIndex] = phi;
245        }
246    }
247}
248
249/** ------------------------------------------------------------------------------------------------------------- *
250 * @brief initializeWhileEntryCarryDataPhis
251 ** ------------------------------------------------------------------------------------------------------------- */
252void CarryManager::initializeWhileEntryCarryDataPhis(BasicBlock * const end) {
253    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
254    mCarryOutAccumPhis.resize(scopeCarryPacks);
255    #ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
256    const unsigned currentScopeBase = scopeBasePack();
257    mCarryInPhis.resize(scopeCarryPacks);
258    #endif
259    for (unsigned index = 0; index < scopeCarryPacks; ++index) {
260        #ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
261        PHINode * phi_in = iBuilder->CreatePHI(mCarryPackType, 2);
262        phi_in->addIncoming(mCarryInPack[currentScopeBase+index], whileEntryBlock);
263        mCarryInPhis[index] = phi_in;
264        #endif
265        PHINode * phi_out = iBuilder->CreatePHI(mCarryPackType, 2);
266        phi_out->addIncoming(Constant::getNullValue(mCarryPackType), end);
267        assert (index < mCarryOutAccumPhis.size());
268        mCarryOutAccumPhis[index] = phi_out;
269    }
270}
271
272/** ------------------------------------------------------------------------------------------------------------- *
273 * @brief finalizeWhileBlockCarryDataPhis
274 ** ------------------------------------------------------------------------------------------------------------- */
275void CarryManager::finalizeWhileBlockCarryDataPhis(BasicBlock * const end) {
276    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
277    const unsigned currentScopeBase = scopeBasePack();
278    for (unsigned index = 0; index < scopeCarryPacks; ++index) {
279        #ifdef SET_WHILE_CARRY_IN_TO_ZERO_AFTER_FIRST_ITERATION
280        mCarryInPhis[index]->addIncoming(Constant::getNullValue(mCarryPackType), whileBodyFinalBlock);
281        #endif
282        assert (index < mCarryOutAccumPhis.size());
283        PHINode * phi = mCarryOutAccumPhis[index];
284        Value * carryOut = iBuilder->CreateOr(phi, mCarryOutPack[currentScopeBase + index]);
285        phi->addIncoming(carryOut, end);
286        mCarryOutPack[currentScopeBase + index] = carryOut;
287    }
288}
289
290/** ------------------------------------------------------------------------------------------------------------- *
291 * @brief ensureCarriesLoadedRecursive
292 ** ------------------------------------------------------------------------------------------------------------- */
293void CarryManager::ensureCarriesLoadedRecursive() {
294    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
295    const unsigned currentScopeBase = scopeBasePack();
296    if (mCarryInfo->getWhileDepth() == 1) {
297        for (auto i = currentScopeBase; i < currentScopeBase + scopeCarryPacks; ++i) {
298            getCarryPack(i);
299        }
300    }
301}
302
303/** ------------------------------------------------------------------------------------------------------------- *
304 * @brief ensureCarriesStoredRecursive
305 ** ------------------------------------------------------------------------------------------------------------- */
306void CarryManager::ensureCarriesStoredRecursive() {
307    const unsigned scopeCarryPacks = mCarryInfo->getScopeCarryPackCount();
308    const unsigned currentScopeBase = scopeBasePack();
309    if (mCarryInfo->getWhileDepth() == 1) {
310        for (auto i = currentScopeBase; i < currentScopeBase + scopeCarryPacks; ++i) {
311            storeCarryOut(i);
312        }
313    }
314}
315
316/** ------------------------------------------------------------------------------------------------------------- *
317 * @brief getCarryIn
318 ** ------------------------------------------------------------------------------------------------------------- */
319Value * CarryManager::getCarryIn(const unsigned localIndex) {
320    return getCarryPack(addPosition(localIndex));
321}
322
323/** ------------------------------------------------------------------------------------------------------------- *
324 * @brief setCarryOut
325 ** ------------------------------------------------------------------------------------------------------------- */
326void CarryManager::setCarryOut(const unsigned localIndex, Value * carryOut) {
327    const unsigned index = addPosition(localIndex);
328    assert (index < mCarryOutPack.size());
329    mCarryOutPack[index] = carryOut;
330    if (LLVM_LIKELY(hasSummary())) {
331        addToSummary(carryOut);
332    }
333    if (mCarryInfo->getWhileDepth() == 0) {
334        storeCarryOut(index);
335    }
336}
337
338/** ------------------------------------------------------------------------------------------------------------- *
339 * @brief enumerate
340 ** ------------------------------------------------------------------------------------------------------------- */
341unsigned CarryManager::enumerate(PabloBlock * blk, unsigned ifDepth, unsigned whileDepth) {
342    unsigned idx = blk->getScopeIndex();
343    CarryData * cd = new CarryData(blk, mBitBlockWidth, 1, mBitBlockWidth);
344    mCarryInfoVector[idx] = cd;
345
346    cd->setIfDepth(ifDepth);
347    cd->setWhileDepth(whileDepth);
348    unsigned nestedOffset = cd->nested.frameOffset;
349
350    for (Statement * stmt : *blk) {
351        if (If * ifStatement = dyn_cast<If>(stmt)) {
352            const unsigned ifCarryDataBits = enumerate(ifStatement->getBody(), ifDepth + 1, whileDepth);
353            CarryData * nestedBlockData = mCarryInfoVector[ifStatement->getBody()->getScopeIndex()];
354            if (1 == mBitBlockWidth) {  // PACKING
355                if (cd->roomInFinalPack(nestedOffset) < ifCarryDataBits) {
356                    nestedOffset = alignCeiling(nestedOffset, mBitBlockWidth);
357                }
358            }
359            nestedBlockData->setFramePosition(nestedOffset);
360            nestedOffset += ifCarryDataBits;
361            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) {
362                cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
363            }
364            cd->nested.entries++;
365        } else if (While * whileStatement = dyn_cast<While>(stmt)) {
366            const unsigned whileCarryDataBits = enumerate(whileStatement->getBody(), ifDepth, whileDepth + 1);
367            CarryData * const nestedBlockData = mCarryInfoVector[whileStatement->getBody()->getScopeIndex()];
368            if (1 == mBitBlockWidth) {  // PACKING
369                if (cd->roomInFinalPack(nestedOffset) < whileCarryDataBits) {
370                    nestedOffset = alignCeiling(nestedOffset, mBitBlockWidth);
371                }
372            }
373            nestedBlockData->setFramePosition(nestedOffset);
374            nestedOffset += whileCarryDataBits;
375            if (cd->maxNestingDepth <= nestedBlockData->maxNestingDepth) {
376                cd->maxNestingDepth = nestedBlockData->maxNestingDepth + 1;
377            }
378            cd->nested.entries++;
379        }
380    }
381
382    cd->scopeCarryDataSize = nestedOffset;
383
384    if (cd->explicitSummaryRequired()) {
385        // Need extra space for the summary variable, always the last
386        // entry within an if block.
387        if (1 == mBitBlockWidth) {  // PACKING
388            cd->scopeCarryDataSize = alignCeiling(cd->scopeCarryDataSize, mBitBlockWidth);
389        }
390        cd->summary.frameOffset = cd->scopeCarryDataSize;
391        cd->scopeCarryDataSize += 1;  //  computed summary is a full pack.
392    } else {
393        cd->summary.frameOffset = 0;
394    }
395
396    return cd->scopeCarryDataSize;
397}
398
399/** ------------------------------------------------------------------------------------------------------------- *
400 * @brief addToSummary
401 ** ------------------------------------------------------------------------------------------------------------- */
402void CarryManager::addToSummary(Value * const value) {
403    const unsigned summaryIndex = summaryPack();
404    assert (summaryIndex < mCarryInPack.size());
405    Value * summary = mCarryOutPack[summaryIndex];
406    assert (summary);
407    assert (value);
408    if (LLVM_UNLIKELY(summary == value)) return;  //Nothing to add.
409   
410    Type * summaryTy = summary->getType();
411    Type * valueTy = value->getType();
412    if (LLVM_UNLIKELY(isa<Constant>(value))) {
413        if (LLVM_LIKELY(cast<Constant>(value)->isZeroValue())) return;
414        if (cast<Constant>(value)->isAllOnesValue()) {
415            mCarryOutPack[summaryIndex] = Constant::getAllOnesValue(summaryTy);
416            return;
417        }
418    }
419    Value * v = value;
420    if (valueTy != summaryTy) {
421        // valueTy must be an integer type.
422        unsigned summaryWidth = summaryTy->isIntegerTy() ? summaryTy->getIntegerBitWidth() : cast<VectorType>(summaryTy)->getBitWidth();       
423        if (valueTy->getIntegerBitWidth() != summaryWidth) {
424            v = iBuilder->CreateZExt(v, iBuilder->getIntNTy(summaryWidth));
425        }
426        if (!(summaryTy->isIntegerTy())) {
427            v = iBuilder->CreateBitCast(v, summaryTy);
428        }
429    }
430    if (LLVM_UNLIKELY(isa<Constant>(summary))) {
431        if (LLVM_LIKELY(cast<Constant>(summary)->isZeroValue())) {
432            mCarryOutPack[summaryIndex] = v;
433            return;
434        } else if (cast<Constant>(summary)->isAllOnesValue()) {
435            return;
436        }
437    }
438    mCarryOutPack[summaryIndex] = iBuilder->CreateOr(summary, v, "summary");
439}
440
441/** ------------------------------------------------------------------------------------------------------------- *
442 * @brief getCarryPack
443 ** ------------------------------------------------------------------------------------------------------------- */
444Value * CarryManager::getCarryPack(const unsigned packIndex) {
445    assert (packIndex < mCarryInPack.size());
446    if (mCarryInPack[packIndex] == nullptr) {
447        Value * const packPtr = iBuilder->CreateGEP(mCarryPackBasePtr, ConstantInt::get(iBuilder->getSizeTy(), packIndex));
448        assert (packIndex < mCarryPackPtr.size());
449        mCarryPackPtr[packIndex] = packPtr;
450        mCarryInPack[packIndex] = iBuilder->CreateBlockAlignedLoad(packPtr);
451    }
452    return mCarryInPack[packIndex];
453}
454
455/** ------------------------------------------------------------------------------------------------------------- *
456 * @brief storeCarryOut
457 ** ------------------------------------------------------------------------------------------------------------- */
458void CarryManager::storeCarryOut(const unsigned packIndex) {
459    assert (packIndex < mCarryOutPack.size());
460    assert (mCarryOutPack[packIndex]);
461    assert (packIndex < mCarryPackPtr.size());
462    Value * const ptr = mCarryPackPtr[packIndex];
463    assert (ptr);
464    assert (cast<PointerType>(ptr->getType())->getElementType() == mBitBlockType);
465    Value * const value = iBuilder->CreateBitCast(mCarryOutPack[packIndex], mBitBlockType);
466    iBuilder->CreateBlockAlignedStore(value, ptr);
467}
468
469/* Helper routines */
470
471inline unsigned CarryManager::relativeFrameOffset(const unsigned frameOffset, const unsigned index) const {
472    return mCurrentFrameIndex + frameOffset + index;
473}
474
475inline unsigned CarryManager::addPosition(const unsigned localIndex) const {
476    assert (mCarryInfo);
477    return relativeFrameOffset(mCarryInfo->addWithCarry.frameOffset, localIndex);
478}
479
480inline unsigned CarryManager::unitAdvancePosition(const unsigned localIndex) const {
481    assert (mCarryInfo);
482    return relativeFrameOffset(mCarryInfo->unitAdvance.frameOffset, localIndex);
483}
484
485inline unsigned CarryManager::shortAdvancePosition(const unsigned localIndex) const {
486    assert (mCarryInfo);
487    return relativeFrameOffset(mCarryInfo->shortAdvance.frameOffset, localIndex);
488}
489
490inline unsigned CarryManager::longAdvancePosition(const unsigned localIndex) const {
491    assert (mCarryInfo);
492    return (mCurrentFrameIndex + mCarryInfo->longAdvance.frameOffset) + localIndex;
493}
494
495inline unsigned CarryManager::localBasePack() const {
496    assert (mCarryInfo);
497    return (mCurrentFrameIndex + mCarryInfo->shortAdvance.frameOffset);
498}
499
500inline unsigned CarryManager::scopeBasePack() const {
501    return mCurrentFrameIndex;
502}
503
504inline unsigned CarryManager::summaryPack() const {
505    assert (mCarryInfo);
506    return relativeFrameOffset(mCarryInfo->summary.frameOffset, 0);
507}
508
509inline bool CarryManager::hasSummary() const {
510    assert (mCarryInfo);
511    return mCarryInfo->explicitSummaryRequired() && !(mCarryInfo->hasLongAdvances());
512}
513
514CarryManager::~CarryManager() {
515    for (auto * cd : mCarryInfoVector) {
516        delete cd;
517    }
518}
519
520}
521
Note: See TracBrowser for help on using the repository browser.