source: icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp @ 5464

Last change on this file since 5464 was 5464, checked in by nmedfort, 2 years ago

Restructuring work for the Driver classes. Start of work to eliminate the memory leaks with the ExecutionEngine?. Replaced custom AlignedMalloc? with backend call to std::aligned_malloc. Salvaged some work on DistributionPass? for reevaluation.

File size: 45.3 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include "carry_manager.h"
8#include <pablo/carry_data.h>
9#include <pablo/codegenstate.h>
10#include <llvm/IR/BasicBlock.h>
11#include <llvm/IR/DerivedTypes.h>
12#include <pablo/branch.h>
13#include <pablo/pe_advance.h>
14#include <pablo/pe_scanthru.h>
15#include <pablo/pe_matchstar.h>
16#include <pablo/pe_var.h>
17#include <kernels/kernel_builder.h>
18#include <llvm/Support/raw_ostream.h>
19
20using namespace llvm;
21
22namespace pablo {
23
24inline static unsigned ceil_log2(const unsigned v) {
25    assert ("log2(0) is undefined!" && v != 0);
26    return 32 - __builtin_clz(v - 1);
27}
28
29inline static unsigned floor_log2(const unsigned v) {
30    assert ("log2(0) is undefined!" && v != 0);
31    return 31 - __builtin_clz(v);
32}
33
34inline static unsigned nearest_pow2(const unsigned v) {
35    assert(v > 0 && v < (UINT32_MAX / 2));
36    return (v < 2) ? 1 : (1 << ceil_log2(v));
37}
38
39inline static unsigned ceil_udiv(const unsigned x, const unsigned y) {
40    return (((x - 1) | (y - 1)) + 1) / y;
41}
42
43using TypeId = PabloAST::ClassTypeId;
44
45inline static bool isNonAdvanceCarryGeneratingStatement(const Statement * const stmt) {
46    switch (stmt->getClassTypeId()) {
47        case TypeId::ScanThru:
48        case TypeId::AdvanceThenScanThru:
49        case TypeId::ScanTo:
50        case TypeId::AdvanceThenScanTo:
51        case TypeId::MatchStar:
52            return true;
53        default:
54            return false;
55    }
56}
57
58#define LONG_ADVANCE_BREAKPOINT 64
59
60/** ------------------------------------------------------------------------------------------------------------- *
61 * @brief initializeCarryData
62 ** ------------------------------------------------------------------------------------------------------------- */
63void CarryManager::initializeCarryData(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, PabloKernel * const kernel) {
64
65    // Each scope constructs its own CarryData struct, which will be added to the final "carries" struct
66    // that is added to the Kernel. The scope index will indicate which struct to access.
67
68    // A CarryData struct either contains an array of CarryPackBlocks or an integer indicating the capacity of
69    // the variable length CarryData struct and pointer. A variable length CarryData struct is required whenever
70    // the streams accessed by a loop could vary between iterations. When resizing a CarryData struct for a
71    // particular loop, the current loop struct and all nested structs need to be resized. This accommodates
72    // the fact every pablo While loop must be executed at least once.
73
74    // A nested loop may also contain a variable length CarryData struct
75
76    // To determine whether we require a variable length CarryData struct, we test the escaped variables of
77    // each loop branch to see whether they are used as the index parameter of a nested Extract statement.
78    // Any scope that requires variable length CarryData, requires that all nested branches have a unique
79    // set of carries for that iteration.
80
81    assert (mKernel == nullptr);
82    mCurrentScope = kernel->getEntryBlock();
83    mKernel = kernel;
84
85    mCarryScopes = 0;
86
87    mCarryMetadata.resize(getScopeCount(mCurrentScope));
88
89    Type * const carryStateTy = analyse(iBuilder, mCurrentScope);
90
91    kernel->addScalar(carryStateTy, "carries");
92
93    if (mHasLoop) {
94        kernel->addScalar(iBuilder->getInt32Ty(), "selector");
95    }
96    if (mHasLongAdvance) {
97        kernel->addScalar(iBuilder->getSizeTy(), "CarryBlockIndex");
98    }
99
100}
101
102/** ------------------------------------------------------------------------------------------------------------- *
103 * @brief initializeCodeGen
104 ** ------------------------------------------------------------------------------------------------------------- */
105void CarryManager::initializeCodeGen(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder) {
106
107    assert(!mCarryMetadata.empty());
108    mCarryInfo = &mCarryMetadata[0];
109    assert (!mCarryInfo->hasSummary());
110
111    mCurrentFrame = iBuilder->getScalarFieldPtr("carries");
112    mCurrentFrameIndex = 0;
113    mCarryScopes = 0;
114    mCarryScopeIndex.push_back(0);
115
116    assert (mCarryFrameStack.empty());
117
118    assert (mCarrySummaryStack.empty());
119
120    Type * const carryTy = iBuilder->getBitBlockType();
121
122    mCarrySummaryStack.push_back(Constant::getNullValue(carryTy));
123
124    if (mHasLoop) {       
125        mLoopSelector = iBuilder->getScalarField("selector");
126        mNextLoopSelector = iBuilder->CreateXor(mLoopSelector, ConstantInt::get(mLoopSelector->getType(), 1));
127    }
128
129}
130
131/** ------------------------------------------------------------------------------------------------------------- *
132 * @brief finalizeCodeGen
133 ** ------------------------------------------------------------------------------------------------------------- */
134void CarryManager::finalizeCodeGen(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder) {
135    if (mHasLoop) {
136        iBuilder->setScalarField("selector", mNextLoopSelector);
137    }
138    if (mHasLongAdvance) {
139        Value * idx = iBuilder->getScalarField("CarryBlockIndex");
140        idx = iBuilder->CreateAdd(idx, iBuilder->getSize(1));
141        iBuilder->setScalarField("CarryBlockIndex", idx);
142    }
143    assert (mCarryFrameStack.empty());   
144    assert ("base summary value was deleted!" && mCarrySummaryStack.size() == 1);
145    assert ("base summary value was overwritten with non-zero value!" && isa<Constant>(mCarrySummaryStack[0]) && cast<Constant>(mCarrySummaryStack[0])->isNullValue());
146    mCarrySummaryStack.clear();
147    assert (mCarryScopeIndex.size() == 1);
148    mCarryScopeIndex.clear();
149}
150
151/** ------------------------------------------------------------------------------------------------------------- *
152 * @brief enterLoopScope
153 ** ------------------------------------------------------------------------------------------------------------- */
154void CarryManager::enterLoopScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope) {
155    assert (scope);
156    assert (mHasLoop);
157    ++mLoopDepth;
158    enterScope(iBuilder, scope);
159}
160
161/** ------------------------------------------------------------------------------------------------------------- *
162 * @brief enterLoopBody
163 ** ------------------------------------------------------------------------------------------------------------- */
164void CarryManager::enterLoopBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * const entryBlock) {
165    if (mCarryInfo->hasSummary()) {
166        Type * const carryTy = iBuilder->getBitBlockType();
167        PHINode * phiCarryOutSummary = iBuilder->CreatePHI(carryTy, 2, "summary");
168        assert (!mCarrySummaryStack.empty());
169        phiCarryOutSummary->addIncoming(mCarrySummaryStack.back(), entryBlock);
170        // Replace the incoming carry summary with the phi node and add the phi node to the stack  so that we can
171        // properly OR it into the outgoing summary value.
172        // NOTE: this may change the base summary value; when exiting to the base scope, replace this summary with
173        // a null value to prevent subsequent nested scopes from inheriting the summary of this scope.
174        mCarrySummaryStack.back() = phiCarryOutSummary;
175        mCarrySummaryStack.push_back(phiCarryOutSummary);
176    }
177    if (LLVM_UNLIKELY(mCarryInfo->nonCarryCollapsingMode())) {
178
179        assert (mCarryInfo->hasSummary());
180
181        Type * const carryTy = iBuilder->getBitBlockType();
182        PointerType * const carryPtrTy = carryTy->getPointerTo();
183
184        // Check whether we need to resize the carry state
185        PHINode * index = iBuilder->CreatePHI(iBuilder->getSizeTy(), 2);
186        mLoopIndicies.push_back(index);
187        index->addIncoming(iBuilder->getSize(0), entryBlock);
188
189        Value * capacityPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
190        Value * capacity = iBuilder->CreateLoad(capacityPtr, false, "capacity");
191        Constant * const ONE = ConstantInt::get(capacity->getType(), 1);
192        Value * arrayPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
193        Value * array = iBuilder->CreateLoad(arrayPtr, false, "array");
194
195        BasicBlock * const entry = iBuilder->GetInsertBlock();
196        BasicBlock * const resizeCarryState = iBuilder->CreateBasicBlock("ResizeCarryState");
197        BasicBlock * const reallocExisting = iBuilder->CreateBasicBlock("ReallocExisting");
198        BasicBlock * const createNew = iBuilder->CreateBasicBlock("CreateNew");
199        BasicBlock * const resumeKernel = iBuilder->CreateBasicBlock("ResumeKernel");
200
201        iBuilder->CreateLikelyCondBr(iBuilder->CreateICmpULT(index, capacity), resumeKernel, resizeCarryState);
202
203        // RESIZE CARRY BLOCK
204        iBuilder->SetInsertPoint(resizeCarryState);
205        const auto BlockWidth = carryTy->getPrimitiveSizeInBits() / 8;
206        const auto Log2BlockWidth = floor_log2(BlockWidth);
207        Constant * const carryStateWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(array->getType()->getPointerElementType()), iBuilder->getSizeTy(), false);
208        Value * summaryPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(2)});
209
210        Value * const hasCarryState = iBuilder->CreateICmpNE(array, ConstantPointerNull::get(cast<PointerType>(array->getType())));
211
212        iBuilder->CreateLikelyCondBr(hasCarryState, reallocExisting, createNew);
213
214        // REALLOCATE EXISTING
215        iBuilder->SetInsertPoint(reallocExisting);
216
217        Value * const capacitySize = iBuilder->CreateMul(capacity, carryStateWidth);
218        Value * const newCapacitySize = iBuilder->CreateShl(capacitySize, 1); // x 2
219
220
221        Value * newArray = iBuilder->CreateAlignedMalloc(newCapacitySize, iBuilder->getCacheAlignment());
222        iBuilder->CreateMemCpy(newArray, array, capacitySize, BlockWidth);
223        iBuilder->CreateMemZero(iBuilder->CreateGEP(newArray, capacitySize), capacitySize, BlockWidth);
224        iBuilder->CreateFree(array);
225        newArray = iBuilder->CreatePointerCast(newArray, array->getType());
226        iBuilder->CreateStore(newArray, arrayPtr);
227
228        Value * const log2capacity = iBuilder->CreateAdd(iBuilder->CreateCeilLog2(capacity), ONE);
229        Value * const summarySize = iBuilder->CreateShl(log2capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
230        Value * const newLog2Capacity = iBuilder->CreateAdd(log2capacity, ONE);
231        Value * const newSummarySize = iBuilder->CreateShl(newLog2Capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
232
233        Value * const summary = iBuilder->CreateLoad(summaryPtr, false);
234        Value * newSummary = iBuilder->CreateAlignedMalloc(newSummarySize, BlockWidth);
235        iBuilder->CreateMemCpy(newSummary, summary, summarySize, BlockWidth);
236        iBuilder->CreateMemZero(iBuilder->CreateGEP(newSummary, summarySize), iBuilder->getSize(2 * BlockWidth), BlockWidth);
237        iBuilder->CreateFree(summary);
238
239        Value * ptr1 = iBuilder->CreateGEP(newSummary, summarySize);
240        ptr1 = iBuilder->CreatePointerCast(ptr1, carryPtrTy);
241
242        Value * ptr2 = iBuilder->CreateGEP(newSummary, iBuilder->CreateAdd(summarySize, iBuilder->getSize(BlockWidth)));
243        ptr2 = iBuilder->CreatePointerCast(ptr2, carryPtrTy);
244
245        newSummary = iBuilder->CreatePointerCast(newSummary, carryPtrTy);
246        iBuilder->CreateStore(newSummary, summaryPtr);
247        Value * const newCapacity = iBuilder->CreateShl(ONE, log2capacity);
248
249        iBuilder->CreateStore(newCapacity, capacityPtr);
250
251        iBuilder->CreateBr(resumeKernel);
252
253        // CREATE NEW
254        iBuilder->SetInsertPoint(createNew);
255
256        Constant * const initialLog2Capacity = iBuilder->getInt64(4);
257        Constant * const initialCapacity = ConstantExpr::getShl(ONE, initialLog2Capacity);
258        Constant * const initialCapacitySize = ConstantExpr::getMul(initialCapacity, carryStateWidth);
259
260        Value * initialArray = iBuilder->CreateAlignedMalloc(initialCapacitySize, iBuilder->getCacheAlignment());
261        iBuilder->CreateMemZero(initialArray, initialCapacitySize, BlockWidth);
262        initialArray = iBuilder->CreatePointerCast(initialArray, array->getType());
263        iBuilder->CreateStore(initialArray, arrayPtr);
264
265        Constant * initialSummarySize = ConstantExpr::getShl(ConstantExpr::getAdd(initialLog2Capacity, iBuilder->getInt64(1)), iBuilder->getInt64(Log2BlockWidth + 1));
266        Value * initialSummary = iBuilder->CreateAlignedMalloc(initialSummarySize, BlockWidth);
267        iBuilder->CreateMemZero(initialSummary, initialSummarySize, BlockWidth);
268        initialSummary = iBuilder->CreatePointerCast(initialSummary, carryPtrTy);
269        iBuilder->CreateStore(initialSummary, summaryPtr);
270
271        iBuilder->CreateStore(initialCapacity, capacityPtr);
272
273        iBuilder->CreateBr(resumeKernel);
274
275        // RESUME KERNEL
276        iBuilder->SetInsertPoint(resumeKernel);
277        // Load the appropriate carry stat block
278        PHINode * phiArrayPtr = iBuilder->CreatePHI(array->getType(), 3);
279        phiArrayPtr->addIncoming(array, entry);
280        phiArrayPtr->addIncoming(initialArray, createNew);
281        phiArrayPtr->addIncoming(newArray, reallocExisting);
282
283        // NOTE: the 3 here is only to pass the assertion later. It refers to the number of elements in the carry data struct.
284        mCarryFrameStack.emplace_back(mCurrentFrame, 3);
285        mCurrentFrame = iBuilder->CreateGEP(phiArrayPtr, index);
286    }
287}
288
289/** ------------------------------------------------------------------------------------------------------------- *
290 * @brief leaveLoopBody
291 ** ------------------------------------------------------------------------------------------------------------- */
292void CarryManager::leaveLoopBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * /* exitBlock */) {
293
294    Type * const carryTy = iBuilder->getBitBlockType();
295
296    if (LLVM_UNLIKELY(mCarryInfo->nonCarryCollapsingMode())) {
297
298        assert (mCarryInfo->hasSummary());
299
300        ConstantInt * const summaryIndex = iBuilder->getInt32(mCarryInfo->hasExplicitSummary() ? mCurrentFrameIndex : (mCurrentFrameIndex - 1));
301
302        Value * const carryInAccumulator = readCarryInSummary(iBuilder, summaryIndex);
303        Value * const carryOutAccumulator = mCarrySummaryStack.back();
304
305        if (mCarryInfo->hasExplicitSummary()) {
306            writeCarryOutSummary(iBuilder, carryOutAccumulator, summaryIndex);
307        }
308
309        std::tie(mCurrentFrame, mCurrentFrameIndex) = mCarryFrameStack.back();
310        mCarryFrameStack.pop_back();
311
312        // In non-carry-collapsing mode, we cannot rely on the fact that performing a single iteration of this
313        // loop will consume all of the incoming carries from the prior block. We need to subtract the carries
314        // consumed by this iteration from our carry summary state. To do so in parallel, we use the the half-
315        // subtractor circuit to do it in ceil log2 steps. Similarly, we compute our carry out summary state
316        // (for the subsequent block to subtract) using a half-adder circuit.
317
318        // NOTE: this requires that, for all loop iterations, i, and all block iterations, j, the carry in
319        // summary, CI_i,j, matches the carry out summary of the prior block iteration, CO_i,j - 1.
320        // Otherwise we may end up with an incorrect result or being trapped in an infinite loop.
321
322        Value * capacityPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
323        Value * capacity = iBuilder->CreateLoad(capacityPtr, false);
324        Value * summaryPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(2)});
325        Value * summary = iBuilder->CreateLoad(summaryPtr, false);
326
327        Constant * const ONE = ConstantInt::get(capacity->getType(), 1);
328
329        Value * loopSelector = iBuilder->CreateZExt(mLoopSelector, capacity->getType());
330
331        BasicBlock * entry = iBuilder->GetInsertBlock();
332        BasicBlock * update = iBuilder->CreateBasicBlock("UpdateNonCarryCollapsingSummary");
333        BasicBlock * resume = iBuilder->CreateBasicBlock("ResumeAfterUpdatingNonCarryCollapsingSummary");
334
335        iBuilder->CreateBr(update);
336
337        iBuilder->SetInsertPoint(update);
338        PHINode * i = iBuilder->CreatePHI(capacity->getType(), 2);
339        i->addIncoming(ConstantInt::getNullValue(capacity->getType()), entry);
340        PHINode * const borrow = iBuilder->CreatePHI(carryInAccumulator->getType(), 2);
341        borrow->addIncoming(carryInAccumulator, entry);
342        PHINode * const carry = iBuilder->CreatePHI(carryOutAccumulator->getType(), 2);
343        carry->addIncoming(carryOutAccumulator, entry);
344        // OR the updated carry in summary later for the summaryTest
345        PHINode * const carryInSummary = iBuilder->CreatePHI(carryTy, 2);
346        carryInSummary->addIncoming(Constant::getNullValue(carryTy), entry);
347
348        // half subtractor
349        Value * const carryInOffset = iBuilder->CreateOr(iBuilder->CreateShl(i, 1), loopSelector);
350        Value * const carryInPtr = iBuilder->CreateGEP(summary, carryInOffset);
351        Value * const carryIn = iBuilder->CreateBlockAlignedLoad(carryInPtr);
352        Value * const nextCarryIn = iBuilder->CreateXor(carryIn, borrow);
353        Value * const nextSummary = iBuilder->CreateOr(carryInSummary, nextCarryIn);
354        iBuilder->CreateBlockAlignedStore(nextCarryIn, carryInPtr);
355        carryInSummary->addIncoming(nextSummary, update);
356        Value * finalBorrow = iBuilder->CreateAnd(iBuilder->CreateNot(carryIn), borrow);
357        borrow->addIncoming(finalBorrow, update);
358
359        // half adder
360        Value * const carryOutOffset = iBuilder->CreateXor(carryInOffset, ConstantInt::get(carryInOffset->getType(), 1));
361        Value * const carryOutPtr = iBuilder->CreateGEP(summary, carryOutOffset);
362        Value * const carryOut = iBuilder->CreateBlockAlignedLoad(carryOutPtr);
363        Value * const nextCarryOut = iBuilder->CreateXor(carryOut, carry);
364        iBuilder->CreateBlockAlignedStore(nextCarryOut, carryOutPtr);
365        Value * finalCarry = iBuilder->CreateAnd(carryOut, carry);
366        carry->addIncoming(finalCarry, update);
367
368        // loop condition
369        i->addIncoming(iBuilder->CreateAdd(i, ONE), update);
370        iBuilder->CreateCondBr(iBuilder->CreateICmpNE(iBuilder->CreateShl(ONE, i), capacity), update, resume);
371
372        iBuilder->SetInsertPoint(resume);
373
374        IntegerType * ty = IntegerType::get(iBuilder->getContext(), carryTy->getPrimitiveSizeInBits());
375        iBuilder->CreateAssert(iBuilder->CreateICmpEQ(iBuilder->CreateBitCast(finalBorrow, ty), ConstantInt::getNullValue(ty)), "finalBorrow != 0");
376        iBuilder->CreateAssert(iBuilder->CreateICmpEQ(iBuilder->CreateBitCast(finalCarry, ty), ConstantInt::getNullValue(ty)), "finalCarry != 0");
377
378        assert (!mLoopIndicies.empty());
379        PHINode * index = mLoopIndicies.back();
380        index->addIncoming(iBuilder->CreateAdd(index, iBuilder->getSize(1)), resume);
381        mLoopIndicies.pop_back();
382
383        mNextSummaryTest = nextSummary;
384    }
385    if (mCarryInfo->hasSummary()) {
386        const auto n = mCarrySummaryStack.size(); assert (n > 1);
387        Value * carryOut = mCarrySummaryStack.back();
388        mCarrySummaryStack.pop_back();
389        PHINode * phiCarryOut = cast<PHINode>(mCarrySummaryStack.back());
390        phiCarryOut->addIncoming(carryOut, iBuilder->GetInsertBlock());
391        // If we're returning to the base scope, reset our accumulated summary value.
392        if (n == 2) {
393            carryOut = Constant::getNullValue(carryTy);
394        }
395        mCarrySummaryStack.back() = carryOut;
396    }
397}
398
399/** ------------------------------------------------------------------------------------------------------------- *
400 * @brief leaveLoopScope
401 ** ------------------------------------------------------------------------------------------------------------- */
402void CarryManager::leaveLoopScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * const /* entryBlock */, BasicBlock * const /* exitBlock */) {
403    assert (mLoopDepth > 0);
404    --mLoopDepth;
405    leaveScope(iBuilder);
406}
407
408/** ------------------------------------------------------------------------------------------------------------- *
409 * @brief enterIfScope
410 ** ------------------------------------------------------------------------------------------------------------- */
411void CarryManager::enterIfScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope) {
412    ++mIfDepth;
413    enterScope(iBuilder, scope);
414    // We zero-initialized the nested summary value and later OR in the current summary into the escaping summary
415    // so that upon processing the subsequent block iteration, we branch into this If scope iff a carry out was
416    // generated by a statement within this If scope and not by a dominating statement in the outer scope.
417    if (LLVM_LIKELY(mCarryInfo->hasSummary())) {
418        assert (mCurrentFrameIndex == 0);
419        mNextSummaryTest = readCarryInSummary(iBuilder, iBuilder->getInt32(0));
420        if (mCarryInfo->hasExplicitSummary()) {
421            mCurrentFrameIndex = 1;
422        }
423    }
424    Type * const carryTy = iBuilder->getBitBlockType();
425    mCarrySummaryStack.push_back(Constant::getNullValue(carryTy));
426}
427
428/** ------------------------------------------------------------------------------------------------------------- *
429 * @brief generateSummaryTest
430 ** ------------------------------------------------------------------------------------------------------------- */
431Value * CarryManager::generateSummaryTest(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * condition) {
432    if (LLVM_LIKELY(mCarryInfo->hasSummary())) {
433        assert ("summary test was not generated" && mNextSummaryTest);
434        condition = iBuilder->simd_or(condition, mNextSummaryTest);
435        mNextSummaryTest = nullptr;
436    }
437    assert ("summary test was not consumed" && (mNextSummaryTest == nullptr));
438    return condition;
439}
440
441/** ------------------------------------------------------------------------------------------------------------- *
442 * @brief enterIfBody
443 ** ------------------------------------------------------------------------------------------------------------- */
444void CarryManager::enterIfBody(const std::unique_ptr<kernel::KernelBuilder> &  /* iBuilder */, BasicBlock * const entryBlock) {
445    assert (entryBlock);
446}
447
448/** ------------------------------------------------------------------------------------------------------------- *
449 * @brief leaveIfBody
450 ** ------------------------------------------------------------------------------------------------------------- */
451void CarryManager::leaveIfBody(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * const exitBlock) {
452    assert (exitBlock);
453    const auto n = mCarrySummaryStack.size();
454    if (LLVM_LIKELY(mCarryInfo->hasExplicitSummary())) {
455        writeCarryOutSummary(iBuilder, mCarrySummaryStack[n - 1], iBuilder->getInt32(0));
456    }
457    if (n > 2) {
458        mCarrySummaryStack[n - 1] = iBuilder->CreateOr(mCarrySummaryStack[n - 1], mCarrySummaryStack[n - 2], "summary");
459    }
460}
461
462/** ------------------------------------------------------------------------------------------------------------- *
463 * @brief leaveIfScope
464 ** ------------------------------------------------------------------------------------------------------------- */
465void CarryManager::leaveIfScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, BasicBlock * const entryBlock, BasicBlock * const exitBlock) {
466    assert (mIfDepth > 0);
467    if (LLVM_LIKELY(mCarryInfo->hasSummary())) {
468        const auto n = mCarrySummaryStack.size(); assert (n > 0);
469        if (n > 2) {
470            // When leaving a nested If scope with a summary value, phi out the summary to ensure the
471            // appropriate summary is stored in the outer scope.
472            Value * nested = mCarrySummaryStack[n - 1];
473            Value * outer = mCarrySummaryStack[n - 2];
474            assert (nested->getType() == outer->getType());
475            PHINode * const phi = iBuilder->CreatePHI(nested->getType(), 2, "summary");
476            phi->addIncoming(outer, entryBlock);
477            phi->addIncoming(nested, exitBlock);
478            mCarrySummaryStack[n - 2] = phi;
479        }
480    }
481    --mIfDepth;
482    leaveScope(iBuilder);
483    mCarrySummaryStack.pop_back();
484}
485
486/** ------------------------------------------------------------------------------------------------------------ *
487 * @brief enterScope
488 ** ------------------------------------------------------------------------------------------------------------- */
489void CarryManager::enterScope(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope) {
490    assert (scope);
491    // Store the state of the current frame and update the scope state
492    mCarryFrameStack.emplace_back(mCurrentFrame, mCurrentFrameIndex + 1);
493    mCurrentScope = scope;
494    mCarryScopeIndex.push_back(++mCarryScopes);
495    mCarryInfo = &mCarryMetadata[mCarryScopes];
496    // Check whether we're still within our struct bounds; if this fails, either the Pablo program changed during
497    // compilation or a memory corruption has occured.
498    assert (mCurrentFrameIndex < mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
499    mCurrentFrame = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex)});
500    // Verify we're pointing to a carry frame struct
501    assert(mCurrentFrame->getType()->getPointerElementType()->isStructTy());
502    mCurrentFrameIndex = 0;
503}
504
505/** ------------------------------------------------------------------------------------------------------------- *
506 * @brief leaveScope
507 ** ------------------------------------------------------------------------------------------------------------- */
508void CarryManager::leaveScope(const std::unique_ptr<kernel::KernelBuilder> &  /* iBuilder */) {
509
510    // Did we use all of the packs in this carry struct?
511    assert (mCurrentFrameIndex == mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
512    // Sanity test: are there remaining carry frames?
513    assert (!mCarryFrameStack.empty());
514
515    std::tie(mCurrentFrame, mCurrentFrameIndex) = mCarryFrameStack.back();
516
517    assert(mCurrentFrame->getType()->getPointerElementType()->isStructTy());
518
519    mCarryFrameStack.pop_back();
520    mCarryScopeIndex.pop_back();
521    assert (!mCarryScopeIndex.empty());
522    mCurrentScope = mCurrentScope->getPredecessor();
523    assert (mCurrentScope);
524    mCarryInfo = &mCarryMetadata[mCarryScopeIndex.back()];
525}
526
527/** ------------------------------------------------------------------------------------------------------------- *
528 * @brief addCarryInCarryOut
529 ** ------------------------------------------------------------------------------------------------------------- */
530Value * CarryManager::addCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const Statement * const operation, Value * const e1, Value * const e2) {
531    assert (operation && (isNonAdvanceCarryGeneratingStatement(operation)));
532    Value * const carryIn = getNextCarryIn(iBuilder);
533    Value * carryOut, * result;
534    std::tie(carryOut, result) = iBuilder->bitblock_add_with_carry(e1, e2, carryIn);
535    setNextCarryOut(iBuilder, carryOut);
536    assert (result->getType() == iBuilder->getBitBlockType());
537    return result;
538}
539
540/** ------------------------------------------------------------------------------------------------------------- *
541 * @brief advanceCarryInCarryOut
542 ** ------------------------------------------------------------------------------------------------------------- */
543Value * CarryManager::advanceCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const Advance * const advance, Value * const value) {
544    const auto shiftAmount = advance->getAmount();
545    if (LLVM_LIKELY(shiftAmount < LONG_ADVANCE_BREAKPOINT)) {
546        Value * const carryIn = getNextCarryIn(iBuilder);
547        Value * carryOut, * result;
548        std::tie(carryOut, result) = iBuilder->bitblock_advance(value, carryIn, shiftAmount);
549        setNextCarryOut(iBuilder, carryOut);
550        assert (result->getType() == iBuilder->getBitBlockType());
551        return result;
552    } else {
553        return longAdvanceCarryInCarryOut(iBuilder, value, shiftAmount);
554    }
555}
556
557/** ------------------------------------------------------------------------------------------------------------- *
558 * @brief longAdvanceCarryInCarryOut
559 ** ------------------------------------------------------------------------------------------------------------- */
560inline Value * CarryManager::longAdvanceCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * const value, const unsigned shiftAmount) {
561
562    assert (mHasLongAdvance);
563    assert (shiftAmount >= LONG_ADVANCE_BREAKPOINT);
564
565    Type * const streamTy = iBuilder->getIntNTy(iBuilder->getBitBlockWidth());
566
567    if (mIfDepth > 0) {
568        if (shiftAmount > iBuilder->getBitBlockWidth()) {
569            const auto frameIndex = mCurrentFrameIndex++;
570            Value * carry = iBuilder->CreateZExt(iBuilder->bitblock_any(value), streamTy);
571            const unsigned summarySize = ceil_udiv(shiftAmount, iBuilder->getBitBlockWidth() * iBuilder->getBitBlockWidth());
572            for (unsigned i = 0;;++i) {
573                Value * const ptr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), iBuilder->getInt32(i)});
574                Value * const prior = iBuilder->CreateBitCast(iBuilder->CreateBlockAlignedLoad(ptr), streamTy);
575                Value * const stream = iBuilder->CreateBitCast(iBuilder->CreateOr(iBuilder->CreateShl(prior, 1), carry), iBuilder->getBitBlockType());
576                if (LLVM_LIKELY(i == summarySize)) {
577                    Value * const maskedStream = iBuilder->CreateAnd(stream, iBuilder->bitblock_mask_from(iBuilder->getInt32(summarySize % iBuilder->getBitBlockWidth())));
578                    addToCarryOutSummary(iBuilder, maskedStream);
579                    iBuilder->CreateBlockAlignedStore(maskedStream, ptr);
580                    break;
581                }
582                addToCarryOutSummary(iBuilder, stream);
583                iBuilder->CreateBlockAlignedStore(stream, ptr);
584                carry = iBuilder->CreateLShr(prior, iBuilder->getBitBlockWidth() - 1);
585            }
586        } else if (LLVM_LIKELY(mCarryInfo->hasExplicitSummary())) {
587            addToCarryOutSummary(iBuilder, value);
588        }
589    }
590    const auto frameIndex = mCurrentFrameIndex++;
591    // special case using a single buffer entry and the carry_out value.
592    if (LLVM_LIKELY((shiftAmount < iBuilder->getBitBlockWidth()) && (mLoopDepth == 0))) {
593        Value * const buffer = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), iBuilder->getInt32(0)});
594        assert (buffer->getType()->getPointerElementType() == iBuilder->getBitBlockType());
595        Value * carryIn = iBuilder->CreateBlockAlignedLoad(buffer);
596        iBuilder->CreateBlockAlignedStore(value, buffer);
597        /* Very special case - no combine */
598        if (LLVM_UNLIKELY(shiftAmount == iBuilder->getBitBlockWidth())) {
599            return iBuilder->CreateBitCast(carryIn, iBuilder->getBitBlockType());
600        }
601        Value* block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carryIn, streamTy), iBuilder->getBitBlockWidth() - shiftAmount);
602        Value* block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(value, streamTy), shiftAmount);
603        return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), iBuilder->getBitBlockType());
604    } else { //
605        const unsigned blockShift = shiftAmount % iBuilder->getBitBlockWidth();
606        const unsigned blocks = ceil_udiv(shiftAmount, iBuilder->getBitBlockWidth());
607        // Create a mask to implement circular buffer indexing
608        Value * indexMask = iBuilder->getSize(nearest_pow2(blocks + ((mLoopDepth != 0) ? 1 : 0)) - 1);
609        Value * blockIndex = iBuilder->getScalarField("CarryBlockIndex");
610        Value * carryIndex0 = iBuilder->CreateSub(blockIndex, iBuilder->getSize(blocks));
611        Value * loadIndex0 = iBuilder->CreateAnd(carryIndex0, indexMask);
612        Value * const carryInPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), loadIndex0});
613        Value * carryIn = iBuilder->CreateBlockAlignedLoad(carryInPtr);
614
615        Value * storeIndex = iBuilder->CreateAnd(blockIndex, indexMask);
616        Value * const carryOutPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), storeIndex});
617        assert (carryIn->getType() == iBuilder->getBitBlockType());
618
619        // If the long advance is an exact multiple of BitBlockWidth, we simply return the oldest
620        // block in the long advance carry data area.
621        if (LLVM_UNLIKELY(blockShift == 0)) {
622            iBuilder->CreateBlockAlignedStore(value, carryOutPtr);
623            return carryIn;
624        } else { // Otherwise we need to combine data from the two oldest blocks.
625            Value * carryIndex1 = iBuilder->CreateSub(blockIndex, iBuilder->getSize(blocks - 1));
626            Value * loadIndex1 = iBuilder->CreateAnd(carryIndex1, indexMask);
627            Value * const carryInPtr2 = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), loadIndex1});
628            Value * carry_block1 = iBuilder->CreateBlockAlignedLoad(carryInPtr2);
629            Value * block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carryIn, streamTy), iBuilder->getBitBlockWidth() - blockShift);
630            Value * block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(carry_block1, streamTy), blockShift);
631            iBuilder->CreateBlockAlignedStore(value, carryOutPtr);
632            return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), iBuilder->getBitBlockType());
633        }
634    }
635}
636
637/** ------------------------------------------------------------------------------------------------------------- *
638 * @brief getNextCarryIn
639 ** ------------------------------------------------------------------------------------------------------------- */
640Value * CarryManager::getNextCarryIn(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder) {
641    assert (mCurrentFrameIndex < mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
642    if (mLoopDepth == 0) {
643        mCarryPackPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex)});
644    } else {
645        mCarryPackPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex), mLoopSelector});
646    }
647    Type * const carryTy = iBuilder->getBitBlockType();
648    assert (mCarryPackPtr->getType()->getPointerElementType() == carryTy);
649    Value * const carryIn = iBuilder->CreateBlockAlignedLoad(mCarryPackPtr);
650    if (mLoopDepth > 0) {
651        iBuilder->CreateBlockAlignedStore(Constant::getNullValue(carryTy), mCarryPackPtr);
652    }
653    return carryIn;
654}
655
656/** ------------------------------------------------------------------------------------------------------------- *
657 * @brief setNextCarryOut
658 ** ------------------------------------------------------------------------------------------------------------- */
659void CarryManager::setNextCarryOut(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * carryOut) {
660    Type * const carryTy = iBuilder->getBitBlockType();
661    assert (mCurrentFrameIndex < mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
662    carryOut = iBuilder->CreateBitCast(carryOut, carryTy);
663    if (mCarryInfo->hasSummary()) {
664        addToCarryOutSummary(iBuilder, carryOut);
665    }
666    if (mLoopDepth != 0) {
667        mCarryPackPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex), mNextLoopSelector});
668        if (LLVM_LIKELY(!mCarryInfo->nonCarryCollapsingMode())) {
669            Value * accum = iBuilder->CreateBlockAlignedLoad(mCarryPackPtr);
670            carryOut = iBuilder->CreateOr(carryOut, accum);
671        }
672    }
673    ++mCurrentFrameIndex;
674    assert (mCarryPackPtr->getType()->getPointerElementType() == carryTy);
675    iBuilder->CreateBlockAlignedStore(carryOut, mCarryPackPtr);
676}
677
678/** ------------------------------------------------------------------------------------------------------------- *
679 * @brief readCarryInSummary
680 ** ------------------------------------------------------------------------------------------------------------- */
681Value * CarryManager::readCarryInSummary(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, ConstantInt * index) const {
682    assert (mCarryInfo->hasSummary());
683    unsigned count = 2;
684    if (LLVM_UNLIKELY(mCarryInfo->hasBorrowedSummary())) {
685        Type * frameTy = mCurrentFrame->getType()->getPointerElementType();
686        count = 1;
687        while (frameTy->isStructTy()) {
688            ++count;
689            frameTy = frameTy->getStructElementType(0);
690        }
691    }
692    const unsigned length = (mLoopDepth == 0) ? count : (count + 1);
693    Value * indicies[length];
694    std::fill(indicies, indicies + count - 1, iBuilder->getInt32(0));
695    indicies[count - 1] = index;
696    if (mLoopDepth != 0) {
697        indicies[count] = mLoopSelector;
698    }
699
700    ArrayRef<Value *> ar(indicies, length);
701    Value * const ptr = iBuilder->CreateGEP(mCurrentFrame, ar);
702    Value * const summary = iBuilder->CreateBlockAlignedLoad(ptr);
703    if (mLoopDepth != 0 && mCarryInfo->hasExplicitSummary()) {
704        Type * const carryTy = iBuilder->getBitBlockType();
705        iBuilder->CreateBlockAlignedStore(Constant::getNullValue(carryTy), ptr);
706    }
707    return summary;
708}
709
710/** ------------------------------------------------------------------------------------------------------------- *
711 * @brief writeCarryOutSummary
712 ** ------------------------------------------------------------------------------------------------------------- */
713inline void CarryManager::writeCarryOutSummary(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * const summary, ConstantInt * index) const {
714    Value * ptr = nullptr;
715    assert (mCarryInfo->hasExplicitSummary());
716    if (mLoopDepth > 0) {
717        ptr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), index, mNextLoopSelector});
718    } else {
719        ptr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), index});
720    }
721    iBuilder->CreateBlockAlignedStore(summary, ptr);
722}
723
724/** ------------------------------------------------------------------------------------------------------------- *
725 * @brief addToCarryOutSummary
726 ** ------------------------------------------------------------------------------------------------------------- */
727inline void CarryManager::addToCarryOutSummary(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, Value * const value) {
728    assert ("cannot add null summary value!" && value);   
729    assert ("summary stack is empty!" && !mCarrySummaryStack.empty());
730    assert (mCarryInfo->hasSummary());
731    mCarrySummaryStack.back() = iBuilder->CreateOr(value, mCarrySummaryStack.back());
732}
733
734/** ------------------------------------------------------------------------------------------------------------- *
735 * @brief enumerate
736 ** ------------------------------------------------------------------------------------------------------------- */
737unsigned CarryManager::getScopeCount(const PabloBlock * const scope, unsigned index) {
738    for (const Statement * stmt : *scope) {
739        if (LLVM_UNLIKELY(isa<Branch>(stmt))) {
740            index = getScopeCount(cast<Branch>(stmt)->getBody(), index);
741        }
742    }
743    return index + 1;
744}
745
746/** ------------------------------------------------------------------------------------------------------------- *
747 * @brief hasIterationSpecificAssignment
748 ** ------------------------------------------------------------------------------------------------------------- */
749bool CarryManager::hasIterationSpecificAssignment(const PabloBlock * const scope) {
750    if (const While * const br = dyn_cast_or_null<While>(scope->getBranch())) {
751        for (const Var * var : br->getEscaped()) {
752            for (const PabloAST * user : var->users()) {
753                if (const Extract * e = dyn_cast<Extract>(user)) {
754                    if (LLVM_UNLIKELY(e->getIndex() == var)) {
755                        // If we assign this Var a value and read the value as the index parameter
756                        // of a nested Extract statement, then we cannot collapse the carries.
757                        const PabloBlock * parent = e->getParent();
758                        for (;;) {
759                            if (parent == scope) {
760                                return true;
761                            }
762                            parent = parent->getPredecessor();
763                            if (parent == nullptr) {
764                                break;
765                            }
766                        }
767                    }
768                }
769            }
770        }
771    }
772    return false;
773}
774
775/** ------------------------------------------------------------------------------------------------------------- *
776 * @brief analyse
777 ** ------------------------------------------------------------------------------------------------------------- */
778StructType * CarryManager::analyse(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder, const PabloBlock * const scope, const unsigned ifDepth, const unsigned loopDepth, const bool isNestedWithinNonCarryCollapsingLoop) {
779    assert ("scope cannot be null!" && scope);
780    assert (mCarryScopes == 0 ? (scope == mKernel->getEntryBlock()) : (scope != mKernel->getEntryBlock()));
781    assert (mCarryScopes < mCarryMetadata.size());
782    Type * const carryTy = iBuilder->getBitBlockType();
783    Type * const blockTy = iBuilder->getBitBlockType();
784
785    const unsigned carryScopeIndex = mCarryScopes++;
786    const bool nonCarryCollapsingMode = hasIterationSpecificAssignment(scope);
787    Type * const carryPackType = (loopDepth == 0) ? carryTy : ArrayType::get(carryTy, 2);
788    std::vector<Type *> state;
789
790    for (const Statement * stmt : *scope) {
791        if (LLVM_UNLIKELY(isa<Advance>(stmt))) {
792            const auto amount = cast<Advance>(stmt)->getAmount();
793            Type * type = carryPackType;
794            if (LLVM_UNLIKELY(amount >= LONG_ADVANCE_BREAKPOINT)) {
795                const unsigned blocks = ceil_udiv(amount, iBuilder->getBitBlockWidth());
796                type = ArrayType::get(blockTy, nearest_pow2(blocks + ((loopDepth != 0) ? 1 : 0)));
797                if (LLVM_UNLIKELY(ifDepth > 0 && amount > iBuilder->getBitBlockWidth())) {
798                    // 1 bit will mark the presense of any bit in each block.
799                    Type * carryType = ArrayType::get(blockTy, ceil_udiv(amount, std::pow(iBuilder->getBitBlockWidth(), 2)));
800                    state.push_back(carryType);
801                }
802                mHasLongAdvance = true;               
803            }
804            state.push_back(type);
805        } else if (LLVM_UNLIKELY(isNonAdvanceCarryGeneratingStatement(stmt))) {
806            state.push_back(carryPackType);
807        } else if (LLVM_UNLIKELY(isa<If>(stmt))) {
808            state.push_back(analyse(iBuilder, cast<If>(stmt)->getBody(), ifDepth + 1, loopDepth, nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop));
809        } else if (LLVM_UNLIKELY(isa<While>(stmt))) {
810            mHasLoop = true;
811            state.push_back(analyse(iBuilder, cast<While>(stmt)->getBody(), ifDepth, loopDepth + 1, nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop));
812        }
813    }
814    // Build the carry state struct and add the summary pack if needed.
815    CarryData & cd = mCarryMetadata[carryScopeIndex];
816    StructType * carryState = nullptr;
817    CarryData::SummaryType summaryType = CarryData::NoSummary;
818    if (LLVM_UNLIKELY(state.empty())) {
819        carryState = StructType::get(iBuilder->getContext());
820    } else {
821        //if (ifDepth > 0 || (nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop)) {
822        if (dyn_cast_or_null<If>(scope->getBranch()) || nonCarryCollapsingMode || isNestedWithinNonCarryCollapsingLoop) {
823            if (LLVM_LIKELY(state.size() > 1)) {
824                summaryType = CarryData::ExplicitSummary;
825                // NOTE: summaries are stored differently depending whether we're entering an If or While branch. With an If branch, they
826                // preceed the carry state data and with a While loop they succeed it. This is to help cache prefectching performance.
827                state.insert(isa<If>(scope->getBranch()) ? state.begin() : state.end(), carryPackType);
828            } else {
829                summaryType = CarryData::ImplicitSummary;
830                if (state[0]->isStructTy()) {
831                    summaryType = CarryData::BorrowedSummary;
832                }
833            }           
834        }
835        carryState = StructType::get(iBuilder->getContext(), state);
836        // If we're in a loop and cannot use collapsing carry mode, convert the carry state struct into a capacity,
837        // carry state pointer, and summary pointer struct.
838        if (LLVM_UNLIKELY(nonCarryCollapsingMode)) {
839            carryState = StructType::get(iBuilder->getSizeTy(), carryState->getPointerTo(), carryTy->getPointerTo(), nullptr);
840        }
841        cd.setNonCollapsingCarryMode(nonCarryCollapsingMode);
842    }
843    cd.setSummaryType(summaryType);
844    return carryState;
845}
846
847/** ------------------------------------------------------------------------------------------------------------- *
848 * @brief constructor
849 ** ------------------------------------------------------------------------------------------------------------- */
850CarryManager::CarryManager() noexcept
851: mKernel(nullptr)
852, mCurrentFrame(nullptr)
853, mCurrentFrameIndex(0)
854, mCurrentScope(nullptr)
855, mCarryInfo(nullptr)
856, mNextSummaryTest(nullptr)
857, mIfDepth(0)
858, mHasLongAdvance(false)
859, mHasLoop(false)
860, mLoopDepth(0)
861, mLoopSelector(nullptr)
862, mNextLoopSelector(nullptr)
863, mCarryPackPtr(nullptr)
864, mCarryScopes(0) {
865
866}
867
868}
Note: See TracBrowser for help on using the repository browser.