source: icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp @ 5486

Last change on this file since 5486 was 5486, checked in by nmedfort, 2 years ago

Initial attempt to improve debugging capabilities with compilation stack traces on error.

File size: 45.5 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include "carry_manager.h"
8#include <pablo/carry_data.h>
9#include <pablo/codegenstate.h>
10#include <llvm/IR/BasicBlock.h>
11#include <llvm/IR/DerivedTypes.h>
12#include <pablo/branch.h>
13#include <pablo/pe_advance.h>
14#include <pablo/pe_scanthru.h>
15#include <pablo/pe_matchstar.h>
16#include <pablo/pe_var.h>
17#include <kernels/kernel_builder.h>
18#include <llvm/Support/raw_ostream.h>
19
20using namespace llvm;
21
22namespace pablo {
23
24inline static unsigned ceil_log2(const unsigned v) {
25    assert ("log2(0) is undefined!" && v != 0);
26    return 32 - __builtin_clz(v - 1);
27}
28
29inline static unsigned floor_log2(const unsigned v) {
30    assert ("log2(0) is undefined!" && v != 0);
31    return 31 - __builtin_clz(v);
32}
33
34inline static unsigned nearest_pow2(const unsigned v) {
35    assert(v > 0 && v < (UINT32_MAX / 2));
36    return (v < 2) ? 1 : (1 << ceil_log2(v));
37}
38
39inline static unsigned ceil_udiv(const unsigned x, const unsigned y) {
40    return (((x - 1) | (y - 1)) + 1) / y;
41}
42
43using TypeId = PabloAST::ClassTypeId;
44
45inline static bool isNonAdvanceCarryGeneratingStatement(const Statement * const stmt) {
46    return isa<CarryProducingStatement>(stmt) && !isa<Advance>(stmt);
47}
48
49#define LONG_ADVANCE_BREAKPOINT 64
50
51/** ------------------------------------------------------------------------------------------------------------- *
52 * @brief initializeCarryData
53 ** ------------------------------------------------------------------------------------------------------------- */
54void CarryManager::initializeCarryData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, PabloKernel * const kernel) {
55
56    // Each scope constructs its own CarryData struct, which will be added to the final "carries" struct
57    // that is added to the Kernel. The scope index will indicate which struct to access.
58
59    // A CarryData struct either contains an array of CarryPackBlocks or an integer indicating the capacity of
60    // the variable length CarryData struct and pointer. A variable length CarryData struct is required whenever
61    // the streams accessed by a loop could vary between iterations. When resizing a CarryData struct for a
62    // particular loop, the current loop struct and all nested structs need to be resized. This accommodates
63    // the fact every pablo While loop must be executed at least once.
64
65    // A nested loop may also contain a variable length CarryData struct
66
67    // To determine whether we require a variable length CarryData struct, we test the escaped variables of
68    // each loop branch to see whether they are used as the index parameter of a nested Extract statement.
69    // Any scope that requires variable length CarryData, requires that all nested branches have a unique
70    // set of carries for that iteration.
71
72    assert (mKernel == nullptr);
73    mCurrentScope = kernel->getEntryBlock();
74    mKernel = kernel;
75
76    mCarryScopes = 0;
77
78    mCarryMetadata.resize(getScopeCount(mCurrentScope));
79
80    Type * const carryStateTy = analyse(iBuilder, mCurrentScope);
81
82    kernel->addScalar(carryStateTy, "carries");
83
84    if (mHasLoop) {
85        kernel->addScalar(iBuilder->getInt32Ty(), "selector");
86    }
87    if (mHasLongAdvance) {
88        kernel->addScalar(iBuilder->getSizeTy(), "CarryBlockIndex");
89    }
90
91}
92
93/** ------------------------------------------------------------------------------------------------------------- *
94 * @brief releaseCarryData
95 ** ------------------------------------------------------------------------------------------------------------- */
96void CarryManager::releaseCarryData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, PabloKernel * const kernel) {
97
98
99}
100
101/** ------------------------------------------------------------------------------------------------------------- *
102 * @brief initializeCodeGen
103 ** ------------------------------------------------------------------------------------------------------------- */
104void CarryManager::initializeCodeGen(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
105
106    assert(!mCarryMetadata.empty());
107    mCarryInfo = &mCarryMetadata[0];
108    assert (!mCarryInfo->hasSummary());
109
110    mCurrentFrame = iBuilder->getScalarFieldPtr("carries");
111    mCurrentFrameIndex = 0;
112    mCarryScopes = 0;
113    mCarryScopeIndex.push_back(0);
114
115    assert (mCarryFrameStack.empty());
116
117    assert (mCarrySummaryStack.empty());
118
119    Type * const carryTy = iBuilder->getBitBlockType();
120
121    mCarrySummaryStack.push_back(Constant::getNullValue(carryTy));
122
123    if (mHasLoop) {       
124        mLoopSelector = iBuilder->getScalarField("selector");
125        mNextLoopSelector = iBuilder->CreateXor(mLoopSelector, ConstantInt::get(mLoopSelector->getType(), 1));
126    }
127
128}
129
130/** ------------------------------------------------------------------------------------------------------------- *
131 * @brief finalizeCodeGen
132 ** ------------------------------------------------------------------------------------------------------------- */
133void CarryManager::finalizeCodeGen(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
134    if (mHasLoop) {
135        iBuilder->setScalarField("selector", mNextLoopSelector);
136    }
137    if (mHasLongAdvance) {
138        Value * idx = iBuilder->getScalarField("CarryBlockIndex");
139        idx = iBuilder->CreateAdd(idx, iBuilder->getSize(1));
140        iBuilder->setScalarField("CarryBlockIndex", idx);
141    }
142    assert (mCarryFrameStack.empty());   
143    assert ("base summary value was deleted!" && mCarrySummaryStack.size() == 1);
144    assert ("base summary value was overwritten with non-zero value!" && isa<Constant>(mCarrySummaryStack[0]) && cast<Constant>(mCarrySummaryStack[0])->isNullValue());
145    mCarrySummaryStack.clear();
146    assert (mCarryScopeIndex.size() == 1);
147    mCarryScopeIndex.clear();
148}
149
150/** ------------------------------------------------------------------------------------------------------------- *
151 * @brief enterLoopScope
152 ** ------------------------------------------------------------------------------------------------------------- */
153void CarryManager::enterLoopScope(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const PabloBlock * const scope) {
154    assert (scope);
155    assert (mHasLoop);
156    ++mLoopDepth;
157    enterScope(iBuilder, scope);
158}
159
160/** ------------------------------------------------------------------------------------------------------------- *
161 * @brief enterLoopBody
162 ** ------------------------------------------------------------------------------------------------------------- */
163void CarryManager::enterLoopBody(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, BasicBlock * const entryBlock) {
164    if (mCarryInfo->hasSummary()) {
165        Type * const carryTy = iBuilder->getBitBlockType();
166        PHINode * phiCarryOutSummary = iBuilder->CreatePHI(carryTy, 2, "summary");
167        assert (!mCarrySummaryStack.empty());
168        phiCarryOutSummary->addIncoming(mCarrySummaryStack.back(), entryBlock);
169        // Replace the incoming carry summary with the phi node and add the phi node to the stack  so that we can
170        // properly OR it into the outgoing summary value.
171        // NOTE: this may change the base summary value; when exiting to the base scope, replace this summary with
172        // a null value to prevent subsequent nested scopes from inheriting the summary of this scope.
173        mCarrySummaryStack.back() = phiCarryOutSummary;
174        mCarrySummaryStack.push_back(phiCarryOutSummary);
175    }
176    if (LLVM_UNLIKELY(mCarryInfo->nonCarryCollapsingMode())) {
177
178        assert (mCarryInfo->hasSummary());
179
180        Type * const int8PtrTy = iBuilder->getInt8PtrTy();
181        Type * const carryTy = iBuilder->getBitBlockType();
182        PointerType * const carryPtrTy = carryTy->getPointerTo();
183
184        // Check whether we need to resize the carry state
185        PHINode * index = iBuilder->CreatePHI(iBuilder->getSizeTy(), 2);
186        mLoopIndicies.push_back(index);
187        index->addIncoming(iBuilder->getSize(0), entryBlock);
188
189        Value * capacityPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
190        Value * capacity = iBuilder->CreateLoad(capacityPtr, false, "capacity");
191        Constant * const ONE = ConstantInt::get(capacity->getType(), 1);
192        Value * arrayPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
193        Value * array = iBuilder->CreateLoad(arrayPtr, false, "array");
194
195        BasicBlock * const entry = iBuilder->GetInsertBlock();
196        BasicBlock * const resizeCarryState = iBuilder->CreateBasicBlock("ResizeCarryState");
197        BasicBlock * const reallocExisting = iBuilder->CreateBasicBlock("ReallocExisting");
198        BasicBlock * const createNew = iBuilder->CreateBasicBlock("CreateNew");
199        BasicBlock * const resumeKernel = iBuilder->CreateBasicBlock("ResumeKernel");
200
201        iBuilder->CreateLikelyCondBr(iBuilder->CreateICmpULT(index, capacity), resumeKernel, resizeCarryState);
202
203        // RESIZE CARRY BLOCK
204        iBuilder->SetInsertPoint(resizeCarryState);
205        const auto BlockWidth = carryTy->getPrimitiveSizeInBits() / 8;
206        const auto Log2BlockWidth = floor_log2(BlockWidth);
207        Constant * const carryStateWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(array->getType()->getPointerElementType()), iBuilder->getSizeTy(), false);
208        Value * summaryPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(2)});
209
210        Value * const hasCarryState = iBuilder->CreateICmpNE(array, ConstantPointerNull::get(cast<PointerType>(array->getType())));
211
212        iBuilder->CreateLikelyCondBr(hasCarryState, reallocExisting, createNew);
213
214        // REALLOCATE EXISTING
215        iBuilder->SetInsertPoint(reallocExisting);
216
217        Value * const capacitySize = iBuilder->CreateMul(capacity, carryStateWidth);
218        Value * const newCapacitySize = iBuilder->CreateShl(capacitySize, 1); // x 2
219
220        Value * newArray = iBuilder->CreateRealloc(array, newCapacitySize);
221
222        Value * const startNewArrayPtr = iBuilder->CreateGEP(iBuilder->CreatePointerCast(newArray, int8PtrTy), capacitySize);
223        iBuilder->CreateMemZero(startNewArrayPtr, capacitySize, BlockWidth);
224
225        iBuilder->CreateStore(newArray, arrayPtr);
226
227        Value * const log2capacity = iBuilder->CreateAdd(iBuilder->CreateCeilLog2(capacity), ONE);
228        Value * const summarySize = iBuilder->CreateShl(log2capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
229        Value * const newLog2Capacity = iBuilder->CreateAdd(log2capacity, ONE);
230        Value * const newSummarySize = iBuilder->CreateShl(newLog2Capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
231
232        Value * const summary = iBuilder->CreateLoad(summaryPtr, false);
233        Value * newSummary = iBuilder->CreateRealloc(summary, newSummarySize);
234        Value * const startNewSummaryPtr = iBuilder->CreateGEP(iBuilder->CreatePointerCast(newArray, int8PtrTy), summarySize);
235        iBuilder->CreateMemZero(startNewSummaryPtr, iBuilder->getSize(2 * BlockWidth), BlockWidth);
236
237        Value * ptr1 = iBuilder->CreateGEP(newSummary, summarySize);
238        ptr1 = iBuilder->CreatePointerCast(ptr1, carryPtrTy);
239
240        Value * ptr2 = iBuilder->CreateGEP(newSummary, iBuilder->CreateAdd(summarySize, iBuilder->getSize(BlockWidth)));
241        ptr2 = iBuilder->CreatePointerCast(ptr2, carryPtrTy);
242
243        newSummary = iBuilder->CreatePointerCast(newSummary, carryPtrTy);
244        iBuilder->CreateStore(newSummary, summaryPtr);
245        Value * const newCapacity = iBuilder->CreateShl(ONE, log2capacity);
246
247        iBuilder->CreateStore(newCapacity, capacityPtr);
248
249        iBuilder->CreateBr(resumeKernel);
250
251        // CREATE NEW
252        iBuilder->SetInsertPoint(createNew);
253
254        Constant * const initialLog2Capacity = iBuilder->getInt64(4);
255        Constant * const initialCapacity = ConstantExpr::getShl(ONE, initialLog2Capacity);
256        Constant * const initialCapacitySize = ConstantExpr::getMul(initialCapacity, carryStateWidth);
257
258        Value * initialArray = iBuilder->CreateAlignedMalloc(initialCapacitySize, iBuilder->getCacheAlignment());
259        iBuilder->CreateMemZero(initialArray, initialCapacitySize, BlockWidth);
260        initialArray = iBuilder->CreatePointerCast(initialArray, array->getType());
261        iBuilder->CreateStore(initialArray, arrayPtr);
262
263        Constant * initialSummarySize = ConstantExpr::getShl(ConstantExpr::getAdd(initialLog2Capacity, iBuilder->getInt64(1)), iBuilder->getInt64(Log2BlockWidth + 1));
264        Value * initialSummary = iBuilder->CreateAlignedMalloc(initialSummarySize, BlockWidth);
265        iBuilder->CreateMemZero(initialSummary, initialSummarySize, BlockWidth);
266        initialSummary = iBuilder->CreatePointerCast(initialSummary, carryPtrTy);
267        iBuilder->CreateStore(initialSummary, summaryPtr);
268
269        iBuilder->CreateStore(initialCapacity, capacityPtr);
270
271        iBuilder->CreateBr(resumeKernel);
272
273        // RESUME KERNEL
274        iBuilder->SetInsertPoint(resumeKernel);
275        // Load the appropriate carry stat block
276        PHINode * phiArrayPtr = iBuilder->CreatePHI(array->getType(), 3);
277        phiArrayPtr->addIncoming(array, entry);
278        phiArrayPtr->addIncoming(initialArray, createNew);
279        phiArrayPtr->addIncoming(newArray, reallocExisting);
280
281        // NOTE: the 3 here is only to pass the assertion later. It refers to the number of elements in the carry data struct.
282        mCarryFrameStack.emplace_back(mCurrentFrame, 3);
283        mCurrentFrame = iBuilder->CreateGEP(phiArrayPtr, index);
284    }
285}
286
287/** ------------------------------------------------------------------------------------------------------------- *
288 * @brief leaveLoopBody
289 ** ------------------------------------------------------------------------------------------------------------- */
290void CarryManager::leaveLoopBody(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, BasicBlock * /* exitBlock */) {
291
292    Type * const carryTy = iBuilder->getBitBlockType();
293
294    if (LLVM_UNLIKELY(mCarryInfo->nonCarryCollapsingMode())) {
295
296        assert (mCarryInfo->hasSummary());
297
298        ConstantInt * const summaryIndex = iBuilder->getInt32(mCarryInfo->hasExplicitSummary() ? mCurrentFrameIndex : (mCurrentFrameIndex - 1));
299
300        Value * const carryInAccumulator = readCarryInSummary(iBuilder, summaryIndex);
301        Value * const carryOutAccumulator = mCarrySummaryStack.back();
302
303        if (mCarryInfo->hasExplicitSummary()) {
304            writeCarryOutSummary(iBuilder, carryOutAccumulator, summaryIndex);
305        }
306
307        std::tie(mCurrentFrame, mCurrentFrameIndex) = mCarryFrameStack.back();
308        mCarryFrameStack.pop_back();
309
310        // In non-carry-collapsing mode, we cannot rely on the fact that performing a single iteration of this
311        // loop will consume all of the incoming carries from the prior block. We need to subtract the carries
312        // consumed by this iteration from our carry summary state. To do so in parallel, we use the the half-
313        // subtractor circuit to do it in ceil log2 steps. Similarly, we compute our carry out summary state
314        // (for the subsequent block to subtract) using a half-adder circuit.
315
316        // NOTE: this requires that, for all loop iterations, i, and all block iterations, j, the carry in
317        // summary, CI_i,j, matches the carry out summary of the prior block iteration, CO_i,j - 1.
318        // Otherwise we may end up with an incorrect result or being trapped in an infinite loop.
319
320        Value * capacityPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
321        Value * capacity = iBuilder->CreateLoad(capacityPtr, false);
322        Value * summaryPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(2)});
323        Value * summary = iBuilder->CreateLoad(summaryPtr, false);
324
325        Constant * const ONE = ConstantInt::get(capacity->getType(), 1);
326
327        Value * loopSelector = iBuilder->CreateZExt(mLoopSelector, capacity->getType());
328
329        BasicBlock * entry = iBuilder->GetInsertBlock();
330        BasicBlock * update = iBuilder->CreateBasicBlock("UpdateNonCarryCollapsingSummary");
331        BasicBlock * resume = iBuilder->CreateBasicBlock("ResumeAfterUpdatingNonCarryCollapsingSummary");
332
333        iBuilder->CreateBr(update);
334
335        iBuilder->SetInsertPoint(update);
336        PHINode * i = iBuilder->CreatePHI(capacity->getType(), 2);
337        i->addIncoming(ConstantInt::getNullValue(capacity->getType()), entry);
338        PHINode * const borrow = iBuilder->CreatePHI(carryInAccumulator->getType(), 2);
339        borrow->addIncoming(carryInAccumulator, entry);
340        PHINode * const carry = iBuilder->CreatePHI(carryOutAccumulator->getType(), 2);
341        carry->addIncoming(carryOutAccumulator, entry);
342        // OR the updated carry in summary later for the summaryTest
343        PHINode * const carryInSummary = iBuilder->CreatePHI(carryTy, 2);
344        carryInSummary->addIncoming(Constant::getNullValue(carryTy), entry);
345
346        // half subtractor
347        Value * const carryInOffset = iBuilder->CreateOr(iBuilder->CreateShl(i, 1), loopSelector);
348        Value * const carryInPtr = iBuilder->CreateGEP(summary, carryInOffset);
349        Value * const carryIn = iBuilder->CreateBlockAlignedLoad(carryInPtr);
350        Value * const nextCarryIn = iBuilder->CreateXor(carryIn, borrow);
351        Value * const nextSummary = iBuilder->CreateOr(carryInSummary, nextCarryIn);
352        iBuilder->CreateBlockAlignedStore(nextCarryIn, carryInPtr);
353        carryInSummary->addIncoming(nextSummary, update);
354        Value * finalBorrow = iBuilder->CreateAnd(iBuilder->CreateNot(carryIn), borrow);
355        borrow->addIncoming(finalBorrow, update);
356
357        // half adder
358        Value * const carryOutOffset = iBuilder->CreateXor(carryInOffset, ConstantInt::get(carryInOffset->getType(), 1));
359        Value * const carryOutPtr = iBuilder->CreateGEP(summary, carryOutOffset);
360        Value * const carryOut = iBuilder->CreateBlockAlignedLoad(carryOutPtr);
361        Value * const nextCarryOut = iBuilder->CreateXor(carryOut, carry);
362        iBuilder->CreateBlockAlignedStore(nextCarryOut, carryOutPtr);
363        Value * finalCarry = iBuilder->CreateAnd(carryOut, carry);
364        carry->addIncoming(finalCarry, update);
365
366        // loop condition
367        i->addIncoming(iBuilder->CreateAdd(i, ONE), update);
368        iBuilder->CreateCondBr(iBuilder->CreateICmpNE(iBuilder->CreateShl(ONE, i), capacity), update, resume);
369
370        iBuilder->SetInsertPoint(resume);
371
372        IntegerType * ty = IntegerType::get(iBuilder->getContext(), carryTy->getPrimitiveSizeInBits());
373        iBuilder->CreateAssert(iBuilder->CreateICmpEQ(iBuilder->CreateBitCast(finalBorrow, ty), ConstantInt::getNullValue(ty)), "finalBorrow != 0");
374        iBuilder->CreateAssert(iBuilder->CreateICmpEQ(iBuilder->CreateBitCast(finalCarry, ty), ConstantInt::getNullValue(ty)), "finalCarry != 0");
375
376        assert (!mLoopIndicies.empty());
377        PHINode * index = mLoopIndicies.back();
378        index->addIncoming(iBuilder->CreateAdd(index, iBuilder->getSize(1)), resume);
379        mLoopIndicies.pop_back();
380
381        mNextSummaryTest = nextSummary;
382    }
383    if (mCarryInfo->hasSummary()) {
384        const auto n = mCarrySummaryStack.size(); assert (n > 1);
385        Value * carryOut = mCarrySummaryStack.back();
386        mCarrySummaryStack.pop_back();
387        PHINode * phiCarryOut = cast<PHINode>(mCarrySummaryStack.back());
388        phiCarryOut->addIncoming(carryOut, iBuilder->GetInsertBlock());
389        // If we're returning to the base scope, reset our accumulated summary value.
390        if (n == 2) {
391            carryOut = Constant::getNullValue(carryTy);
392        }
393        mCarrySummaryStack.back() = carryOut;
394    }
395}
396
397/** ------------------------------------------------------------------------------------------------------------- *
398 * @brief leaveLoopScope
399 ** ------------------------------------------------------------------------------------------------------------- */
400void CarryManager::leaveLoopScope(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, BasicBlock * const /* entryBlock */, BasicBlock * const /* exitBlock */) {
401    assert (mLoopDepth > 0);
402    --mLoopDepth;
403    leaveScope(iBuilder);
404}
405
406/** ------------------------------------------------------------------------------------------------------------- *
407 * @brief enterIfScope
408 ** ------------------------------------------------------------------------------------------------------------- */
409void CarryManager::enterIfScope(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const PabloBlock * const scope) {
410    ++mIfDepth;
411    enterScope(iBuilder, scope);
412    // We zero-initialized the nested summary value and later OR in the current summary into the escaping summary
413    // so that upon processing the subsequent block iteration, we branch into this If scope iff a carry out was
414    // generated by a statement within this If scope and not by a dominating statement in the outer scope.
415    if (LLVM_LIKELY(mCarryInfo->hasSummary())) {
416        assert (mCurrentFrameIndex == 0);
417        mNextSummaryTest = readCarryInSummary(iBuilder, iBuilder->getInt32(0));
418        if (mCarryInfo->hasExplicitSummary()) {
419            mCurrentFrameIndex = 1;
420        }
421    }
422    Type * const carryTy = iBuilder->getBitBlockType();
423    mCarrySummaryStack.push_back(Constant::getNullValue(carryTy));
424}
425
426/** ------------------------------------------------------------------------------------------------------------- *
427 * @brief generateSummaryTest
428 ** ------------------------------------------------------------------------------------------------------------- */
429Value * CarryManager::generateSummaryTest(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, Value * condition) {
430    if (LLVM_LIKELY(mCarryInfo->hasSummary())) {
431        assert ("summary test was not generated" && mNextSummaryTest);
432        condition = iBuilder->simd_or(condition, mNextSummaryTest);
433        mNextSummaryTest = nullptr;
434    }
435    assert ("summary test was not consumed" && (mNextSummaryTest == nullptr));
436    return condition;
437}
438
439/** ------------------------------------------------------------------------------------------------------------- *
440 * @brief enterIfBody
441 ** ------------------------------------------------------------------------------------------------------------- */
442void CarryManager::enterIfBody(const std::unique_ptr<kernel::KernelBuilder> & /* iBuilder */, BasicBlock * const entryBlock) {
443    assert (entryBlock);
444}
445
446/** ------------------------------------------------------------------------------------------------------------- *
447 * @brief leaveIfBody
448 ** ------------------------------------------------------------------------------------------------------------- */
449void CarryManager::leaveIfBody(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, BasicBlock * const exitBlock) {
450    assert (exitBlock);
451    const auto n = mCarrySummaryStack.size();
452    if (LLVM_LIKELY(mCarryInfo->hasExplicitSummary())) {
453        writeCarryOutSummary(iBuilder, mCarrySummaryStack[n - 1], iBuilder->getInt32(0));
454    }
455    if (n > 2) {
456        mCarrySummaryStack[n - 1] = iBuilder->CreateOr(mCarrySummaryStack[n - 1], mCarrySummaryStack[n - 2], "summary");
457    }
458}
459
460/** ------------------------------------------------------------------------------------------------------------- *
461 * @brief leaveIfScope
462 ** ------------------------------------------------------------------------------------------------------------- */
463void CarryManager::leaveIfScope(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, BasicBlock * const entryBlock, BasicBlock * const exitBlock) {
464    assert (mIfDepth > 0);
465    if (LLVM_LIKELY(mCarryInfo->hasSummary())) {
466        const auto n = mCarrySummaryStack.size(); assert (n > 0);
467        if (n > 2) {
468            // When leaving a nested If scope with a summary value, phi out the summary to ensure the
469            // appropriate summary is stored in the outer scope.
470            Value * nested = mCarrySummaryStack[n - 1];
471            Value * outer = mCarrySummaryStack[n - 2];
472            assert (nested->getType() == outer->getType());
473            PHINode * const phi = iBuilder->CreatePHI(nested->getType(), 2, "summary");
474            phi->addIncoming(outer, entryBlock);
475            phi->addIncoming(nested, exitBlock);
476            mCarrySummaryStack[n - 2] = phi;
477        }
478    }
479    --mIfDepth;
480    leaveScope(iBuilder);
481    mCarrySummaryStack.pop_back();
482}
483
484/** ------------------------------------------------------------------------------------------------------------ *
485 * @brief enterScope
486 ** ------------------------------------------------------------------------------------------------------------- */
487void CarryManager::enterScope(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const PabloBlock * const scope) {
488    assert (scope);
489    // Store the state of the current frame and update the scope state
490    mCarryFrameStack.emplace_back(mCurrentFrame, mCurrentFrameIndex + 1);
491    mCurrentScope = scope;
492    mCarryScopeIndex.push_back(++mCarryScopes);
493    mCarryInfo = &mCarryMetadata[mCarryScopes];
494    // Check whether we're still within our struct bounds; if this fails, either the Pablo program changed during
495    // compilation or a memory corruption has occured.
496    assert (mCurrentFrameIndex < mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
497    mCurrentFrame = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex)});
498    // Verify we're pointing to a carry frame struct
499    assert(mCurrentFrame->getType()->getPointerElementType()->isStructTy());
500    mCurrentFrameIndex = 0;
501}
502
503/** ------------------------------------------------------------------------------------------------------------- *
504 * @brief leaveScope
505 ** ------------------------------------------------------------------------------------------------------------- */
506void CarryManager::leaveScope(const std::unique_ptr<kernel::KernelBuilder> & /* iBuilder */) {
507
508    // Did we use all of the packs in this carry struct?
509    assert (mCurrentFrameIndex == mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
510    // Sanity test: are there remaining carry frames?
511    assert (!mCarryFrameStack.empty());
512
513    std::tie(mCurrentFrame, mCurrentFrameIndex) = mCarryFrameStack.back();
514
515    assert(mCurrentFrame->getType()->getPointerElementType()->isStructTy());
516
517    mCarryFrameStack.pop_back();
518    mCarryScopeIndex.pop_back();
519    assert (!mCarryScopeIndex.empty());
520    mCurrentScope = mCurrentScope->getPredecessor();
521    assert (mCurrentScope);
522    mCarryInfo = &mCarryMetadata[mCarryScopeIndex.back()];
523}
524
525/** ------------------------------------------------------------------------------------------------------------- *
526 * @brief addCarryInCarryOut
527 ** ------------------------------------------------------------------------------------------------------------- */
528Value * CarryManager::addCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const Statement * const operation, Value * const e1, Value * const e2) {
529    assert (operation && (isNonAdvanceCarryGeneratingStatement(operation)));
530    Value * const carryIn = getNextCarryIn(iBuilder);
531    Value * carryOut, * result;
532    std::tie(carryOut, result) = iBuilder->bitblock_add_with_carry(e1, e2, carryIn);
533    setNextCarryOut(iBuilder, carryOut);
534    assert (result->getType() == iBuilder->getBitBlockType());
535    return result;
536}
537
538/** ------------------------------------------------------------------------------------------------------------- *
539 * @brief advanceCarryInCarryOut
540 ** ------------------------------------------------------------------------------------------------------------- */
541Value * CarryManager::advanceCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const Advance * const advance, Value * const value) {
542    const auto shiftAmount = advance->getAmount();
543    if (LLVM_LIKELY(shiftAmount < LONG_ADVANCE_BREAKPOINT)) {
544        Value * const carryIn = getNextCarryIn(iBuilder);
545        Value * carryOut, * result;
546        std::tie(carryOut, result) = iBuilder->bitblock_advance(value, carryIn, shiftAmount);
547        setNextCarryOut(iBuilder, carryOut);
548        assert (result->getType() == iBuilder->getBitBlockType());
549        return result;
550    } else {
551        return longAdvanceCarryInCarryOut(iBuilder, value, shiftAmount);
552    }
553}
554
555/** ------------------------------------------------------------------------------------------------------------- *
556 * @brief longAdvanceCarryInCarryOut
557 ** ------------------------------------------------------------------------------------------------------------- */
558inline Value * CarryManager::longAdvanceCarryInCarryOut(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, Value * const value, const unsigned shiftAmount) {
559
560    assert (mHasLongAdvance);
561    assert (shiftAmount >= LONG_ADVANCE_BREAKPOINT);
562
563    Type * const streamTy = iBuilder->getIntNTy(iBuilder->getBitBlockWidth());
564
565    if (mIfDepth > 0) {
566        if (shiftAmount > iBuilder->getBitBlockWidth()) {
567            const auto frameIndex = mCurrentFrameIndex++;
568            Value * carry = iBuilder->CreateZExt(iBuilder->bitblock_any(value), streamTy);
569            const unsigned summarySize = ceil_udiv(shiftAmount, iBuilder->getBitBlockWidth() * iBuilder->getBitBlockWidth());
570            for (unsigned i = 0;;++i) {
571                Value * const ptr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), iBuilder->getInt32(i)});
572                Value * const prior = iBuilder->CreateBitCast(iBuilder->CreateBlockAlignedLoad(ptr), streamTy);
573                Value * const stream = iBuilder->CreateBitCast(iBuilder->CreateOr(iBuilder->CreateShl(prior, 1), carry), iBuilder->getBitBlockType());
574                if (LLVM_LIKELY(i == summarySize)) {
575                    Value * const maskedStream = iBuilder->CreateAnd(stream, iBuilder->bitblock_mask_from(iBuilder->getInt32(summarySize % iBuilder->getBitBlockWidth())));
576                    addToCarryOutSummary(iBuilder, maskedStream);
577                    iBuilder->CreateBlockAlignedStore(maskedStream, ptr);
578                    break;
579                }
580                addToCarryOutSummary(iBuilder, stream);
581                iBuilder->CreateBlockAlignedStore(stream, ptr);
582                carry = iBuilder->CreateLShr(prior, iBuilder->getBitBlockWidth() - 1);
583            }
584        } else if (LLVM_LIKELY(mCarryInfo->hasExplicitSummary())) {
585            addToCarryOutSummary(iBuilder, value);
586        }
587    }
588    const auto frameIndex = mCurrentFrameIndex++;
589    // special case using a single buffer entry and the carry_out value.
590    if (LLVM_LIKELY((shiftAmount < iBuilder->getBitBlockWidth()) && (mLoopDepth == 0))) {
591        Value * const buffer = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), iBuilder->getInt32(0)});
592        assert (buffer->getType()->getPointerElementType() == iBuilder->getBitBlockType());
593        Value * carryIn = iBuilder->CreateBlockAlignedLoad(buffer);
594        iBuilder->CreateBlockAlignedStore(value, buffer);
595        /* Very special case - no combine */
596        if (LLVM_UNLIKELY(shiftAmount == iBuilder->getBitBlockWidth())) {
597            return iBuilder->CreateBitCast(carryIn, iBuilder->getBitBlockType());
598        }
599        Value* block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carryIn, streamTy), iBuilder->getBitBlockWidth() - shiftAmount);
600        Value* block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(value, streamTy), shiftAmount);
601        return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), iBuilder->getBitBlockType());
602    } else { //
603        const unsigned blockShift = shiftAmount % iBuilder->getBitBlockWidth();
604        const unsigned blocks = ceil_udiv(shiftAmount, iBuilder->getBitBlockWidth());
605        // Create a mask to implement circular buffer indexing
606        Value * indexMask = iBuilder->getSize(nearest_pow2(blocks + ((mLoopDepth != 0) ? 1 : 0)) - 1);
607        Value * blockIndex = iBuilder->getScalarField("CarryBlockIndex");
608        Value * carryIndex0 = iBuilder->CreateSub(blockIndex, iBuilder->getSize(blocks));
609        Value * loadIndex0 = iBuilder->CreateAnd(carryIndex0, indexMask);
610        Value * const carryInPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), loadIndex0});
611        Value * carryIn = iBuilder->CreateBlockAlignedLoad(carryInPtr);
612
613        Value * storeIndex = iBuilder->CreateAnd(blockIndex, indexMask);
614        Value * const carryOutPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), storeIndex});
615        assert (carryIn->getType() == iBuilder->getBitBlockType());
616
617        // If the long advance is an exact multiple of BitBlockWidth, we simply return the oldest
618        // block in the long advance carry data area.
619        if (LLVM_UNLIKELY(blockShift == 0)) {
620            iBuilder->CreateBlockAlignedStore(value, carryOutPtr);
621            return carryIn;
622        } else { // Otherwise we need to combine data from the two oldest blocks.
623            Value * carryIndex1 = iBuilder->CreateSub(blockIndex, iBuilder->getSize(blocks - 1));
624            Value * loadIndex1 = iBuilder->CreateAnd(carryIndex1, indexMask);
625            Value * const carryInPtr2 = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(frameIndex), loadIndex1});
626            Value * carry_block1 = iBuilder->CreateBlockAlignedLoad(carryInPtr2);
627            Value * block0_shr = iBuilder->CreateLShr(iBuilder->CreateBitCast(carryIn, streamTy), iBuilder->getBitBlockWidth() - blockShift);
628            Value * block1_shl = iBuilder->CreateShl(iBuilder->CreateBitCast(carry_block1, streamTy), blockShift);
629            iBuilder->CreateBlockAlignedStore(value, carryOutPtr);
630            return iBuilder->CreateBitCast(iBuilder->CreateOr(block1_shl, block0_shr), iBuilder->getBitBlockType());
631        }
632    }
633}
634
635/** ------------------------------------------------------------------------------------------------------------- *
636 * @brief getNextCarryIn
637 ** ------------------------------------------------------------------------------------------------------------- */
638Value * CarryManager::getNextCarryIn(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
639    assert (mCurrentFrameIndex < mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
640    if (mLoopDepth == 0) {
641        mCarryPackPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex)});
642    } else {
643        mCarryPackPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex), mLoopSelector});
644    }
645    Type * const carryTy = iBuilder->getBitBlockType();
646    assert (mCarryPackPtr->getType()->getPointerElementType() == carryTy);
647    Value * const carryIn = iBuilder->CreateBlockAlignedLoad(mCarryPackPtr);
648    if (mLoopDepth > 0) {
649        iBuilder->CreateBlockAlignedStore(Constant::getNullValue(carryTy), mCarryPackPtr);
650    }
651    return carryIn;
652}
653
654/** ------------------------------------------------------------------------------------------------------------- *
655 * @brief setNextCarryOut
656 ** ------------------------------------------------------------------------------------------------------------- */
657void CarryManager::setNextCarryOut(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, Value * carryOut) {
658    Type * const carryTy = iBuilder->getBitBlockType();
659    assert (mCurrentFrameIndex < mCurrentFrame->getType()->getPointerElementType()->getStructNumElements());
660    carryOut = iBuilder->CreateBitCast(carryOut, carryTy);
661    if (mCarryInfo->hasSummary()) {
662        addToCarryOutSummary(iBuilder, carryOut);
663    }
664    if (mLoopDepth != 0) {
665        mCarryPackPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(mCurrentFrameIndex), mNextLoopSelector});
666        if (LLVM_LIKELY(!mCarryInfo->nonCarryCollapsingMode())) {
667            Value * accum = iBuilder->CreateBlockAlignedLoad(mCarryPackPtr);
668            carryOut = iBuilder->CreateOr(carryOut, accum);
669        }
670    }
671    ++mCurrentFrameIndex;
672    assert (mCarryPackPtr->getType()->getPointerElementType() == carryTy);
673    iBuilder->CreateBlockAlignedStore(carryOut, mCarryPackPtr);
674}
675
676/** ------------------------------------------------------------------------------------------------------------- *
677 * @brief readCarryInSummary
678 ** ------------------------------------------------------------------------------------------------------------- */
679Value * CarryManager::readCarryInSummary(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, ConstantInt * index) const {
680    assert (mCarryInfo->hasSummary());
681    unsigned count = 2;
682    if (LLVM_UNLIKELY(mCarryInfo->hasBorrowedSummary())) {
683        Type * frameTy = mCurrentFrame->getType()->getPointerElementType();
684        count = 1;
685        while (frameTy->isStructTy()) {
686            ++count;
687            frameTy = frameTy->getStructElementType(0);
688        }
689    }
690    const unsigned length = (mLoopDepth == 0) ? count : (count + 1);
691    Value * indicies[length];
692    std::fill(indicies, indicies + count - 1, iBuilder->getInt32(0));
693    indicies[count - 1] = index;
694    if (mLoopDepth != 0) {
695        indicies[count] = mLoopSelector;
696    }
697
698    ArrayRef<Value *> ar(indicies, length);
699    Value * const ptr = iBuilder->CreateGEP(mCurrentFrame, ar);
700    Value * const summary = iBuilder->CreateBlockAlignedLoad(ptr);
701    if (mLoopDepth != 0 && mCarryInfo->hasExplicitSummary()) {
702        Type * const carryTy = iBuilder->getBitBlockType();
703        iBuilder->CreateBlockAlignedStore(Constant::getNullValue(carryTy), ptr);
704    }
705    return summary;
706}
707
708/** ------------------------------------------------------------------------------------------------------------- *
709 * @brief writeCarryOutSummary
710 ** ------------------------------------------------------------------------------------------------------------- */
711inline void CarryManager::writeCarryOutSummary(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, Value * const summary, ConstantInt * index) const {
712    Value * ptr = nullptr;
713    assert (mCarryInfo->hasExplicitSummary());
714    if (mLoopDepth > 0) {
715        ptr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), index, mNextLoopSelector});
716    } else {
717        ptr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), index});
718    }
719    iBuilder->CreateBlockAlignedStore(summary, ptr);
720}
721
722/** ------------------------------------------------------------------------------------------------------------- *
723 * @brief addToCarryOutSummary
724 ** ------------------------------------------------------------------------------------------------------------- */
725inline void CarryManager::addToCarryOutSummary(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, Value * const value) {
726    assert ("cannot add null summary value!" && value);   
727    assert ("summary stack is empty!" && !mCarrySummaryStack.empty());
728    assert (mCarryInfo->hasSummary());
729    mCarrySummaryStack.back() = iBuilder->CreateOr(value, mCarrySummaryStack.back());
730}
731
732/** ------------------------------------------------------------------------------------------------------------- *
733 * @brief enumerate
734 ** ------------------------------------------------------------------------------------------------------------- */
735unsigned CarryManager::getScopeCount(const PabloBlock * const scope, unsigned index) {
736    for (const Statement * stmt : *scope) {
737        if (LLVM_UNLIKELY(isa<Branch>(stmt))) {
738            index = getScopeCount(cast<Branch>(stmt)->getBody(), index);
739        }
740    }
741    return index + 1;
742}
743
744/** ------------------------------------------------------------------------------------------------------------- *
745 * @brief hasIterationSpecificAssignment
746 ** ------------------------------------------------------------------------------------------------------------- */
747bool CarryManager::hasIterationSpecificAssignment(const PabloBlock * const scope) {
748#if 0
749    return dyn_cast_or_null<While>(scope->getBranch()) != nullptr;
750#else
751    if (const While * const br = dyn_cast_or_null<While>(scope->getBranch())) {
752        for (const Var * var : br->getEscaped()) {
753            for (const PabloAST * user : var->users()) {
754                if (const Extract * e = dyn_cast<Extract>(user)) {
755                    if (LLVM_UNLIKELY(e->getIndex() == var)) {
756                        // If we assign this Var a value and read the value as the index parameter
757                        // of a nested Extract statement, then we cannot collapse the carries.
758                        const PabloBlock * parent = e->getParent();
759                        for (;;) {
760                            if (parent == scope) {
761                                return true;
762                            }
763                            parent = parent->getPredecessor();
764                            if (parent == nullptr) {
765                                break;
766                            }
767                        }
768                    }
769                }
770            }
771        }
772    }
773    return false;
774#endif
775}
776
777/** ------------------------------------------------------------------------------------------------------------- *
778 * @brief analyse
779 ** ------------------------------------------------------------------------------------------------------------- */
780StructType * CarryManager::analyse(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const PabloBlock * const scope, const unsigned ifDepth, const unsigned loopDepth, const bool isNestedWithinNonCarryCollapsingLoop) {
781    assert ("scope cannot be null!" && scope);
782    assert (mCarryScopes == 0 ? (scope == mKernel->getEntryBlock()) : (scope != mKernel->getEntryBlock()));
783    assert (mCarryScopes < mCarryMetadata.size());
784    Type * const carryTy = iBuilder->getBitBlockType();
785    Type * const blockTy = iBuilder->getBitBlockType();
786
787    const unsigned carryScopeIndex = mCarryScopes++;
788    const bool nonCarryCollapsingMode = hasIterationSpecificAssignment(scope);
789    Type * const carryPackType = (loopDepth == 0) ? carryTy : ArrayType::get(carryTy, 2);
790    std::vector<Type *> state;
791
792    for (const Statement * stmt : *scope) {
793        if (LLVM_UNLIKELY(isa<Advance>(stmt))) {
794            const auto amount = cast<Advance>(stmt)->getAmount();
795            Type * type = carryPackType;
796            if (LLVM_UNLIKELY(amount >= LONG_ADVANCE_BREAKPOINT)) {
797                const unsigned blocks = ceil_udiv(amount, iBuilder->getBitBlockWidth());
798                type = ArrayType::get(blockTy, nearest_pow2(blocks + ((loopDepth != 0) ? 1 : 0)));
799                if (LLVM_UNLIKELY(ifDepth > 0 && amount > iBuilder->getBitBlockWidth())) {
800                    // 1 bit will mark the presense of any bit in each block.
801                    Type * carryType = ArrayType::get(blockTy, ceil_udiv(amount, std::pow(iBuilder->getBitBlockWidth(), 2)));
802                    state.push_back(carryType);
803                }
804                mHasLongAdvance = true;               
805            }
806            state.push_back(type);
807        } else if (LLVM_UNLIKELY(isNonAdvanceCarryGeneratingStatement(stmt))) {
808            state.push_back(carryPackType);
809        } else if (LLVM_UNLIKELY(isa<If>(stmt))) {
810            state.push_back(analyse(iBuilder, cast<If>(stmt)->getBody(), ifDepth + 1, loopDepth, nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop));
811        } else if (LLVM_UNLIKELY(isa<While>(stmt))) {
812            mHasLoop = true;
813            state.push_back(analyse(iBuilder, cast<While>(stmt)->getBody(), ifDepth, loopDepth + 1, nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop));
814        }
815    }
816    // Build the carry state struct and add the summary pack if needed.
817    CarryData & cd = mCarryMetadata[carryScopeIndex];
818    StructType * carryState = nullptr;
819    CarryData::SummaryType summaryType = CarryData::NoSummary;
820    if (LLVM_UNLIKELY(state.empty())) {
821        carryState = StructType::get(iBuilder->getContext());
822    } else {
823        //if (ifDepth > 0 || (nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop)) {
824        if (dyn_cast_or_null<If>(scope->getBranch()) || nonCarryCollapsingMode || isNestedWithinNonCarryCollapsingLoop) {
825            if (LLVM_LIKELY(state.size() > 1)) {
826                summaryType = CarryData::ExplicitSummary;
827                // NOTE: summaries are stored differently depending whether we're entering an If or While branch. With an If branch, they
828                // preceed the carry state data and with a While loop they succeed it. This is to help cache prefectching performance.
829                state.insert(isa<If>(scope->getBranch()) ? state.begin() : state.end(), carryPackType);
830            } else {
831                summaryType = CarryData::ImplicitSummary;
832                if (state[0]->isStructTy()) {
833                    summaryType = CarryData::BorrowedSummary;
834                }
835            }           
836        }
837        carryState = StructType::get(iBuilder->getContext(), state);
838        // If we're in a loop and cannot use collapsing carry mode, convert the carry state struct into a capacity,
839        // carry state pointer, and summary pointer struct.
840        if (LLVM_UNLIKELY(nonCarryCollapsingMode)) {
841            carryState = StructType::get(iBuilder->getSizeTy(), carryState->getPointerTo(), carryTy->getPointerTo(), nullptr);
842        }
843        cd.setNonCollapsingCarryMode(nonCarryCollapsingMode);
844    }
845    cd.setSummaryType(summaryType);
846    return carryState;
847}
848
849/** ------------------------------------------------------------------------------------------------------------- *
850 * @brief constructor
851 ** ------------------------------------------------------------------------------------------------------------- */
852CarryManager::CarryManager() noexcept
853: mKernel(nullptr)
854, mCurrentFrame(nullptr)
855, mCurrentFrameIndex(0)
856, mCurrentScope(nullptr)
857, mCarryInfo(nullptr)
858, mNextSummaryTest(nullptr)
859, mIfDepth(0)
860, mHasLongAdvance(false)
861, mHasLoop(false)
862, mLoopDepth(0)
863, mLoopSelector(nullptr)
864, mNextLoopSelector(nullptr)
865, mCarryPackPtr(nullptr)
866, mCarryScopes(0) {
867
868}
869
870}
Note: See TracBrowser for help on using the repository browser.