source: icGREP/icgrep-devel/icgrep/kernels/pipeline/core_logic.hpp @ 6233

Last change on this file since 6233 was 6233, checked in by nmedfort, 4 months ago

Moved termination signals into pipeline kernel

File size: 25.3 KB
Line 
1#include "pipeline_compiler.hpp"
2
3const static std::string TERMINATION_SIGNAL = "terminationSignal";
4
5namespace kernel {
6
7/** ------------------------------------------------------------------------------------------------------------- *
8 * @brief addInternalKernelProperties
9 ** ------------------------------------------------------------------------------------------------------------- */
10inline void PipelineCompiler::addInternalKernelProperties(BuilderRef b) {
11    initializePopCounts();
12    const auto numOfKernels = mPipeline.size();
13    b->setKernel(mPipelineKernel);
14    IntegerType * const boolTy = b->getInt1Ty();
15    for (unsigned i = 0; i < numOfKernels; ++i) {
16        // TODO: prove two termination signals can be fused into a single counter?
17        const auto prefix = makeKernelName(i);
18        mPipelineKernel->addInternalScalar(boolTy, prefix + TERMINATION_SIGNAL);
19        addBufferHandlesToPipelineKernel(b, i);
20        addPopCountScalarsToPipelineKernel(b, i);
21    }
22    b->setKernel(mPipelineKernel);
23}
24
25/** ------------------------------------------------------------------------------------------------------------- *
26 * @brief generateInitializeMethod
27 ** ------------------------------------------------------------------------------------------------------------- */
28void PipelineCompiler::generateInitializeMethod(BuilderRef b) {
29    const auto numOfKernels = mPipeline.size();
30    for (unsigned i = 0; i < numOfKernels; ++i) {
31        mPipeline[i]->addKernelDeclarations(b);
32    }
33    for (unsigned i = 0; i < numOfKernels; ++i) {
34        Kernel * const kernel = mPipeline[i];
35        if (!kernel->hasFamilyName()) {
36            Value * const handle = kernel->createInstance(b);
37            b->setScalarField(makeKernelName(i), handle);
38        }
39    }
40    constructBuffers(b);
41    std::vector<Value *> args;
42    for (unsigned i = 0; i < numOfKernels; ++i) {
43        setActiveKernel(b, i);
44        args.resize(in_degree(i, mScalarDependencyGraph) + 1);
45        #ifndef NDEBUG
46        std::fill(args.begin(), args.end(), nullptr);
47        #endif
48        args[0] = mKernel->getHandle();
49        b->setKernel(mPipelineKernel);
50        for (const auto ce : make_iterator_range(in_edges(i, mScalarDependencyGraph))) {
51            const auto j = mScalarDependencyGraph[ce] + 1;
52            const auto pe = in_edge(source(ce, mScalarDependencyGraph), mScalarDependencyGraph);
53            const auto k = mScalarDependencyGraph[pe];
54            const Binding & input = mPipelineKernel->getInputScalarBinding(k);
55            assert (args[j] == nullptr);
56            args[j] = b->getScalarField(input.getName());
57        }
58        b->setKernel(mKernel);
59        Value * const terminatedOnInit = b->CreateCall(getInitializationFunction(b), args);
60        if (mKernel->canSetTerminateSignal()) {
61            setTerminated(b, terminatedOnInit);
62        }
63    }
64}
65
66/** ------------------------------------------------------------------------------------------------------------- *
67 * @brief start
68 ** ------------------------------------------------------------------------------------------------------------- */
69void PipelineCompiler::start(BuilderRef b, Value * const initialSegNo) {
70
71    // Create the basic blocks for the loop.
72    BasicBlock * const entryBlock = b->GetInsertBlock();
73    mPipelineLoop = b->CreateBasicBlock("pipelineLoop");
74    mPipelineEnd = b->CreateBasicBlock("pipelineEnd");
75
76    mKernel = nullptr;
77    mKernelIndex = 0;
78    b->CreateBr(mPipelineLoop);
79
80    b->SetInsertPoint(mPipelineLoop);
81    mSegNo = b->CreatePHI(b->getSizeTy(), 2, "segNo");
82    mSegNo->addIncoming(initialSegNo, entryBlock);
83    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
84        mDeadLockCounter = b->CreatePHI(b->getSizeTy(), 2, "deadLockCounter");
85        mDeadLockCounter->addIncoming(b->getSize(0), entryBlock);
86        mPipelineProgress = b->getFalse();
87    }
88    #ifdef PRINT_DEBUG_MESSAGES
89    b->CallPrintInt("+++ pipeline start +++", mSegNo);
90    #endif
91    startOptionalCycleCounter(b);
92}
93
94/** ------------------------------------------------------------------------------------------------------------- *
95 * @brief executeKernel
96 ** ------------------------------------------------------------------------------------------------------------- */
97void PipelineCompiler::executeKernel(BuilderRef b) {
98
99    resetMemoizedFields();
100    mPortOrdering = lexicalOrderingOfStreamIO();
101    loadBufferHandles(b);
102
103    mKernelEntry = b->GetInsertBlock();
104
105    const auto kernelName = makeKernelName(mKernelIndex);
106    BasicBlock * const checkProducers = b->CreateBasicBlock(kernelName + "_checkProducers", mPipelineEnd);
107    mKernelLoopEntry = b->CreateBasicBlock(kernelName + "_loopEntry", mPipelineEnd);
108    mKernelLoopCall = b->CreateBasicBlock(kernelName + "_executeKernel", mPipelineEnd);
109    mKernelLoopExit = b->CreateBasicBlock(kernelName + "_loopExit", mPipelineEnd);
110    mKernelExit = b->CreateBasicBlock(kernelName + "_kernelExit", mPipelineEnd);
111    // The phi catch simplifies compilation logic by "forward declaring" the loop exit point.
112    // Subsequent optimization phases will collapse it into the correct exit block.
113    mKernelLoopExitPhiCatch = b->CreateBasicBlock(kernelName + "_kernelExitPhiCatch", mPipelineEnd);
114
115    /// -------------------------------------------------------------------------------------
116    /// KERNEL ENTRY
117    /// -------------------------------------------------------------------------------------
118    Value * const initiallyTerminated = getInitialTerminationSignal(b);
119    #ifdef PRINT_DEBUG_MESSAGES
120    if (1) {
121    Constant * const MAX_INT = ConstantInt::getAllOnesValue(mSegNo->getType());
122    Value * const round = b->CreateSelect(initiallyTerminated, MAX_INT, mSegNo);
123    b->CallPrintInt("--- " + kernelName + "_start ---", round);
124    }
125    #endif
126    b->CreateUnlikelyCondBr(initiallyTerminated, mKernelExit, checkProducers);
127
128    /// -------------------------------------------------------------------------------------
129    /// KERNEL CHECK PRODUCERS
130    /// -------------------------------------------------------------------------------------
131
132    b->SetInsertPoint(checkProducers);
133    readInitialProducedItemCounts(b);
134    b->CreateBr(mKernelLoopEntry);
135
136    // Set up some PHI nodes "early" to simplify accumulating their incoming values.
137
138    /// -------------------------------------------------------------------------------------
139    /// KERNEL LOOP ENTRY
140    /// -------------------------------------------------------------------------------------
141
142    b->SetInsertPoint(mKernelLoopEntry);
143    // Since we may loop and call the kernel again, we want to mark that we've progressed
144    // if we execute any kernel even if we could not complete a full segment.
145    if (mPipelineProgress) {
146        mAlreadyProgressedPhi = b->CreatePHI(b->getInt1Ty(), 2, kernelName + "_madeProgress");
147        mAlreadyProgressedPhi->addIncoming(mPipelineProgress, checkProducers);
148    }
149
150    /// -------------------------------------------------------------------------------------
151    /// KERNEL CALL
152    /// -------------------------------------------------------------------------------------
153
154    b->SetInsertPoint(mKernelLoopCall);
155    initializeKernelCallPhis(b);
156
157    /// -------------------------------------------------------------------------------------
158    /// KERNEL LOOP EXIT
159    /// -------------------------------------------------------------------------------------
160
161    b->SetInsertPoint(mKernelLoopExit);
162    mTerminatedPhi = b->CreatePHI(b->getInt1Ty(), 2, kernelName + "_terminated");
163    if (mPipelineProgress) {
164        mHasProgressedPhi = b->CreatePHI(b->getInt1Ty(), 2, kernelName + "_anyProgress");
165    }
166
167    /// -------------------------------------------------------------------------------------
168    /// KERNEL EXIT
169    /// -------------------------------------------------------------------------------------
170
171    b->SetInsertPoint(mKernelExit);
172    initializeKernelExitPhis(b);
173
174    /// -------------------------------------------------------------------------------------
175    /// KERNEL LOOP ENTRY (CONTINUED)
176    /// -------------------------------------------------------------------------------------
177
178    b->SetInsertPoint(mKernelLoopEntry);
179    checkForSufficientInputDataAndOutputSpace(b);
180    determineNumOfLinearStrides(b);
181
182    Value * isFinal = nullptr;
183
184    ConstantInt * const ZERO = b->getSize(0);
185
186    if (mNumOfLinearStrides) {
187
188        BasicBlock * const enteringNonFinalSegment = b->CreateBasicBlock(kernelName + "_nonFinalSegment", mKernelLoopCall);
189        BasicBlock * const enteringFinalStride = b->CreateBasicBlock(kernelName + "_finalStride", mKernelLoopCall);
190
191        isFinal = b->CreateICmpEQ(mNumOfLinearStrides, ZERO);
192
193        b->CreateUnlikelyCondBr(isFinal, enteringFinalStride, enteringNonFinalSegment);
194
195        /// -------------------------------------------------------------------------------------
196        /// KERNEL ENTERING FINAL STRIDE
197        /// -------------------------------------------------------------------------------------
198
199        b->SetInsertPoint(enteringFinalStride);
200        calculateFinalItemCounts(b);
201        b->CreateBr(mKernelLoopCall);
202
203        /// -------------------------------------------------------------------------------------
204        /// KERNEL ENTERING NON-FINAL SEGMENT
205        /// -------------------------------------------------------------------------------------
206
207        b->SetInsertPoint(enteringNonFinalSegment);
208        calculateNonFinalItemCounts(b);
209        b->CreateBr(mKernelLoopCall);
210
211    } else {
212        mNumOfLinearStrides = ZERO;
213        b->CreateBr(mKernelLoopCall);
214    }
215
216    /// -------------------------------------------------------------------------------------
217    /// KERNEL CALL (CONTINUED)
218    /// -------------------------------------------------------------------------------------
219
220    b->SetInsertPoint(mKernelLoopCall);
221    expandOutputBuffers(b);
222    writeKernelCall(b);
223
224    BasicBlock * const incrementItemCounts = b->CreateBasicBlock(kernelName + "_incrementItemCounts", mKernelLoopExit);
225    BasicBlock * const terminationCheck = b->CreateBasicBlock(kernelName + "_normalTerminationCheck", mKernelLoopExit);
226    BasicBlock * const terminated = b->CreateBasicBlock(kernelName + "_terminated", mKernelLoopExit);
227
228    // If the kernel itself terminates, it must set the final processed/produced item counts.
229    // Otherwise, the pipeline will update any countable rates, even upon termination.
230    b->CreateUnlikelyCondBr(mTerminationExplicitly, terminated, incrementItemCounts);
231
232    /// -------------------------------------------------------------------------------------
233    /// KERNEL INCREMENT ITEM COUNTS
234    /// -------------------------------------------------------------------------------------
235
236    b->SetInsertPoint(incrementItemCounts);
237    // TODO: phi out the item counts and set them once at the end.
238    incrementItemCountsOfCountableRateStreams(b);
239    writeCopyBackLogic(b);
240    b->CreateBr(terminationCheck);
241
242    /// -------------------------------------------------------------------------------------
243    /// KERNEL NORMAL TERMINATION CHECK
244    /// -------------------------------------------------------------------------------------
245
246    b->SetInsertPoint(terminationCheck);
247    if (isFinal) {
248        if (mAlreadyProgressedPhi) {
249            mAlreadyProgressedPhi->addIncoming(b->getTrue(), terminationCheck);
250        }
251        b->CreateUnlikelyCondBr(isFinal, terminated, mKernelLoopEntry);
252    } else { // just exit the loop
253        if (mHasProgressedPhi) {
254            mHasProgressedPhi->addIncoming(b->getTrue(), terminationCheck);
255        }
256        mTerminatedPhi->addIncoming(b->getFalse(), terminationCheck);
257        b->CreateBr(mKernelLoopExit);
258    }
259
260    /// -------------------------------------------------------------------------------------
261    /// KERNEL TERMINATED
262    /// -------------------------------------------------------------------------------------
263
264    b->SetInsertPoint(terminated);
265    zeroFillPartiallyWrittenOutputStreams(b);
266    setTerminated(b, b->getTrue());
267    BasicBlock * const kernelTerminatedEnd = b->GetInsertBlock();
268    mTerminatedPhi->addIncoming(b->getTrue(), kernelTerminatedEnd);
269    if (mHasProgressedPhi) {
270        mHasProgressedPhi->addIncoming(b->getTrue(), kernelTerminatedEnd);
271    }
272    b->CreateBr(mKernelLoopExit);
273
274    /// -------------------------------------------------------------------------------------
275    /// KERNEL LOOP EXIT (CONTINUED)
276    /// -------------------------------------------------------------------------------------
277
278    b->SetInsertPoint(mKernelLoopExit);
279    computeFullyProcessedItemCounts(b);
280    computeMinimumConsumedItemCounts(b);
281    computeMinimumPopCountReferenceCounts(b);
282    writeCopyForwardLogic(b);
283    writePopCountComputationLogic(b);
284    b->CreateBr(mKernelLoopExitPhiCatch);
285    b->SetInsertPoint(mKernelLoopExitPhiCatch);
286    b->CreateBr(mKernelExit);
287
288    /// -------------------------------------------------------------------------------------
289    /// KERNEL EXIT (CONTINUED)
290    /// -------------------------------------------------------------------------------------
291
292    b->SetInsertPoint(mKernelExit);
293    writeFinalConsumedItemCounts(b);
294    updatePopCountReferenceCounts(b);
295
296    // TODO: logically we should only need to read produced item counts in the loop exit; however, that
297    // would mean that we'd first need to load the initial produced item counts prior to the loop entry
298    // to have access to them here and then PHI them out within the kernel loop
299
300    readFinalProducedItemCounts(b);
301    releaseCurrentSegment(b);
302    updateOptionalCycleCounter(b);
303
304    assert (mKernel == mPipeline[mKernelIndex] && b->getKernel() == mKernel);
305}
306
307/** ------------------------------------------------------------------------------------------------------------- *
308 * @brief wait
309 ** ------------------------------------------------------------------------------------------------------------- */
310void PipelineCompiler::synchronize(BuilderRef b) {
311
312    const auto kernelName = makeKernelName(mKernelIndex);
313
314    BasicBlock * const kernelWait = b->CreateBasicBlock(kernelName + "Wait", mPipelineEnd);
315    b->CreateBr(kernelWait);
316
317    b->SetInsertPoint(kernelWait);
318    const Kernel * waitingOn = mKernel;
319    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::SerializeThreads))) {
320        waitingOn = mPipeline.back();
321    }
322    b->setKernel(waitingOn);
323    Value * const processedSegmentCount = b->acquireLogicalSegmentNo();
324    assert (processedSegmentCount->getType() == mSegNo->getType());
325    Value * const ready = b->CreateICmpEQ(mSegNo, processedSegmentCount);
326
327    BasicBlock * const kernelCheck = b->CreateBasicBlock(kernelName + "Check", mPipelineEnd);
328    b->CreateCondBr(ready, kernelCheck, kernelWait);
329
330    b->SetInsertPoint(kernelCheck);
331    b->setKernel(mKernel);
332}
333
334
335/** ------------------------------------------------------------------------------------------------------------- *
336 * @brief next
337 ** ------------------------------------------------------------------------------------------------------------- */
338void PipelineCompiler::end(BuilderRef b, const unsigned step) {
339    b->setKernel(mPipelineKernel);
340    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
341        ConstantInt * const ZERO = b->getSize(0);
342        ConstantInt * const ONE = b->getSize(1);
343        ConstantInt * const TWO = b->getSize(2);
344        Value * const plusOne = b->CreateAdd(mDeadLockCounter, ONE);
345        Value * const newCount = b->CreateSelect(mPipelineProgress, ZERO, plusOne);
346        b->CreateAssert(b->CreateICmpNE(newCount, TWO),
347                        "Dead lock detected: pipeline could not progress after two iterations");
348        mDeadLockCounter->addIncoming(newCount, b->GetInsertBlock());
349    }
350    // check whether every sink has terminated
351    Value * allTerminated = b->getTrue();
352    const auto pipelineOutputVertex = mPipeline.size();
353    for (const auto e : make_iterator_range(in_edges(pipelineOutputVertex, mTerminationGraph))) {
354        const auto u = source(e, mTerminationGraph);
355        assert (mTerminationGraph[u]);
356        allTerminated = b->CreateAnd(allTerminated, mTerminationGraph[u]);
357    }
358    // or if any output stream of this pipeline cannot support a full stride
359    Value * notEnoughSpace = b->getFalse();
360    for (const auto e : make_iterator_range(in_edges(pipelineOutputVertex, mBufferGraph))) {
361        // TODO: not a very elegant way here; revise
362        const auto bufferVertex = source(e, mBufferGraph);
363        setActiveKernel(b, parent(bufferVertex, mBufferGraph));
364        resetMemoizedFields();
365        const auto outputPort = mBufferGraph[e].Port;
366        Value * const writable = getWritableOutputItems(b, outputPort);
367        // NOTE: this method doesn't check a popcount's ref stream to determine how many
368        // items we actually require. Instead it just calculates them as bounded rates.
369        // To support a precise bound, we'd need to produce more ref items than the kernel
370        // that writes to this output actually consumes. Since this effectively adds a
371        // delay equivalent to a LookAhead of a full stride, this doesn't seem useful.
372        Value * const strideLength = getMaximumStrideLength(b, Port::Output, outputPort);
373        notEnoughSpace = b->CreateOr(b->CreateICmpULT(writable, strideLength), notEnoughSpace);
374    }
375    b->setKernel(mPipelineKernel);
376    Value * const done = b->CreateOr(allTerminated, notEnoughSpace);
377    #ifdef PRINT_DEBUG_MESSAGES
378    Constant * const ONES = Constant::getAllOnesValue(mSegNo->getType());
379    b->CallPrintInt("+++ pipeline end +++", b->CreateSelect(done, ONES, mSegNo));
380    #endif
381
382    Value * const nextSegNo = b->CreateAdd(mSegNo, b->getSize(step));
383    mSegNo->addIncoming(nextSegNo, b->GetInsertBlock());
384    b->CreateUnlikelyCondBr(done, mPipelineEnd, mPipelineLoop);
385
386    b->SetInsertPoint(mPipelineEnd);
387    mSegNo = nullptr;
388}
389
390/** ------------------------------------------------------------------------------------------------------------- *
391 * @brief generateFinalizeMethod
392 ** ------------------------------------------------------------------------------------------------------------- */
393void PipelineCompiler::generateFinalizeMethod(BuilderRef b) {
394    printOptionalCycleCounter(b);
395    const auto numOfKernels = mPipeline.size();
396    mOutputScalars.resize(numOfKernels);
397    for (unsigned i = 0; i < numOfKernels; ++i) {
398        setActiveKernel(b, i);
399        loadBufferHandles(b);
400        mOutputScalars[i] = b->CreateCall(getFinalizeFunction(b), mKernel->getHandle());
401    }
402    releaseBuffers(b);
403}
404
405/** ------------------------------------------------------------------------------------------------------------- *
406 * @brief writeOutputScalars
407 ** ------------------------------------------------------------------------------------------------------------- */
408void PipelineCompiler::writeOutputScalars(BuilderRef b, const unsigned u, std::vector<Value *> & args) {
409    args.clear();
410    const auto n = in_degree(u, mScalarDependencyGraph);
411    args.resize(n, nullptr);
412    const auto numOfKernels = mPipeline.size();
413    for (const auto e : make_iterator_range(in_edges(u, mScalarDependencyGraph))) {
414        const auto bufferVertex = source(e, mScalarDependencyGraph);
415        if (LLVM_LIKELY(mScalarDependencyGraph[bufferVertex] == nullptr)) {
416            const auto producer = in_edge(source(e, mScalarDependencyGraph), mScalarDependencyGraph);
417            const auto i = source(producer, mScalarDependencyGraph);
418            const auto j = mScalarDependencyGraph[producer];
419            Value * val = nullptr;
420            if (i == numOfKernels) {
421                const Binding & input = mPipelineKernel->getInputScalarBinding(j);
422                val = b->getScalarField(input.getName());
423            } else { // output scalar of some kernel
424                Value * const outputScalars = mOutputScalars[i]; assert (outputScalars);
425                if (outputScalars->getType()->isAggregateType()) {
426                    val = b->CreateExtractValue(outputScalars, {j});
427                } else { assert (j == 0 && "scalar type is not an aggregate");
428                    val = outputScalars;
429                }
430            }
431            mScalarDependencyGraph[bufferVertex] = val;
432        }
433        const auto k = mScalarDependencyGraph[e];
434        assert (args[k] == nullptr);
435        args[k] = mScalarDependencyGraph[bufferVertex];
436    }
437}
438
439/** ------------------------------------------------------------------------------------------------------------- *
440 * @brief getFinalOutputScalars
441 ** ------------------------------------------------------------------------------------------------------------- */
442std::vector<Value *> PipelineCompiler::getFinalOutputScalars(BuilderRef b) {
443    const auto numOfKernels = mPipeline.size();
444    const auto & calls = mPipelineKernel->getCallBindings();
445    const auto numOfCalls = calls.size();
446    std::vector<Value *> args;
447    b->setKernel(mPipelineKernel);
448    for (unsigned k = 0; k < numOfCalls; ++k) {
449        writeOutputScalars(b, numOfKernels + k + 1, args);
450        Function * const f = cast<Function>(calls[k].Callee);
451        auto i = f->arg_begin();
452        for (auto j = args.begin(); j != args.end(); ++i, ++j) {
453            assert (i != f->arg_end());
454            *j = b->CreateZExtOrTrunc(*j, i->getType());
455        }
456        assert (i == f->arg_end());
457        b->CreateCall(f, args);
458    }
459    writeOutputScalars(b, numOfKernels, args);
460    return args;
461}
462
463/** ------------------------------------------------------------------------------------------------------------- *
464 * @brief initializeKernelCallPhis
465 ** ------------------------------------------------------------------------------------------------------------- */
466inline void PipelineCompiler::initializeKernelCallPhis(BuilderRef b) {
467    const auto numOfInputs = mKernel->getNumOfStreamInputs();
468    Type * const sizeTy = b->getSizeTy();
469    for (unsigned i = 0; i < numOfInputs; ++i) {
470        const Binding & input = mKernel->getInputStreamSetBinding(i);
471        const auto prefix = makeBufferName(mKernelIndex, input);
472        mLinearInputItemsPhi[i] = b->CreatePHI(sizeTy, 2, prefix + "_linearlyAccessible");
473    }
474    const auto numOfOutputs = mKernel->getNumOfStreamOutputs();
475    for (unsigned i = 0; i < numOfOutputs; ++i) {
476        if (LLVM_LIKELY(getOutputBufferType(i) != BufferType::Managed)) {
477            const Binding & output = mKernel->getOutputStreamSetBinding(i);
478            const auto prefix = makeBufferName(mKernelIndex, output);
479            mLinearOutputItemsPhi[i] = b->CreatePHI(sizeTy, 2, prefix + "_linearlyWritable");
480        }
481    }
482}
483
484/** ------------------------------------------------------------------------------------------------------------- *
485 * @brief initializeKernelExitPhis
486 ** ------------------------------------------------------------------------------------------------------------- */
487inline void PipelineCompiler::initializeKernelExitPhis(BuilderRef b) {
488    const auto kernelName = makeKernelName(mKernelIndex);
489    mTerminatedFlag = b->CreatePHI(b->getInt1Ty(), 2, kernelName + "_terminated");
490    mTerminatedFlag->addIncoming(b->getTrue(), mKernelEntry);
491    mTerminatedFlag->addIncoming(mTerminatedPhi, mKernelLoopExitPhiCatch);
492    mTerminationGraph[mKernelIndex] = mTerminatedFlag;
493    if (mPipelineProgress) {
494        PHINode * const pipelineProgress = b->CreatePHI(b->getInt1Ty(), 2, "pipelineProgress");
495        pipelineProgress->addIncoming(mPipelineProgress, mKernelEntry);
496        pipelineProgress->addIncoming(mHasProgressedPhi, mKernelLoopExitPhiCatch);
497        mPipelineProgress = pipelineProgress;
498    }
499    createConsumedPhiNodes(b);
500    createPopCountReferenceCounts(b);
501}
502
503/** ------------------------------------------------------------------------------------------------------------- *
504 * @brief getInitialTerminationSignal
505 ** ------------------------------------------------------------------------------------------------------------- */
506inline Value * PipelineCompiler::getInitialTerminationSignal(BuilderRef b) const {
507    b->setKernel(mPipelineKernel);
508    const auto prefix = makeKernelName(mKernelIndex);
509    Value * const terminated = b->getScalarField(prefix + TERMINATION_SIGNAL);
510    b->setKernel(mKernel);
511    return terminated;
512}
513
514/** ------------------------------------------------------------------------------------------------------------- *
515 * @brief setTerminated
516 ** ------------------------------------------------------------------------------------------------------------- */
517inline void PipelineCompiler::setTerminated(BuilderRef b, Value * const value) {
518    const auto prefix = makeKernelName(mKernelIndex);
519    b->setKernel(mPipelineKernel);
520    b->setScalarField(prefix + TERMINATION_SIGNAL, value);
521    #ifdef PRINT_DEBUG_MESSAGES
522    b->CallPrintInt("*** " + prefix + "_terminated ***", value);
523    #endif
524    b->setKernel(mKernel);
525}
526
527/** ------------------------------------------------------------------------------------------------------------- *
528 * @brief releaseCurrentSegment
529 ** ------------------------------------------------------------------------------------------------------------- */
530inline void PipelineCompiler::releaseCurrentSegment(BuilderRef b) {
531    Value * const nextSegNo = b->CreateAdd(mSegNo, b->getSize(1));
532    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableMProtect))) {
533        b->CreateMProtect(mKernel->getHandle(), CBuilder::Protect::WRITE);
534    }
535    b->releaseLogicalSegmentNo(nextSegNo);
536    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableMProtect))) {
537        b->CreateMProtect(mKernel->getHandle(), CBuilder::Protect::READ);
538    }
539}
540
541}
Note: See TracBrowser for help on using the repository browser.