source: icGREP/icgrep-devel/icgrep/kernels/kernel.cpp @ 5435

Last change on this file since 5435 was 5435, checked in by nmedfort, 2 years ago

Continued refactoring work.

File size: 42.2 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5
6#include "kernel.h"
7#include <toolchain/toolchain.h>
8#include <kernels/streamset.h>
9#include <llvm/IR/Constants.h>
10#include <llvm/IR/Function.h>
11#include <llvm/IR/Instructions.h>
12#include <llvm/IR/MDBuilder.h>
13#include <llvm/IR/Module.h>
14#include <llvm/Support/raw_ostream.h>
15#include <llvm/Bitcode/ReaderWriter.h>
16#include <llvm/Transforms/Utils/Local.h>
17#include <kernels/streamset.h>
18#include <sstream>
19
20using namespace llvm;
21using namespace parabix;
22
23namespace kernel {
24
25const std::string Kernel::DO_BLOCK_SUFFIX = "_DoBlock";
26const std::string Kernel::FINAL_BLOCK_SUFFIX = "_FinalBlock";
27const std::string Kernel::LOGICAL_SEGMENT_NO_SCALAR = "logicalSegNo";
28const std::string Kernel::PROCESSED_ITEM_COUNT_SUFFIX = "_processedItemCount";
29const std::string Kernel::CONSUMED_ITEM_COUNT_SUFFIX = "_consumedItemCount";
30const std::string Kernel::PRODUCED_ITEM_COUNT_SUFFIX = "_producedItemCount";
31const std::string Kernel::TERMINATION_SIGNAL = "terminationSignal";
32const std::string Kernel::BUFFER_PTR_SUFFIX = "_bufferPtr";
33const std::string Kernel::CONSUMER_SUFFIX = "_consumerLocks";
34
35unsigned Kernel::addScalar(Type * const type, const std::string & name) {
36    if (LLVM_UNLIKELY(mKernelStateType != nullptr)) {
37        report_fatal_error("Cannot add field " + name + " to " + getName() + " after kernel state finalized");
38    }
39    if (LLVM_UNLIKELY(mKernelMap.count(name))) {
40        report_fatal_error(getName() + " already contains scalar field " + name);
41    }
42    const auto index = mKernelFields.size();
43    mKernelMap.emplace(name, index);
44    mKernelFields.push_back(type);
45    return index;
46}
47
48unsigned Kernel::addUnnamedScalar(Type * const type) {
49    if (LLVM_UNLIKELY(mKernelStateType != nullptr)) {
50        report_fatal_error("Cannot add unnamed field  to " + getName() + " after kernel state finalized");
51    }
52    const auto index = mKernelFields.size();
53    mKernelFields.push_back(type);
54    return index;
55}
56
57void Kernel::prepareStreamSetNameMap() {
58    for (unsigned i = 0; i < mStreamSetInputs.size(); i++) {
59        mStreamMap.emplace(mStreamSetInputs[i].name, std::make_pair(Port::Input, i));
60    }
61    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
62        mStreamMap.emplace(mStreamSetOutputs[i].name, std::make_pair(Port::Output, i));
63    }
64}
65   
66void Kernel::prepareKernel() {
67    assert ("KernelBuilder does not have a valid IDISA Builder" && iBuilder);
68    if (LLVM_UNLIKELY(mKernelStateType != nullptr)) {
69        report_fatal_error("Cannot prepare kernel after kernel state finalized");
70    }
71    if (mStreamSetInputs.size() != mStreamSetInputBuffers.size()) {
72        std::string tmp;
73        raw_string_ostream out(tmp);
74        out << "kernel contains " << mStreamSetInputBuffers.size() << " input buffers for "
75            << mStreamSetInputs.size() << " input stream sets.";
76        report_fatal_error(out.str());
77    }
78    if (mStreamSetOutputs.size() != mStreamSetOutputBuffers.size()) {
79        std::string tmp;
80        raw_string_ostream out(tmp);
81        out << "kernel contains " << mStreamSetOutputBuffers.size() << " output buffers for "
82            << mStreamSetOutputs.size() << " output stream sets.";
83        report_fatal_error(out.str());
84    }
85    const auto blockSize = iBuilder->getBitBlockWidth();
86    for (unsigned i = 0; i < mStreamSetInputs.size(); i++) {
87        if ((mStreamSetInputBuffers[i]->getBufferBlocks() > 0) && (mStreamSetInputBuffers[i]->getBufferBlocks() < codegen::SegmentSize + (blockSize + mLookAheadPositions - 1)/blockSize)) {
88            report_fatal_error("Kernel preparation: Buffer size too small " + mStreamSetInputs[i].name);
89        }
90        mScalarInputs.emplace_back(mStreamSetInputBuffers[i]->getPointerType(), mStreamSetInputs[i].name + BUFFER_PTR_SUFFIX);
91        if ((i == 0) || !mStreamSetInputs[i].rate.isExact()) {
92            addScalar(iBuilder->getSizeTy(), mStreamSetInputs[i].name + PROCESSED_ITEM_COUNT_SUFFIX);
93        }       
94    }
95
96    IntegerType * const sizeTy = iBuilder->getSizeTy();
97    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
98        mScalarInputs.emplace_back(mStreamSetOutputBuffers[i]->getPointerType(), mStreamSetOutputs[i].name + BUFFER_PTR_SUFFIX);
99        if ((mStreamSetInputs.empty() && (i == 0)) || !mStreamSetOutputs[i].rate.isExact()) {
100            addScalar(sizeTy, mStreamSetOutputs[i].name + PRODUCED_ITEM_COUNT_SUFFIX);
101        }
102    }
103    for (const auto binding : mScalarInputs) {
104        addScalar(binding.type, binding.name);
105    }
106    for (const auto binding : mScalarOutputs) {
107        addScalar(binding.type, binding.name);
108    }
109    if (mStreamMap.empty()) {
110        prepareStreamSetNameMap();
111    }
112    for (auto binding : mInternalScalars) {
113        addScalar(binding.type, binding.name);
114    }
115
116    Type * const consumerSetTy = StructType::get(sizeTy, sizeTy->getPointerTo()->getPointerTo(), nullptr)->getPointerTo();
117    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
118        addScalar(consumerSetTy, mStreamSetOutputs[i].name + CONSUMER_SUFFIX);
119    }
120
121    addScalar(sizeTy, LOGICAL_SEGMENT_NO_SCALAR);
122    addScalar(iBuilder->getInt1Ty(), TERMINATION_SIGNAL);
123
124    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
125        addScalar(sizeTy, mStreamSetOutputs[i].name + CONSUMED_ITEM_COUNT_SUFFIX);
126    }
127
128    mKernelStateType = StructType::create(iBuilder->getContext(), mKernelFields, getName());
129}
130
131void Kernel::createKernelStub(const StreamSetBuffers & inputs, const StreamSetBuffers & outputs) {
132    assert ("KernelBuilder does not have a valid IDISA Builder" && iBuilder);
133    assert ("IDISA Builder does not have a valid Module" && iBuilder->getModule());
134    std::stringstream cacheName;   
135    cacheName << getName() << '_' << iBuilder->getBuilderUniqueName();
136    for (const StreamSetBuffer * b: inputs) {
137        cacheName <<  ':' <<  b->getUniqueID();
138    }
139    for (const StreamSetBuffer * b: outputs) {
140        cacheName <<  ':' <<  b->getUniqueID();
141    }
142    Module * const kernelModule = new Module(cacheName.str(), iBuilder->getContext());
143    kernelModule->setTargetTriple(iBuilder->getModule()->getTargetTriple());
144    createKernelStub(inputs, outputs, kernelModule);
145}
146
147void Kernel::createKernelStub(const StreamSetBuffers & inputs, const StreamSetBuffers & outputs, Module * const kernelModule) {
148    assert (mModule == nullptr);
149    assert ("KernelBuilder does not have a valid IDISA Builder" && iBuilder);
150    assert (mStreamSetInputBuffers.empty());
151    assert (mStreamSetOutputBuffers.empty());
152
153    if (LLVM_UNLIKELY(mStreamSetInputs.size() != inputs.size())) {
154        report_fatal_error(getName() + ": expected " + std::to_string(mStreamSetInputs.size()) +
155                           " input stream sets but was given "
156                           + std::to_string(inputs.size()));
157    }
158
159    for (unsigned i = 0; i < inputs.size(); ++i) {
160        StreamSetBuffer * const buf = inputs[i];
161        if (LLVM_UNLIKELY(buf == nullptr)) {
162            report_fatal_error(getName() + ": input stream set " + std::to_string(i)
163                               + " cannot be null");
164        }
165        buf->addConsumer(this);
166    }
167
168    if (LLVM_UNLIKELY(mStreamSetOutputs.size() != outputs.size())) {
169        report_fatal_error(getName() + ": expected " + std::to_string(mStreamSetOutputs.size())
170                           + " output stream sets but was given "
171                           + std::to_string(outputs.size()));
172    }
173
174    for (unsigned i = 0; i < outputs.size(); ++i) {
175        StreamSetBuffer * const buf = outputs[i];
176        if (LLVM_UNLIKELY(buf == nullptr)) {
177            report_fatal_error(getName() + ": output stream set " + std::to_string(i) + " cannot be null");
178        }
179        if (LLVM_LIKELY(buf->getProducer() == nullptr)) {
180            buf->setProducer(this);
181        } else {
182            report_fatal_error(getName() + ": output stream set " + std::to_string(i)
183                               + " is already produced by kernel " + buf->getProducer()->getName());
184        }
185    }
186
187    mModule = kernelModule;
188
189    mStreamSetInputBuffers.assign(inputs.begin(), inputs.end());
190    mStreamSetOutputBuffers.assign(outputs.begin(), outputs.end());
191
192    prepareKernel();
193}
194
195
196// Default kernel signature: generate the IR and emit as byte code.
197std::string Kernel::makeSignature() {
198    assert ("KernelBuilder does not have a valid IDISA Builder" && iBuilder);
199    if (LLVM_LIKELY(moduleIDisSignature())) {
200        return getModule()->getModuleIdentifier();
201    } else {
202        generateKernel();
203        std::string signature;
204        raw_string_ostream OS(signature);
205        WriteBitcodeToFile(getModule(), OS);
206        return signature;
207    }
208}
209
210void Kernel::generateKernel() {
211    assert ("KernelBuilder does not have a valid IDISA Builder" && iBuilder);
212    // If the module id cannot uniquely identify this kernel, "generateKernelSignature()" will have already
213    // generated the unoptimized IR.
214    if (!mIsGenerated) {
215        auto ip = iBuilder->saveIP();
216        auto saveInstance = getInstance();
217        addKernelDeclarations();
218        callGenerateInitializeMethod();
219        callGenerateDoSegmentMethod();       
220        callGenerateFinalizeMethod();
221        setInstance(saveInstance);
222        iBuilder->restoreIP(ip);
223        mIsGenerated = true;
224    }
225}
226
227inline void Kernel::callGenerateInitializeMethod() {
228    mCurrentMethod = getInitFunction(iBuilder->getModule());
229    iBuilder->SetInsertPoint(CreateBasicBlock("entry"));
230    Function::arg_iterator args = mCurrentMethod->arg_begin();
231    setInstance(&*(args++));
232    iBuilder->CreateStore(ConstantAggregateZero::get(mKernelStateType), getInstance());
233    for (auto binding : mScalarInputs) {
234        setScalarField(binding.name, &*(args++));
235    }
236    for (auto binding : mStreamSetOutputs) {
237        setConsumerLock(binding.name, &*(args++));
238    }
239    generateInitializeMethod();
240    iBuilder->CreateRetVoid();
241}
242
243inline void Kernel::callGenerateDoSegmentMethod() {
244    mCurrentMethod = getDoSegmentFunction(iBuilder->getModule());
245    BasicBlock * const entry = CreateBasicBlock(getName() + "_entry");
246    iBuilder->SetInsertPoint(entry);
247    auto args = mCurrentMethod->arg_begin();
248    setInstance(&*(args++));
249    mIsFinal = &*(args++);
250    const auto n = mStreamSetInputs.size();
251    mAvailableItemCount.resize(n, nullptr);
252    for (unsigned i = 0; i < mStreamSetInputs.size(); i++) {
253        mAvailableItemCount[i] = &*(args++);
254    }
255    generateDoSegmentMethod(); // must be overridden by the KernelBuilder subtype
256    mIsFinal = nullptr;
257    mAvailableItemCount.clear();
258    iBuilder->CreateRetVoid();
259}
260
261inline void Kernel::callGenerateFinalizeMethod() {
262    mCurrentMethod = getTerminateFunction(iBuilder->getModule());
263    iBuilder->SetInsertPoint(CreateBasicBlock("entry"));
264    auto args = mCurrentMethod->arg_begin();
265    setInstance(&*(args++));
266    generateFinalizeMethod(); // may be overridden by the KernelBuilder subtype
267    const auto n = mScalarOutputs.size();
268    if (n == 0) {
269        iBuilder->CreateRetVoid();
270    } else {
271        Value * outputs[n];
272        for (unsigned i = 0; i < n; ++i) {
273            outputs[i] = getScalarField(mScalarOutputs[i].name);
274        }
275        if (n == 1) {
276            iBuilder->CreateRet(outputs[0]);
277        } else {
278            iBuilder->CreateAggregateRet(outputs, n);
279        }
280    }
281}
282
283unsigned Kernel::getScalarIndex(const std::string & name) const {
284    assert ("getScalarIndex was given a null IDISA Builder" && iBuilder);
285    const auto f = mKernelMap.find(name);
286    if (LLVM_UNLIKELY(f == mKernelMap.end())) {
287        report_fatal_error(getName() + " does not contain scalar: " + name);
288    }
289    return f->second;
290}
291
292Value * Kernel::getProducedItemCount(const std::string & name, Value * doFinal) const {
293    Port port; unsigned ssIdx;
294    std::tie(port, ssIdx) = getStreamPort(name);
295    assert (port == Port::Output);
296    if (mStreamSetOutputs[ssIdx].rate.isExact()) {
297        std::string refSet = mStreamSetOutputs[ssIdx].rate.referenceStreamSet();
298        std::string principalField;
299        if (refSet.empty()) {
300            if (mStreamSetInputs.empty()) {
301                principalField = mStreamSetOutputs[0].name + PRODUCED_ITEM_COUNT_SUFFIX;
302            } else {
303                principalField = mStreamSetInputs[0].name + PROCESSED_ITEM_COUNT_SUFFIX;
304            }
305        } else {
306            Port port; unsigned pfIndex;
307            std::tie(port, pfIndex) = getStreamPort(refSet);
308            if (port == Port::Input) {
309               principalField = refSet + PROCESSED_ITEM_COUNT_SUFFIX;
310            } else {
311               principalField = refSet + PRODUCED_ITEM_COUNT_SUFFIX;
312            }
313        }
314        Value * principalItemsProcessed = getScalarField(principalField);
315        return mStreamSetOutputs[ssIdx].rate.CreateRatioCalculation(iBuilder, principalItemsProcessed, doFinal);
316    }
317    return getScalarField(name + PRODUCED_ITEM_COUNT_SUFFIX);
318}
319
320llvm::Value * Kernel::getAvailableItemCount(const std::string & name) const {
321    for (unsigned i = 0; i < mStreamSetInputs.size(); ++i) {
322        if (mStreamSetInputs[i].name == name) {
323            return mAvailableItemCount[i];
324        }
325    }
326    return nullptr;
327}
328
329Value * Kernel::getProcessedItemCount(const std::string & name) const {
330    Port port; unsigned ssIdx;
331    std::tie(port, ssIdx) = getStreamPort(name);
332    assert (port == Port::Input);
333    if (mStreamSetInputs[ssIdx].rate.isExact()) {
334        std::string refSet = mStreamSetInputs[ssIdx].rate.referenceStreamSet();
335        if (refSet.empty()) {
336            refSet = mStreamSetInputs[0].name;
337        }
338        Value * principalItemsProcessed = getScalarField(refSet + PROCESSED_ITEM_COUNT_SUFFIX);
339        return mStreamSetInputs[ssIdx].rate.CreateRatioCalculation(iBuilder, principalItemsProcessed);
340    }
341    return getScalarField(name + PROCESSED_ITEM_COUNT_SUFFIX);
342}
343
344Value * Kernel::getConsumedItemCount(const std::string & name) const {
345    return getScalarField(name + CONSUMED_ITEM_COUNT_SUFFIX);
346}
347
348void Kernel::setProducedItemCount(const std::string & name, Value * value) const {
349    setScalarField(name + PRODUCED_ITEM_COUNT_SUFFIX, value);
350}
351
352void Kernel::setProcessedItemCount(const std::string & name, Value * value) const {
353    setScalarField(name + PROCESSED_ITEM_COUNT_SUFFIX, value);
354}
355
356void Kernel::setConsumedItemCount(const std::string & name, Value * value) const {
357    setScalarField(name + CONSUMED_ITEM_COUNT_SUFFIX, value);
358}
359
360Value * Kernel::getTerminationSignal() const {
361    return getScalarField(TERMINATION_SIGNAL);
362}
363
364void Kernel::setTerminationSignal() const {
365    setScalarField(TERMINATION_SIGNAL, iBuilder->getTrue());
366}
367
368LoadInst * Kernel::acquireLogicalSegmentNo() const {
369    assert (iBuilder);
370    return iBuilder->CreateAtomicLoadAcquire(getScalarFieldPtr(LOGICAL_SEGMENT_NO_SCALAR));
371}
372
373void Kernel::releaseLogicalSegmentNo(Value * nextSegNo) const {
374    iBuilder->CreateAtomicStoreRelease(nextSegNo, getScalarFieldPtr(LOGICAL_SEGMENT_NO_SCALAR));
375}
376
377llvm::Value * Kernel::getLinearlyAccessibleItems(const std::string & name, llvm::Value * fromPosition) const {
378    llvm::Value * instance = getStreamSetBufferPtr(name);
379    const StreamSetBuffer * const buf = getInputStreamSetBuffer(name);
380    return buf->getLinearlyAccessibleItems(iBuilder, instance, fromPosition);
381}
382
383llvm::Value * Kernel::getConsumerLock(const std::string & name) const {
384    return getScalarField(name + CONSUMER_SUFFIX);
385}
386
387void Kernel::setConsumerLock(const std::string & name, llvm::Value * value) const {
388    setScalarField(name + CONSUMER_SUFFIX, value);
389}
390
391inline Value * Kernel::computeBlockIndex(const std::vector<Binding> & bindings, const std::string & name, Value * itemCount) const {
392    for (const Binding & b : bindings) {
393        if (b.name == name) {
394            const auto divisor = iBuilder->getBitBlockWidth();
395            if (LLVM_LIKELY((divisor & (divisor - 1)) == 0)) {
396                return iBuilder->CreateLShr(itemCount, std::log2(divisor));
397            } else {
398                return iBuilder->CreateUDiv(itemCount, iBuilder->getSize(divisor));
399            }
400        }
401    }
402    report_fatal_error("Error: no binding in " + getName() + " for " + name);
403}
404
405Value * Kernel::getInputStreamBlockPtr(const std::string & name, Value * streamIndex) const {
406    Value * const blockIndex = computeBlockIndex(mStreamSetInputs, name, getProcessedItemCount(name));
407    const StreamSetBuffer * const buf = getInputStreamSetBuffer(name);
408    return buf->getStreamBlockPtr(iBuilder, getStreamSetBufferPtr(name), streamIndex, blockIndex, true);
409}
410
411Value * Kernel::loadInputStreamBlock(const std::string & name, Value * streamIndex) const {
412    return iBuilder->CreateBlockAlignedLoad(getInputStreamBlockPtr(name, streamIndex));
413}
414
415Value * Kernel::getInputStreamPackPtr(const std::string & name, Value * streamIndex, Value * packIndex) const {
416    Value * const blockIndex = computeBlockIndex(mStreamSetInputs, name, getProcessedItemCount(name));
417    const StreamSetBuffer * const buf = getInputStreamSetBuffer(name);
418    return buf->getStreamPackPtr(iBuilder, getStreamSetBufferPtr(name), streamIndex, blockIndex, packIndex, true);
419}
420
421Value * Kernel::loadInputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex) const {
422    return iBuilder->CreateBlockAlignedLoad(getInputStreamPackPtr(name, streamIndex, packIndex));
423}
424
425llvm::Value * Kernel::getInputStreamSetCount(const std::string & name) const {
426    return getInputStreamSetBuffer(name)->getStreamSetCount(iBuilder, getStreamSetBufferPtr(name));
427}
428
429llvm::Value * Kernel::getAdjustedInputStreamBlockPtr(Value * blockAdjustment, const std::string & name, llvm::Value * streamIndex) const {
430    Value * blockIndex = computeBlockIndex(mStreamSetInputs, name, getProcessedItemCount(name));
431    blockIndex = iBuilder->CreateAdd(blockIndex, blockAdjustment);
432    const StreamSetBuffer * const buf = getInputStreamSetBuffer(name);
433    return buf->getStreamBlockPtr(iBuilder, getStreamSetBufferPtr(name), streamIndex, blockIndex, true);
434}
435
436Value * Kernel::getOutputStreamBlockPtr(const std::string & name, Value * streamIndex) const {
437    Value * const blockIndex = computeBlockIndex(mStreamSetOutputs, name, getProducedItemCount(name));
438    const StreamSetBuffer * const buf = getOutputStreamSetBuffer(name);
439    return buf->getStreamBlockPtr(iBuilder, getStreamSetBufferPtr(name), streamIndex, blockIndex, false);
440}
441
442void Kernel::storeOutputStreamBlock(const std::string & name, Value * streamIndex, Value * toStore) const {
443    return iBuilder->CreateBlockAlignedStore(toStore, getOutputStreamBlockPtr(name, streamIndex));
444}
445
446Value * Kernel::getOutputStreamPackPtr(const std::string & name, Value * streamIndex, Value * packIndex) const {
447    Value * const blockIndex = computeBlockIndex(mStreamSetOutputs, name, getProducedItemCount(name));
448    const StreamSetBuffer * const buf = getOutputStreamSetBuffer(name);
449    return buf->getStreamPackPtr(iBuilder, getStreamSetBufferPtr(name), streamIndex, blockIndex, packIndex, false);
450}
451
452void Kernel::storeOutputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex, Value * toStore) const {
453    return iBuilder->CreateBlockAlignedStore(toStore, getOutputStreamPackPtr(name, streamIndex, packIndex));
454}
455
456llvm::Value * Kernel::getOutputStreamSetCount(const std::string & name) const {
457    return getOutputStreamSetBuffer(name)->getStreamSetCount(iBuilder, getStreamSetBufferPtr(name));
458}
459
460Value * Kernel::getRawInputPointer(const std::string & name, Value * streamIndex, Value * absolutePosition) const {
461    return getInputStreamSetBuffer(name)->getRawItemPointer(iBuilder, getStreamSetBufferPtr(name), streamIndex, absolutePosition);
462}
463
464Value * Kernel::getRawOutputPointer(const std::string & name, Value * streamIndex, Value * absolutePosition) const {
465    return getOutputStreamSetBuffer(name)->getRawItemPointer(iBuilder, getStreamSetBufferPtr(name), streamIndex, absolutePosition);
466}
467
468Value * Kernel::getBaseAddress(const std::string & name) const {
469    return getAnyStreamSetBuffer(name)->getBaseAddress(iBuilder, getStreamSetBufferPtr(name));
470}
471
472void Kernel::setBaseAddress(const std::string & name, Value * const addr) const {
473    return getAnyStreamSetBuffer(name)->setBaseAddress(iBuilder, getStreamSetBufferPtr(name), addr);
474}
475
476Value * Kernel::getBufferedSize(const std::string & name) const {
477    return getAnyStreamSetBuffer(name)->getBufferedSize(iBuilder, getStreamSetBufferPtr(name));
478}
479
480void Kernel::setBufferedSize(const std::string & name, Value * size) const {
481    unsigned index; Port port;
482    std::tie(port, index) = getStreamPort(name);
483    const StreamSetBuffer * buf = nullptr;
484    if (port == Port::Input) {
485        assert (index < mStreamSetInputBuffers.size());
486        buf = mStreamSetInputBuffers[index];
487    } else {
488        assert (index < mStreamSetOutputBuffers.size());
489        buf = mStreamSetOutputBuffers[index];
490    }
491    buf->setBufferedSize(iBuilder, getStreamSetBufferPtr(name), size);
492}
493
494BasicBlock * Kernel::CreateWaitForConsumers() const {
495
496    const auto consumers = getStreamOutputs();
497    BasicBlock * const entry = iBuilder->GetInsertBlock();
498    if (consumers.empty()) {
499        return entry;
500    } else {
501        Function * const parent = entry->getParent();
502        IntegerType * const sizeTy = iBuilder->getSizeTy();
503        ConstantInt * const zero = iBuilder->getInt32(0);
504        ConstantInt * const one = iBuilder->getInt32(1);
505        ConstantInt * const size0 = iBuilder->getSize(0);
506
507        Value * const segNo = acquireLogicalSegmentNo();
508        const auto n = consumers.size();
509        BasicBlock * load[n + 1];
510        BasicBlock * wait[n];
511        for (unsigned i = 0; i < n; ++i) {
512            load[i] = BasicBlock::Create(iBuilder->getContext(), consumers[i].name + "Load", parent);
513            wait[i] = BasicBlock::Create(iBuilder->getContext(), consumers[i].name + "Wait", parent);
514        }
515        load[n] = BasicBlock::Create(iBuilder->getContext(), "Resume", parent);
516        iBuilder->CreateBr(load[0]);
517        for (unsigned i = 0; i < n; ++i) {
518
519            iBuilder->SetInsertPoint(load[i]);
520            Value * const outputConsumers = getConsumerLock(consumers[i].name);
521
522            Value * const consumerCount = iBuilder->CreateLoad(iBuilder->CreateGEP(outputConsumers, {zero, zero}));
523            Value * const consumerPtr = iBuilder->CreateLoad(iBuilder->CreateGEP(outputConsumers, {zero, one}));
524            Value * const noConsumers = iBuilder->CreateICmpEQ(consumerCount, size0);
525            iBuilder->CreateUnlikelyCondBr(noConsumers, load[i + 1], wait[i]);
526
527            iBuilder->SetInsertPoint(wait[i]);
528            PHINode * const consumerPhi = iBuilder->CreatePHI(sizeTy, 2);
529            consumerPhi->addIncoming(size0, load[i]);
530
531            Value * const conSegPtr = iBuilder->CreateLoad(iBuilder->CreateGEP(consumerPtr, consumerPhi));
532            Value * const processedSegmentCount = iBuilder->CreateAtomicLoadAcquire(conSegPtr);
533            Value * const ready = iBuilder->CreateICmpEQ(segNo, processedSegmentCount);
534            assert (ready->getType() == iBuilder->getInt1Ty());
535            Value * const nextConsumerIdx = iBuilder->CreateAdd(consumerPhi, iBuilder->CreateZExt(ready, sizeTy));
536            consumerPhi->addIncoming(nextConsumerIdx, wait[i]);
537            Value * const next = iBuilder->CreateICmpEQ(nextConsumerIdx, consumerCount);
538            iBuilder->CreateCondBr(next, load[i + 1], wait[i]);
539        }
540
541        BasicBlock * const exit = load[n];
542        iBuilder->SetInsertPoint(exit);
543        return exit;
544    }
545
546}
547
548Value * Kernel::getStreamSetBufferPtr(const std::string & name) const {
549    return getScalarField(name + BUFFER_PTR_SUFFIX);
550}
551
552Argument * Kernel::getParameter(Function * const f, const std::string & name) const {
553    for (auto & arg : f->getArgumentList()) {
554        if (arg.getName().equals(name)) {
555            return &arg;
556        }
557    }
558    report_fatal_error(getName() + " does not have parameter " + name);
559}
560
561CallInst * Kernel::createDoSegmentCall(const std::vector<Value *> & args) const {
562    Function * const doSegment = getDoSegmentFunction(iBuilder->getModule());
563    assert (doSegment->getArgumentList().size() == args.size());
564    return iBuilder->CreateCall(doSegment, args);
565}
566
567Value * Kernel::getAccumulator(const std::string & accumName) const {
568    assert ("KernelBuilder does not have a valid IDISA Builder" && iBuilder);
569    if (LLVM_UNLIKELY(mOutputScalarResult == nullptr)) {
570        report_fatal_error("Cannot get accumulator " + accumName + " until " + getName() + " has terminated.");
571    }
572    const auto n = mScalarOutputs.size();
573    if (LLVM_UNLIKELY(n == 0)) {
574        report_fatal_error(getName() + " has no output scalars.");
575    } else {
576        for (unsigned i = 0; i < n; ++i) {
577            const Binding & b = mScalarOutputs[i];
578            if (b.name == accumName) {
579                if (n == 1) {
580                    return mOutputScalarResult;
581                } else {
582                    return iBuilder->CreateExtractValue(mOutputScalarResult, {i});
583                }
584            }
585        }
586        report_fatal_error(getName() + " has no output scalar named " + accumName);
587    }
588}
589
590BasicBlock * Kernel::CreateBasicBlock(std::string && name) const {
591    return BasicBlock::Create(iBuilder->getContext(), name, mCurrentMethod);
592}
593
594Value * Kernel::createInstance() {
595    assert ("KernelBuilder does not have a valid IDISA Builder" && iBuilder);
596    if (LLVM_UNLIKELY(mKernelStateType == nullptr)) {
597        report_fatal_error("Cannot instantiate " + getName() + " before calling prepareKernel()");
598    }
599    setInstance(iBuilder->CreateCacheAlignedAlloca(mKernelStateType));
600    return getInstance();
601}
602
603void Kernel::initializeInstance() {
604    assert ("KernelBuilder does not have a valid IDISA Builder" && iBuilder);
605    if (LLVM_UNLIKELY(getInstance() == nullptr)) {
606        report_fatal_error("Cannot initialize " + getName() + " before calling createInstance()");
607    }
608    std::vector<Value *> args;
609    args.reserve(1 + mInitialArguments.size() + mStreamSetInputBuffers.size() + (mStreamSetOutputBuffers.size() * 2));
610    args.push_back(getInstance());
611    for (unsigned i = 0; i < mInitialArguments.size(); ++i) {
612        Value * arg = mInitialArguments[i];
613        if (LLVM_UNLIKELY(arg == nullptr)) {
614            report_fatal_error(getName() + ": initial argument " + std::to_string(i)
615                               + " cannot be null when calling createInstance()");
616        }
617        args.push_back(arg);
618    }
619    for (unsigned i = 0; i < mStreamSetInputBuffers.size(); ++i) {
620        assert (mStreamSetInputBuffers[i]);
621        Value * arg = mStreamSetInputBuffers[i]->getStreamSetBasePtr();
622        if (LLVM_UNLIKELY(arg == nullptr)) {
623            report_fatal_error(getName() + ": input stream set " + std::to_string(i)
624                               + " was not allocated prior to calling createInstance()");
625        }
626        args.push_back(arg);
627    }
628    assert (mStreamSetInputs.size() == mStreamSetInputBuffers.size());
629    for (unsigned i = 0; i < mStreamSetOutputBuffers.size(); ++i) {
630        assert (mStreamSetOutputBuffers[i]);
631        Value * arg = mStreamSetOutputBuffers[i]->getStreamSetBasePtr();
632        if (LLVM_UNLIKELY(arg == nullptr)) {
633            report_fatal_error(getName() + ": output stream set " + std::to_string(i)
634                               + " was not allocated prior to calling createInstance()");
635        }
636        args.push_back(arg);
637    }
638    assert (mStreamSetOutputs.size() == mStreamSetOutputBuffers.size());
639    IntegerType * const sizeTy = iBuilder->getSizeTy();
640    PointerType * const sizePtrTy = sizeTy->getPointerTo();
641    PointerType * const sizePtrPtrTy = sizePtrTy->getPointerTo();
642    StructType * const consumerTy = StructType::get(sizeTy, sizePtrPtrTy, nullptr);
643    for (unsigned i = 0; i < mStreamSetOutputBuffers.size(); ++i) {
644        const auto output = mStreamSetOutputBuffers[i];
645        const auto & consumers = output->getConsumers();
646        const auto n = consumers.size();
647        AllocaInst * const outputConsumers = iBuilder->CreateAlloca(consumerTy);
648        Value * const consumerSegNoArray = iBuilder->CreateAlloca(ArrayType::get(sizePtrTy, n));
649        for (unsigned i = 0; i < n; ++i) {
650            Kernel * const consumer = consumers[i];
651            assert ("all instances must be created prior to initialization of any instance" && consumer->getInstance());
652            consumer->setBuilder(iBuilder);
653            Value * const segmentNoPtr = consumer->getScalarFieldPtr(LOGICAL_SEGMENT_NO_SCALAR);
654            iBuilder->CreateStore(segmentNoPtr, iBuilder->CreateGEP(consumerSegNoArray, { iBuilder->getInt32(0), iBuilder->getInt32(i) }));
655        }
656        Value * const consumerCountPtr = iBuilder->CreateGEP(outputConsumers, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
657        iBuilder->CreateStore(iBuilder->getSize(n), consumerCountPtr);
658        Value * const consumerSegNoArrayPtr = iBuilder->CreateGEP(outputConsumers, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
659        iBuilder->CreateStore(iBuilder->CreatePointerCast(consumerSegNoArray, sizePtrPtrTy), consumerSegNoArrayPtr);
660        args.push_back(outputConsumers);
661    }
662
663    iBuilder->CreateCall(getInitFunction(iBuilder->getModule()), args);
664}
665
666//  The default doSegment method dispatches to the doBlock routine for
667//  each block of the given number of blocksToDo, and then updates counts.
668
669void BlockOrientedKernel::generateDoSegmentMethod() {   
670    BasicBlock * const entryBlock = iBuilder->GetInsertBlock();
671    BasicBlock * const strideLoopCond = CreateBasicBlock(getName() + "_strideLoopCond");
672    mStrideLoopBody = CreateBasicBlock(getName() + "_strideLoopBody");
673    BasicBlock * const stridesDone = CreateBasicBlock(getName() + "_stridesDone");
674    BasicBlock * const doFinalBlock = CreateBasicBlock(getName() + "_doFinalBlock");
675    BasicBlock * const segmentDone = CreateBasicBlock(getName() + "_segmentDone");
676
677    Value * baseTarget = nullptr;
678    if (useIndirectBr()) {
679        baseTarget = iBuilder->CreateSelect(mIsFinal, BlockAddress::get(doFinalBlock), BlockAddress::get(segmentDone));
680    }
681
682    ConstantInt * stride = iBuilder->getSize(iBuilder->getStride());
683    Value * availablePos = mAvailableItemCount[0];
684    Value * processed = getProcessedItemCount(mStreamSetInputs[0].name);
685    Value * itemsAvail = iBuilder->CreateSub(availablePos, processed);
686    Value * stridesToDo = iBuilder->CreateUDiv(itemsAvail, stride);
687
688    iBuilder->CreateBr(strideLoopCond);
689
690    iBuilder->SetInsertPoint(strideLoopCond);
691
692    PHINode * branchTarget = nullptr;
693    if (useIndirectBr()) {
694        branchTarget = iBuilder->CreatePHI(baseTarget->getType(), 2, "branchTarget");
695        branchTarget->addIncoming(baseTarget, entryBlock);
696    }
697
698    PHINode * const stridesRemaining = iBuilder->CreatePHI(iBuilder->getSizeTy(), 2, "stridesRemaining");
699    stridesRemaining->addIncoming(stridesToDo, entryBlock);
700    // NOTE: stridesRemaining may go to a negative number in the final block if the generateFinalBlockMethod(...)
701    // calls CreateDoBlockMethodCall(). Do *not* replace the comparator with an unsigned one!
702    Value * notDone = iBuilder->CreateICmpSGT(stridesRemaining, iBuilder->getSize(0));
703    iBuilder->CreateLikelyCondBr(notDone, mStrideLoopBody, stridesDone);
704
705    iBuilder->SetInsertPoint(mStrideLoopBody);
706
707    if (useIndirectBr()) {
708        mStrideLoopTarget = iBuilder->CreatePHI(baseTarget->getType(), 2, "strideTarget");
709        mStrideLoopTarget->addIncoming(branchTarget, strideLoopCond);
710    }
711
712    /// GENERATE DO BLOCK METHOD
713
714    writeDoBlockMethod();
715
716    /// UPDATE PROCESSED COUNTS
717
718    processed = getProcessedItemCount(mStreamSetInputs[0].name);
719    Value * itemsDone = iBuilder->CreateAdd(processed, stride);
720    setProcessedItemCount(mStreamSetInputs[0].name, itemsDone);
721
722    stridesRemaining->addIncoming(iBuilder->CreateSub(stridesRemaining, iBuilder->getSize(1)), iBuilder->GetInsertBlock());
723
724    BasicBlock * bodyEnd = iBuilder->GetInsertBlock();
725    if (useIndirectBr()) {
726        branchTarget->addIncoming(mStrideLoopTarget, bodyEnd);
727    }
728    iBuilder->CreateBr(strideLoopCond);
729
730    stridesDone->moveAfter(bodyEnd);
731
732    iBuilder->SetInsertPoint(stridesDone);
733
734    // Now conditionally perform the final block processing depending on the doFinal parameter.
735    if (useIndirectBr()) {
736        mStrideLoopBranch = iBuilder->CreateIndirectBr(branchTarget, 3);
737        mStrideLoopBranch->addDestination(doFinalBlock);
738        mStrideLoopBranch->addDestination(segmentDone);
739    } else {
740        iBuilder->CreateUnlikelyCondBr(mIsFinal, doFinalBlock, segmentDone);
741    }
742
743    doFinalBlock->moveAfter(stridesDone);
744
745    iBuilder->SetInsertPoint(doFinalBlock);
746
747    Value * remainingItems = iBuilder->CreateSub(mAvailableItemCount[0], getProcessedItemCount(mStreamSetInputs[0].name));
748    writeFinalBlockMethod(remainingItems);
749
750    itemsDone = mAvailableItemCount[0];
751    setProcessedItemCount(mStreamSetInputs[0].name, itemsDone);
752    setTerminationSignal();
753    iBuilder->CreateBr(segmentDone);
754
755    segmentDone->moveAfter(iBuilder->GetInsertBlock());
756
757    iBuilder->SetInsertPoint(segmentDone);
758
759    // Update the branch prediction metadata to indicate that the likely target will be segmentDone
760    if (useIndirectBr()) {
761        MDBuilder mdb(iBuilder->getContext());
762        const auto destinations = mStrideLoopBranch->getNumDestinations();
763        uint32_t weights[destinations];
764        for (unsigned i = 0; i < destinations; ++i) {
765            weights[i] = (mStrideLoopBranch->getDestination(i) == segmentDone) ? 100 : 1;
766        }
767        ArrayRef<uint32_t> bw(weights, destinations);
768        mStrideLoopBranch->setMetadata(LLVMContext::MD_prof, mdb.createBranchWeights(bw));
769    }
770
771}
772
773inline void BlockOrientedKernel::writeDoBlockMethod() {
774
775    Value * const self = getInstance();
776    Function * const cp = mCurrentMethod;
777    auto ip = iBuilder->saveIP();
778
779    /// Check if the do block method is called and create the function if necessary   
780    if (!useIndirectBr()) {
781        FunctionType * const type = FunctionType::get(iBuilder->getVoidTy(), {self->getType()}, false);
782        mCurrentMethod = Function::Create(type, GlobalValue::InternalLinkage, getName() + DO_BLOCK_SUFFIX, iBuilder->getModule());
783        mCurrentMethod->setCallingConv(CallingConv::C);
784        mCurrentMethod->setDoesNotThrow();
785        mCurrentMethod->setDoesNotCapture(1);
786        auto args = mCurrentMethod->arg_begin();
787        args->setName("self");
788        setInstance(&*args);
789        iBuilder->SetInsertPoint(CreateBasicBlock("entry"));
790    }
791
792    std::vector<Value *> priorProduced;
793    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
794        if (isa<CircularCopybackBuffer>(mStreamSetOutputBuffers[i]) || isa<SwizzledCopybackBuffer>(mStreamSetOutputBuffers[i]))  {
795            priorProduced.push_back(getProducedItemCount(mStreamSetOutputs[i].name));
796        }
797    }
798
799    generateDoBlockMethod(); // must be implemented by the BlockOrientedKernelBuilder subtype
800
801    unsigned priorIdx = 0;
802    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
803        Value * log2BlockSize = iBuilder->getSize(std::log2(iBuilder->getBitBlockWidth()));
804        if (SwizzledCopybackBuffer * const cb = dyn_cast<SwizzledCopybackBuffer>(mStreamSetOutputBuffers[i]))  {
805            BasicBlock * copyBack = CreateBasicBlock(mStreamSetOutputs[i].name + "_copyBack");
806            BasicBlock * done = CreateBasicBlock(mStreamSetOutputs[i].name + "_copyBackDone");
807            Value * newlyProduced = iBuilder->CreateSub(getProducedItemCount(mStreamSetOutputs[i].name), priorProduced[priorIdx]);
808            Value * priorBlock = iBuilder->CreateLShr(priorProduced[priorIdx], log2BlockSize);
809            Value * priorOffset = iBuilder->CreateAnd(priorProduced[priorIdx], iBuilder->getSize(iBuilder->getBitBlockWidth() - 1));
810            Value * instance = getStreamSetBufferPtr(mStreamSetOutputs[i].name);
811            Value * accessibleBlocks = cb->getLinearlyAccessibleBlocks(iBuilder, instance, priorBlock);
812            Value * accessible = iBuilder->CreateSub(iBuilder->CreateShl(accessibleBlocks, log2BlockSize), priorOffset);
813            Value * wraparound = iBuilder->CreateICmpULT(accessible, newlyProduced);
814            iBuilder->CreateCondBr(wraparound, copyBack, done);
815            iBuilder->SetInsertPoint(copyBack);
816            Value * copyItems = iBuilder->CreateSub(newlyProduced, accessible);
817            cb->createCopyBack(iBuilder, instance, copyItems);
818            iBuilder->CreateBr(done);
819            iBuilder->SetInsertPoint(done);
820            priorIdx++;
821        }
822        if (CircularCopybackBuffer * const cb = dyn_cast<CircularCopybackBuffer>(mStreamSetOutputBuffers[i]))  {
823            BasicBlock * copyBack = CreateBasicBlock(mStreamSetOutputs[i].name + "_copyBack");
824            BasicBlock * done = CreateBasicBlock(mStreamSetOutputs[i].name + "_copyBackDone");
825            Value * instance = getStreamSetBufferPtr(mStreamSetOutputs[i].name);
826            Value * newlyProduced = iBuilder->CreateSub(getProducedItemCount(mStreamSetOutputs[i].name), priorProduced[priorIdx]);
827            Value * accessible = cb->getLinearlyAccessibleItems(iBuilder, instance, priorProduced[priorIdx]);
828            Value * wraparound = iBuilder->CreateICmpULT(accessible, newlyProduced);
829            iBuilder->CreateCondBr(wraparound, copyBack, done);
830            iBuilder->SetInsertPoint(copyBack);
831            Value * copyItems = iBuilder->CreateSub(newlyProduced, accessible);
832            cb->createCopyBack(iBuilder, instance, copyItems);
833            iBuilder->CreateBr(done);
834            iBuilder->SetInsertPoint(done);
835            priorIdx++;
836        }
837    }
838
839
840    /// Call the do block method if necessary then restore the current function state to the do segement method
841    if (!useIndirectBr()) {
842        iBuilder->CreateRetVoid();
843        mDoBlockMethod = mCurrentMethod;
844        iBuilder->restoreIP(ip);
845        iBuilder->CreateCall(mCurrentMethod, self);
846        setInstance(self);
847        mCurrentMethod = cp;
848    }
849
850}
851
852inline void BlockOrientedKernel::writeFinalBlockMethod(Value * remainingItems) {
853
854    Value * const self = getInstance();
855    Function * const cp = mCurrentMethod;
856    Value * const remainingItemCount = remainingItems;
857    auto ip = iBuilder->saveIP();
858
859    if (!useIndirectBr()) {
860        FunctionType * const type = FunctionType::get(iBuilder->getVoidTy(), {self->getType(), iBuilder->getSizeTy()}, false);
861        mCurrentMethod = Function::Create(type, GlobalValue::InternalLinkage, getName() + FINAL_BLOCK_SUFFIX, iBuilder->getModule());
862        mCurrentMethod->setCallingConv(CallingConv::C);
863        mCurrentMethod->setDoesNotThrow();
864        mCurrentMethod->setDoesNotCapture(1);
865        auto args = mCurrentMethod->arg_begin();
866        args->setName("self");
867        setInstance(&*args);
868        remainingItems = &*(++args);
869        remainingItems->setName("remainingItems");
870        iBuilder->SetInsertPoint(CreateBasicBlock("entry"));
871    }
872
873    generateFinalBlockMethod(remainingItems); // may be implemented by the BlockOrientedKernel subtype
874
875    RecursivelyDeleteTriviallyDeadInstructions(remainingItems); // if remainingItems was not used, this will eliminate it.
876
877    if (!useIndirectBr()) {
878        iBuilder->CreateRetVoid();       
879        iBuilder->restoreIP(ip);
880        iBuilder->CreateCall(mCurrentMethod, {self, remainingItemCount});
881        mCurrentMethod = cp;
882        setInstance(self);
883    }
884
885}
886
887//  The default finalBlock method simply dispatches to the doBlock routine.
888void BlockOrientedKernel::generateFinalBlockMethod(Value * /* remainingItems */) {
889    CreateDoBlockMethodCall();
890}
891
892void BlockOrientedKernel::CreateDoBlockMethodCall() {
893    if (useIndirectBr()) {
894        BasicBlock * bb = CreateBasicBlock("resume");
895        mStrideLoopBranch->addDestination(bb);
896        mStrideLoopTarget->addIncoming(BlockAddress::get(bb), iBuilder->GetInsertBlock());
897        iBuilder->CreateBr(mStrideLoopBody);
898        bb->moveAfter(iBuilder->GetInsertBlock());
899        iBuilder->SetInsertPoint(bb);
900    } else {
901        iBuilder->CreateCall(mDoBlockMethod, getInstance());
902    }
903}
904
905void Kernel::finalizeInstance() {
906    assert ("KernelBuilder does not have a valid IDISA Builder" && iBuilder);
907    mOutputScalarResult = iBuilder->CreateCall(getTerminateFunction(iBuilder->getModule()), { getInstance() });
908}
909
910Kernel::StreamPort Kernel::getStreamPort(const std::string & name) const {
911    const auto f = mStreamMap.find(name);
912    if (LLVM_UNLIKELY(f == mStreamMap.end())) {
913        report_fatal_error(getName() + " does not contain stream set " + name);
914    }
915    return f->second;
916}
917
918// CONSTRUCTOR
919Kernel::Kernel(std::string && kernelName,
920                             std::vector<Binding> && stream_inputs,
921                             std::vector<Binding> && stream_outputs,
922                             std::vector<Binding> && scalar_parameters,
923                             std::vector<Binding> && scalar_outputs,
924                             std::vector<Binding> && internal_scalars)
925: KernelInterface(std::move(kernelName), std::move(stream_inputs), std::move(stream_outputs), std::move(scalar_parameters), std::move(scalar_outputs), std::move(internal_scalars))
926, mCurrentMethod(nullptr)
927, mNoTerminateAttribute(false)
928, mIsGenerated(false)
929, mIsFinal(nullptr)
930, mOutputScalarResult(nullptr) {
931
932}
933
934Kernel::~Kernel() {
935
936}
937
938// CONSTRUCTOR
939BlockOrientedKernel::BlockOrientedKernel(std::string && kernelName,
940                                         std::vector<Binding> && stream_inputs,
941                                         std::vector<Binding> && stream_outputs,
942                                         std::vector<Binding> && scalar_parameters,
943                                         std::vector<Binding> && scalar_outputs,
944                                         std::vector<Binding> && internal_scalars)
945: Kernel(std::move(kernelName), std::move(stream_inputs), std::move(stream_outputs), std::move(scalar_parameters), std::move(scalar_outputs), std::move(internal_scalars))
946, mDoBlockMethod(nullptr)
947, mStrideLoopBody(nullptr)
948, mStrideLoopBranch(nullptr)
949, mStrideLoopTarget(nullptr) {
950
951}
952
953// CONSTRUCTOR
954SegmentOrientedKernel::SegmentOrientedKernel(std::string && kernelName,
955                                             std::vector<Binding> && stream_inputs,
956                                             std::vector<Binding> && stream_outputs,
957                                             std::vector<Binding> && scalar_parameters,
958                                             std::vector<Binding> && scalar_outputs,
959                                             std::vector<Binding> && internal_scalars)
960: Kernel(std::move(kernelName), std::move(stream_inputs), std::move(stream_outputs), std::move(scalar_parameters), std::move(scalar_outputs), std::move(internal_scalars)) {
961
962}
963
964}
Note: See TracBrowser for help on using the repository browser.