source: icGREP/icgrep-devel/icgrep/kernels/kernel.cpp @ 6135

Last change on this file since 6135 was 6047, checked in by nmedfort, 10 months ago

Major refactoring of buffer types. Static buffers replace Circular and CircularCopyback?. External buffers unify Source/External?.

File size: 34.0 KB
Line 
1/*
2 *  Copyright (c) 2018 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5
6#include "kernel.h"
7#include <toolchain/toolchain.h>
8#include <kernels/streamset.h>
9#include <llvm/IR/Constants.h>
10#include <llvm/IR/Function.h>
11#include <llvm/IR/Instructions.h>
12#include <llvm/IR/MDBuilder.h>
13#include <llvm/IR/Module.h>
14#include <llvm/Support/raw_ostream.h>
15#if LLVM_VERSION_INTEGER < LLVM_VERSION_CODE(4, 0, 0)
16#include <llvm/Bitcode/ReaderWriter.h>
17#else
18#include <llvm/Bitcode/BitcodeWriter.h>
19#endif
20#include <llvm/Transforms/Utils/Local.h>
21#include <kernels/streamset.h>
22#include <sstream>
23#include <kernels/kernel_builder.h>
24#include <llvm/Support/Debug.h>
25
26using namespace llvm;
27using namespace parabix;
28
29namespace kernel {
30
31/** ------------------------------------------------------------------------------------------------------------- *
32 * @brief addScalar
33 ** ------------------------------------------------------------------------------------------------------------- */
34unsigned Kernel::addScalar(Type * const type, const std::string & name) {
35    if (LLVM_UNLIKELY(mKernelStateType != nullptr)) {
36        report_fatal_error("Cannot add field " + name + " to " + getName() + " after kernel state finalized");
37    }
38    if (LLVM_UNLIKELY(mKernelFieldMap.count(name))) {
39        report_fatal_error(getName() + " already contains scalar field " + name);
40    }
41    const auto index = mKernelFields.size();
42    mKernelFieldMap.emplace(name, index);
43    mKernelFields.push_back(type);
44    return index;
45}
46
47/** ------------------------------------------------------------------------------------------------------------- *
48 * @brief addUnnamedScalar
49 ** ------------------------------------------------------------------------------------------------------------- */
50unsigned Kernel::addUnnamedScalar(Type * const type) {
51    if (LLVM_UNLIKELY(mKernelStateType != nullptr)) {
52        report_fatal_error("Cannot add unnamed field  to " + getName() + " after kernel state finalized");
53    }
54    const auto index = mKernelFields.size();
55    mKernelFields.push_back(type);
56    return index;
57}
58
59/** ------------------------------------------------------------------------------------------------------------- *
60 * @brief bindPorts
61 ** ------------------------------------------------------------------------------------------------------------- */
62void Kernel::bindPorts(const StreamSetBuffers & inputs, const StreamSetBuffers & outputs) {
63
64    if (LLVM_UNLIKELY(mStreamSetInputs.size() != inputs.size())) {
65        report_fatal_error(getName() + ": expected " + std::to_string(mStreamSetInputs.size()) +
66                           " input stream sets but was given "
67                           + std::to_string(inputs.size()));
68    }
69
70    for (unsigned i = 0; i < mStreamSetInputs.size(); i++) {
71        mStreamMap.emplace(mStreamSetInputs[i].getName(), std::make_pair(Port::Input, i));
72    }
73
74    for (unsigned i = 0; i < inputs.size(); ++i) {
75        StreamSetBuffer * const buf = inputs[i];
76        if (LLVM_UNLIKELY(buf == nullptr)) {
77            report_fatal_error(getName() + ": input stream " + std::to_string(i) + " cannot be null");
78        }
79       // const Binding & input = mStreamSetInputs[i];
80       // verifyBufferSize(input, buf);
81        buf->addConsumer(this);
82    }
83
84    if (LLVM_UNLIKELY(mStreamSetOutputs.size() != outputs.size())) {
85        report_fatal_error(getName() + ": expected " + std::to_string(mStreamSetOutputs.size())
86                           + " output stream sets but was given "
87                           + std::to_string(outputs.size()));
88    }
89
90    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
91        mStreamMap.emplace(mStreamSetOutputs[i].getName(), std::make_pair(Port::Output, i));
92    }
93
94    for (unsigned i = 0; i < outputs.size(); ++i) {
95        StreamSetBuffer * const buf = outputs[i];
96        if (LLVM_UNLIKELY(buf == nullptr)) {
97            report_fatal_error(getName() + ": output stream set " + std::to_string(i) + " cannot be null");
98        }
99        const Binding & output = mStreamSetOutputs[i];
100       // verifyBufferSize(output, buf);
101        if (LLVM_LIKELY(buf->getProducer() == nullptr)) {
102            buf->setProducer(this);
103        } else {
104            report_fatal_error(getName() + ": output stream set " + output.getName()
105                               + " is already produced by kernel " + buf->getProducer()->getName());
106        }
107    }
108
109    mStreamSetInputBuffers.assign(inputs.begin(), inputs.end());
110    mStreamSetOutputBuffers.assign(outputs.begin(), outputs.end());
111}
112
113/** ------------------------------------------------------------------------------------------------------------- *
114 * @brief getCacheName
115 ** ------------------------------------------------------------------------------------------------------------- */
116std::string Kernel::getCacheName(const std::unique_ptr<KernelBuilder> & b) const {
117    std::stringstream cacheName;
118    cacheName << getName() << '_' << b->getBuilderUniqueName();
119    for (const StreamSetBuffer * b: mStreamSetInputBuffers) {
120        cacheName <<  ':' <<  b->getUniqueID();
121    }
122    for (const StreamSetBuffer * b: mStreamSetOutputBuffers) {
123        cacheName <<  ':' <<  b->getUniqueID();
124    }
125    return cacheName.str();
126}
127
128
129/** ------------------------------------------------------------------------------------------------------------- *
130 * @brief setModule
131 ** ------------------------------------------------------------------------------------------------------------- */
132Module * Kernel::setModule(Module * const module) {
133    assert (mModule == nullptr || mModule == module);
134    assert (module != nullptr);
135    mModule = module;
136    return mModule;
137}
138
139/** ------------------------------------------------------------------------------------------------------------- *
140 * @brief makeModule
141 ** ------------------------------------------------------------------------------------------------------------- */
142Module * Kernel::makeModule(const std::unique_ptr<kernel::KernelBuilder> & idb) {
143    Module * m = new Module(getCacheName(idb), idb->getContext());
144    m->setTargetTriple(idb->getModule()->getTargetTriple());
145    m->setDataLayout(idb->getModule()->getDataLayout());
146    return setModule(m);
147}
148
149
150/** ------------------------------------------------------------------------------------------------------------- *
151 * @brief prepareKernel
152 ** ------------------------------------------------------------------------------------------------------------- */
153void Kernel::prepareKernel(const std::unique_ptr<KernelBuilder> & b) {
154    assert ("KernelBuilder does not have a valid IDISA Builder" && b);
155    if (LLVM_UNLIKELY(mKernelStateType != nullptr)) {
156        report_fatal_error(getName() + ": cannot prepare kernel after kernel state finalized");
157    }
158    // verifyStreamSetDefinitions();
159    addBaseKernelProperties(b);
160    addInternalKernelProperties(b);
161    // NOTE: StructType::create always creates a new type even if an identical one exists.
162    if (LLVM_UNLIKELY(mModule == nullptr)) {
163        makeModule(b);
164    }
165    mKernelStateType = mModule->getTypeByName(getName());
166    if (LLVM_LIKELY(mKernelStateType == nullptr)) {
167        mKernelStateType = StructType::create(b->getContext(), mKernelFields, getName());
168        assert (mKernelStateType);
169    }
170}
171
172
173/** ------------------------------------------------------------------------------------------------------------- *
174 * @brief prepareCachedKernel
175 ** ------------------------------------------------------------------------------------------------------------- */
176void Kernel::prepareCachedKernel(const std::unique_ptr<KernelBuilder> & b) {
177    assert ("KernelBuilder does not have a valid IDISA Builder" && b);
178    if (LLVM_UNLIKELY(mKernelStateType != nullptr)) {
179        report_fatal_error(getName() + ": cannot prepare kernel after kernel state finalized");
180    }
181    assert (getModule());   
182    addBaseKernelProperties(b);
183    mKernelStateType = getModule()->getTypeByName(getName());
184    if (LLVM_UNLIKELY(mKernelStateType == nullptr)) {
185        report_fatal_error("Kernel definition for " + getName() + " could not be found in the cache object");
186    }
187}
188
189/** ------------------------------------------------------------------------------------------------------------- *
190 * @brief containsFixedRate
191 ** ------------------------------------------------------------------------------------------------------------- */
192bool containsFixedRate(const Bindings & bindings) {
193    for (const Binding & binding : bindings) {
194        const ProcessingRate & rate = binding.getRate();
195        if (rate.isFixed()) {
196            return true;
197        }
198    }
199    return false;
200}
201
202/** ------------------------------------------------------------------------------------------------------------- *
203 * @brief addBaseKernelProperties
204 ** ------------------------------------------------------------------------------------------------------------- */
205void Kernel::addBaseKernelProperties(const std::unique_ptr<KernelBuilder> & b) {
206
207    const unsigned inputSetCount = mStreamSetInputs.size();
208    const unsigned outputSetCount = mStreamSetOutputs.size();
209
210    assert (inputSetCount == mStreamSetInputBuffers.size());
211    assert (outputSetCount == mStreamSetOutputBuffers.size());
212
213    if (mStride == 0) {
214        // Set the default kernel stride.
215        mStride = b->getBitBlockWidth();
216    }
217
218    IntegerType * const sizeTy = b->getSizeTy();
219
220    addScalar(sizeTy, LOGICAL_SEGMENT_NO_SCALAR);
221    addScalar(sizeTy, TERMINATION_SIGNAL);
222    // TODO: if we had a way of easily calculating the number of processed/produced items of the
223    // final stride of a non-deferred fixed rate stream, we could avoid storing the item counts.
224    for (unsigned i = 0; i < inputSetCount; i++) {
225        const Binding & input = mStreamSetInputs[i];
226        addScalar(sizeTy, input.getName() + PROCESSED_ITEM_COUNT_SUFFIX);
227        if (LLVM_UNLIKELY(input.isDeferred())) {
228            addScalar(sizeTy, input.getName() + NON_DEFERRED_ITEM_COUNT_SUFFIX);
229        }
230    }
231    for (unsigned i = 0; i < outputSetCount; i++) {
232        const Binding & output = mStreamSetOutputs[i];
233        addScalar(sizeTy, output.getName() + PRODUCED_ITEM_COUNT_SUFFIX);
234        if (LLVM_UNLIKELY(output.isDeferred())) {
235            addScalar(sizeTy, output.getName() + NON_DEFERRED_ITEM_COUNT_SUFFIX);
236        }
237    }
238    for (unsigned i = 0; i < inputSetCount; i++) {
239        mScalarInputs.emplace_back(mStreamSetInputBuffers[i]->getStreamSetHandle()->getType(), mStreamSetInputs[i].getName() + BUFFER_SUFFIX);
240    }
241    for (unsigned i = 0; i < outputSetCount; i++) {
242        mScalarInputs.emplace_back(mStreamSetOutputBuffers[i]->getStreamSetHandle()->getType(), mStreamSetOutputs[i].getName() + BUFFER_SUFFIX);
243    }
244    for (const auto & binding : mScalarInputs) {
245        addScalar(binding.getType(), binding.getName());
246    }
247    for (const auto & binding : mScalarOutputs) {
248        addScalar(binding.getType(), binding.getName());
249    }
250    for (const auto & binding : mInternalScalars) {
251        addScalar(binding.getType(), binding.getName());
252    }
253    Type * const consumerSetTy = StructType::get(b->getContext(), {sizeTy, sizeTy->getPointerTo()->getPointerTo()})->getPointerTo();
254    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
255        addScalar(consumerSetTy, mStreamSetOutputs[i].getName() + CONSUMER_SUFFIX);
256    }
257    for (unsigned i = 0; i < mStreamSetOutputs.size(); i++) {
258        addScalar(sizeTy, mStreamSetOutputs[i].getName() + CONSUMED_ITEM_COUNT_SUFFIX);
259    }
260    // We compile in a 64-bit CPU cycle counter into every kernel.   It will remain unused
261    // in normal execution, but when codegen::EnableCycleCounter is specified, pipelines
262    // will be able to add instrumentation to cached modules without recompilation.
263    addScalar(b->getInt64Ty(), CYCLECOUNT_SCALAR);
264}
265
266
267/** ------------------------------------------------------------------------------------------------------------- *
268 * @brief makeSignature
269 *
270 * Default kernel signature: generate the IR and emit as byte code.
271 ** ------------------------------------------------------------------------------------------------------------- */
272std::string Kernel::makeSignature(const std::unique_ptr<kernel::KernelBuilder> & idb) {
273    assert ("KernelBuilder does not have a valid IDISA Builder" && idb.get());
274    if (LLVM_UNLIKELY(hasSignature())) {
275        generateKernel(idb);
276        std::string tmp;
277        raw_string_ostream signature(tmp);
278        WriteBitcodeToFile(getModule(), signature);
279        return signature.str();
280    } else {
281        return getModule()->getModuleIdentifier();
282    }
283}
284
285
286/** ------------------------------------------------------------------------------------------------------------- *
287 * @brief generateKernel
288 ** ------------------------------------------------------------------------------------------------------------- */
289void Kernel::generateKernel(const std::unique_ptr<kernel::KernelBuilder> & idb) {
290    assert ("Kernel does not have a valid IDISA Builder" && idb.get());
291    if (LLVM_UNLIKELY(mIsGenerated)) return;
292    idb->setModule(mModule);
293    addKernelDeclarations(idb);
294    callGenerateInitializeMethod(idb);
295    callGenerateDoSegmentMethod(idb);
296    callGenerateFinalizeMethod(idb);
297    mIsGenerated = true;
298}
299
300
301/** ------------------------------------------------------------------------------------------------------------- *
302 * @brief callGenerateInitializeMethod
303 ** ------------------------------------------------------------------------------------------------------------- */
304inline void Kernel::callGenerateInitializeMethod(const std::unique_ptr<kernel::KernelBuilder> & b) {
305    mCurrentMethod = getInitFunction(b->getModule());
306    b->SetInsertPoint(BasicBlock::Create(b->getContext(), "entry", mCurrentMethod));
307    Function::arg_iterator args = mCurrentMethod->arg_begin();
308    setInstance(&*(args++));
309    b->CreateStore(ConstantAggregateZero::get(mKernelStateType), getInstance());
310    for (const auto & binding : mScalarInputs) {
311        b->setScalarField(binding.getName(), &*(args++));
312    }
313    for (const auto & binding : mStreamSetOutputs) {
314        b->setConsumerLock(binding.getName(), &*(args++));
315    }
316    generateInitializeMethod(b);
317    b->CreateRetVoid();
318}
319
320/** ------------------------------------------------------------------------------------------------------------- *
321 * @brief callGenerateDoSegmentMethod
322 ** ------------------------------------------------------------------------------------------------------------- */
323inline void Kernel::callGenerateDoSegmentMethod(const std::unique_ptr<kernel::KernelBuilder> & b) {
324    mCurrentMethod = getDoSegmentFunction(b->getModule());
325    b->SetInsertPoint(BasicBlock::Create(b->getContext(), "entry", mCurrentMethod));
326    auto args = mCurrentMethod->arg_begin();
327    setInstance(&*(args++));
328    mIsFinal = &*(args++);
329    const auto n = mStreamSetInputs.size();
330    mAvailableItemCount.resize(n, nullptr);
331    for (unsigned i = 0; i < n; i++) {
332        assert (args != mCurrentMethod->arg_end());
333        mAvailableItemCount[i] = &*(args++);
334    }
335    assert (args == mCurrentMethod->arg_end());
336    generateKernelMethod(b); // must be overridden by the Kernel subtype
337    mIsFinal = nullptr;
338    mAvailableItemCount.clear();
339    b->CreateRetVoid();
340}
341
342
343/** ------------------------------------------------------------------------------------------------------------- *
344 * @brief callGenerateFinalizeMethod
345 ** ------------------------------------------------------------------------------------------------------------- */
346inline void Kernel::callGenerateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
347    mCurrentMethod = getTerminateFunction(b->getModule());
348    b->SetInsertPoint(BasicBlock::Create(b->getContext(), "entry", mCurrentMethod));
349    auto args = mCurrentMethod->arg_begin();
350    setInstance(&*(args++));
351    generateFinalizeMethod(b); // may be overridden by the Kernel subtype
352    const auto n = mScalarOutputs.size();
353    if (n == 0) {
354        b->CreateRetVoid();
355    } else {
356        Value * outputs[n];
357        for (unsigned i = 0; i < n; ++i) {
358            outputs[i] = b->getScalarField(mScalarOutputs[i].getName());
359        }
360        if (n == 1) {
361            b->CreateRet(outputs[0]);
362        } else {
363            b->CreateAggregateRet(outputs, n);
364        }
365    }
366}
367
368
369/** ------------------------------------------------------------------------------------------------------------- *
370 * @brief getScalarIndex
371 ** ------------------------------------------------------------------------------------------------------------- */
372unsigned Kernel::getScalarIndex(const std::string & name) const {
373    const auto f = mKernelFieldMap.find(name);
374    if (LLVM_UNLIKELY(f == mKernelFieldMap.end())) {
375        assert ("kernel does not contain the requested scalar" && false);
376        report_fatal_error(getName() + " does not contain scalar: " + name);
377    }
378    return f->second;
379}
380
381
382/** ------------------------------------------------------------------------------------------------------------- *
383 * @brief createInstance
384 ** ------------------------------------------------------------------------------------------------------------- */
385Value * Kernel::createInstance(const std::unique_ptr<KernelBuilder> & idb) {
386    assert ("KernelBuilder does not have a valid IDISA Builder" && idb);
387    if (LLVM_UNLIKELY(mKernelStateType == nullptr)) {
388        report_fatal_error("Cannot instantiate " + getName() + " before calling prepareKernel()");
389    }
390    setInstance(idb->CreateCacheAlignedAlloca(mKernelStateType));
391    return getInstance();
392}
393
394
395/** ------------------------------------------------------------------------------------------------------------- *
396 * @brief initializeInstance
397 ** ------------------------------------------------------------------------------------------------------------- */
398void Kernel::initializeInstance(const std::unique_ptr<KernelBuilder> & b) {
399    assert ("KernelBuilder does not have a valid IDISA Builder" && b);
400    if (LLVM_UNLIKELY(getInstance() == nullptr)) {
401        report_fatal_error("Cannot initialize " + getName() + " before calling createInstance()");
402    }
403    std::vector<Value *> args;
404    args.reserve(1 + mInitialArguments.size() + mStreamSetInputBuffers.size() + (mStreamSetOutputBuffers.size() * 2));
405    args.push_back(getInstance());
406    for (unsigned i = 0; i < mInitialArguments.size(); ++i) {
407        Value * arg = mInitialArguments[i];
408        if (LLVM_UNLIKELY(arg == nullptr)) {
409            report_fatal_error(getName() + ": initial argument " + std::to_string(i)
410                               + " cannot be null when calling createInstance()");
411        }
412        args.push_back(arg);
413    }
414    for (unsigned i = 0; i < mStreamSetInputBuffers.size(); ++i) {
415        assert (mStreamSetInputBuffers[i]);
416        Value * arg = mStreamSetInputBuffers[i]->getStreamSetHandle();
417        if (LLVM_UNLIKELY(arg == nullptr)) {
418            report_fatal_error(getName() + ": input stream set " + std::to_string(i)
419                               + " was not allocated prior to calling createInstance()");
420        }
421        args.push_back(arg);
422    }
423    assert (mStreamSetInputs.size() == mStreamSetInputBuffers.size());
424    for (unsigned i = 0; i < mStreamSetOutputBuffers.size(); ++i) {
425        assert (mStreamSetOutputBuffers[i]);
426        Value * arg = mStreamSetOutputBuffers[i]->getStreamSetHandle();
427        if (LLVM_UNLIKELY(arg == nullptr)) {
428            report_fatal_error(getName() + ": output stream set " + std::to_string(i)
429                               + " was not allocated prior to calling createInstance()");
430        }
431        args.push_back(arg);
432    }
433    assert (mStreamSetOutputs.size() == mStreamSetOutputBuffers.size());
434    IntegerType * const sizeTy = b->getSizeTy();
435    PointerType * const sizePtrTy = sizeTy->getPointerTo();
436    PointerType * const sizePtrPtrTy = sizePtrTy->getPointerTo();
437    StructType * const consumerTy = StructType::get(b->getContext(), {sizeTy, sizePtrPtrTy});
438    for (unsigned i = 0; i < mStreamSetOutputBuffers.size(); ++i) {
439        const auto output = mStreamSetOutputBuffers[i];
440        const auto & consumers = output->getConsumers();
441        const auto n = consumers.size();
442        AllocaInst * const outputConsumers = b->CreateAlloca(consumerTy);
443        Value * const consumerSegNoArray = b->CreateAlloca(ArrayType::get(sizePtrTy, n));
444        for (unsigned i = 0; i < n; ++i) {
445            Kernel * const consumer = consumers[i];
446            assert ("all instances must be created prior to initialization of any instance" && consumer->getInstance());
447            b->setKernel(consumer);
448            Value * const segmentNoPtr = b->getScalarFieldPtr(LOGICAL_SEGMENT_NO_SCALAR);
449            b->CreateStore(segmentNoPtr, b->CreateGEP(consumerSegNoArray, { b->getInt32(0), b->getInt32(i) }));
450        }
451        b->setKernel(this);
452        Value * const consumerCountPtr = b->CreateGEP(outputConsumers, {b->getInt32(0), b->getInt32(0)});
453        b->CreateStore(b->getSize(n), consumerCountPtr);
454        Value * const consumerSegNoArrayPtr = b->CreateGEP(outputConsumers, {b->getInt32(0), b->getInt32(1)});
455        b->CreateStore(b->CreatePointerCast(consumerSegNoArray, sizePtrPtrTy), consumerSegNoArrayPtr);
456        args.push_back(outputConsumers);
457    }
458    b->CreateCall(getInitFunction(b->getModule()), args);
459}
460
461/** ------------------------------------------------------------------------------------------------------------- *
462 * @brief finalizeInstance
463 ** ------------------------------------------------------------------------------------------------------------- */
464void Kernel::finalizeInstance(const std::unique_ptr<KernelBuilder> & idb) {
465    assert ("KernelBuilder does not have a valid IDISA Builder" && idb);
466    mOutputScalarResult = idb->CreateCall(getTerminateFunction(idb->getModule()), { getInstance() });
467}
468
469/** ------------------------------------------------------------------------------------------------------------- *
470 * @brief getStreamPort
471 ** ------------------------------------------------------------------------------------------------------------- */
472Kernel::StreamPort Kernel::getStreamPort(const std::string & name) const {
473    const auto f = mStreamMap.find(name);
474    if (LLVM_UNLIKELY(f == mStreamMap.end())) {
475        assert (!mStreamMap.empty());
476        report_fatal_error(getName() + " does not contain stream set " + name);
477    }
478    return f->second;
479}
480
481/** ------------------------------------------------------------------------------------------------------------- *
482 * @brief getStreamPort
483 ** ------------------------------------------------------------------------------------------------------------- */
484const Binding & Kernel::getBinding(const std::string & name) const {
485    Port port; unsigned index;
486    std::tie(port, index) = getStreamPort(name);
487    return (port == Port::Input) ? getStreamInput(index) : getStreamOutput(index);
488}
489
490/** ------------------------------------------------------------------------------------------------------------- *
491 * @brief getLowerBound
492 ** ------------------------------------------------------------------------------------------------------------- */
493ProcessingRate::RateValue Kernel::getLowerBound(const ProcessingRate & rate) const {
494    if (rate.isFixed() || rate.isBounded()) {
495        return rate.getLowerBound();
496    } else if (rate.hasReference()) {
497        return rate.getLowerBound() * getLowerBound(getBinding(rate.getReference()).getRate());
498    } else { // if (rate.isUnknown())
499        return 0;
500    }
501}
502
503/** ------------------------------------------------------------------------------------------------------------- *
504 * @brief getUpperBound
505 ** ------------------------------------------------------------------------------------------------------------- */
506ProcessingRate::RateValue Kernel::getUpperBound(const ProcessingRate &rate) const {
507    if (rate.isFixed() || rate.isBounded() || rate.isPopCount()) {
508        return rate.getUpperBound();
509    } else if (rate.hasReference()) {
510        return rate.getUpperBound() * getUpperBound(getBinding(rate.getReference()).getRate());
511    } else { // if (rate.isUnknown())
512        return 0;
513    }
514}
515
516/** ------------------------------------------------------------------------------------------------------------- *
517 * @brief verifyStreamSetDefinitions
518 ** ------------------------------------------------------------------------------------------------------------- */
519void Kernel::verifyStreamSetDefinitions() const {
520    unsigned numOfPrincipalStreams = 0;
521    for (unsigned i = 0; i < mStreamSetInputs.size(); ++i) {
522        const Binding & input = mStreamSetInputs[i];
523        const ProcessingRate & rate = input.getRate();
524        // If a stream can be relative to a relative or fixed rate stream, it complicates the pipeline and
525        // multiblock kernel. For now, report an error.
526        if (LLVM_UNLIKELY(rate.hasReference())) {
527            Port port; unsigned index;
528            std::tie(port, index) = getStreamPort(rate.getReference());
529            if (LLVM_UNLIKELY(port == Port::Output)) {
530                report_fatal_error(getName() + ": input stream \"" + input.getName() + "\" cannot refer to an output stream");
531            }
532            if (LLVM_UNLIKELY(index >= i)) {
533                report_fatal_error(getName() + ": \"" + input.getName() + "\" must be ordered after its reference stream");
534            }
535            if (rate.isRelative()) {
536                const ProcessingRate & refRate = getStreamInput(index).getRate();
537                if (LLVM_UNLIKELY(refRate.isRelative() || refRate.isFixed())) {
538                    report_fatal_error(getName() + ": \"" + input.getName() + "\" cannot be relative to a fixed or relative rate stream");
539                }
540            }
541        } else if (LLVM_UNLIKELY(rate.isUnknown())) {
542            report_fatal_error(getName() + ": \"" + input.getName() + "\" cannot be an unknown rate");
543        }
544        if (LLVM_UNLIKELY(input.isPrincipal())) {
545            ++numOfPrincipalStreams;
546        }
547        bool hasFixedOnlyAttribute = false;
548        for (const Attribute & attr : input.getAttributes()) {
549            switch (attr.getKind()) {
550                case Attribute::KindId::Add:
551                case Attribute::KindId::RoundUpTo:
552                case Attribute::KindId::Deferred:
553                    hasFixedOnlyAttribute = false;
554                    break;
555                default: break;
556            }
557        }
558        if (rate.isFixed()) {
559
560
561
562        } else if (LLVM_UNLIKELY(hasFixedOnlyAttribute)) {
563            report_fatal_error(getName() + ": Add, RoundUpTo and Deferred cannot be applied to non-Fixed rate input stream \"" + input.getName() + "\"");
564        }
565    }
566    if (LLVM_UNLIKELY(numOfPrincipalStreams > 1)) {
567        report_fatal_error(getName() + ": may only have one principal stream set");
568    }
569    for (unsigned i = 0; i < mStreamSetOutputs.size(); ++i) {
570        const Binding & output = mStreamSetOutputs[i];
571        const ProcessingRate & rate = output.getRate();
572        if (LLVM_UNLIKELY(rate.hasReference())) {
573            Port port; unsigned index;
574            std::tie(port, index) = getStreamPort(rate.getReference());
575            if (LLVM_UNLIKELY(rate.isPopCount() && port == Port::Output)) {
576                report_fatal_error(getName() + ": the popcount rate of \"" + output.getName() + "\" cannot refer to another output stream");
577            }
578            if (LLVM_UNLIKELY(port == Port::Output && index >= i)) {
579                report_fatal_error(getName() + ": \"" + output.getName() + "\" must be ordered after its reference stream");
580            }
581            if (rate.isRelative()) {
582                const Binding & ref = (port == Port::Input) ? getStreamInput(index) : getStreamOutput(index);
583                const ProcessingRate & refRate = ref.getRate();
584                if (LLVM_UNLIKELY(refRate.isRelative() || refRate.isFixed())) {
585                    report_fatal_error(getName() + ": \"" + output.getName() + "\" cannot be relative to a fixed or relative rate stream");
586                }
587            }
588        }
589        if (LLVM_UNLIKELY(output.isPrincipal())) {
590            report_fatal_error(getName() + ": output stream \"" + output.getName() + "\" cannot be a principal stream");
591        }
592
593        bool hasAddOrRoundUpTo = false;
594        bool hasDeferred = false;
595        for (const Attribute & attr : output.getAttributes()) {
596            switch (attr.getKind()) {
597                case Attribute::KindId::Add:
598                case Attribute::KindId::RoundUpTo:
599                    hasAddOrRoundUpTo = true;
600                    break;
601                case Attribute::KindId::Deferred:
602                    hasDeferred = false;
603                    break;
604                default: break;
605            }
606        }
607
608        if (LLVM_UNLIKELY((hasAddOrRoundUpTo || hasDeferred) && !(rate.isFixed() || rate.isPopCount()))) {
609            report_fatal_error(getName() + ": " + output.getName() + " cannot have an Add, RoundUpTo or Deferred attribute");
610        }       
611        if (LLVM_UNLIKELY(hasDeferred && hasAddOrRoundUpTo)) {
612            report_fatal_error(getName() + ": cannot apply Add or RoundUpTo attributes to the Deferred output stream " + output.getName());
613        }
614    }
615}
616
617/** ------------------------------------------------------------------------------------------------------------- *
618 * @brief requiresCopyBack
619 ** ------------------------------------------------------------------------------------------------------------- */
620bool Kernel::requiresCopyBack(const Binding & binding) const {
621    const ProcessingRate & rate = binding.getRate();
622    if (rate.isFixed() || binding.hasAttribute(Attribute::KindId::BlockSize)) {
623        return false;
624    } else if (rate.isRelative()) {
625        return requiresCopyBack(getBinding(rate.getReference()));
626    }
627    return true;
628}
629
630/** ------------------------------------------------------------------------------------------------------------- *
631 * @brief requiresLinearAccess
632 ** ------------------------------------------------------------------------------------------------------------- */
633bool Kernel::requiresLinearAccess(const Binding & binding) const {
634    return binding.hasAttribute(Attribute::KindId::RequiresLinearAccess);
635}
636
637/** ------------------------------------------------------------------------------------------------------------- *
638 * @brief strideOffsetIsTriviallyCalculable
639 ** ------------------------------------------------------------------------------------------------------------- */
640bool Kernel::strideOffsetIsTriviallyCalculable(const Binding & binding) const {
641    if (requiresCopyBack(binding)) {
642        const ProcessingRate & rate = binding.getRate();
643        return rate.isPopCount() || rate.isNegatedPopCount();
644    }
645    return true;
646}
647
648/** ------------------------------------------------------------------------------------------------------------- *
649 * @brief permitsNonLinearAccess
650 ** ------------------------------------------------------------------------------------------------------------- */
651bool Kernel::permitsNonLinearAccess(const Binding & binding) const {
652    if (LLVM_UNLIKELY(requiresLinearAccess(binding))) {
653        return false;
654    } else if (LLVM_UNLIKELY(binding.hasAttribute(Attribute::KindId::PermitsNonLinearAccess))) {
655        return true;
656    } else {
657        return strideOffsetIsTriviallyCalculable(binding);
658    }
659}
660
661/** ------------------------------------------------------------------------------------------------------------- *
662 * @brief mustClearOverflowPriorToCopyback
663 ** ------------------------------------------------------------------------------------------------------------- */
664bool Kernel::mustClearOverflowPriorToCopyback(const Binding & binding) const {
665    return requiresCopyBack(binding) && permitsNonLinearAccess(binding) && !strideOffsetIsTriviallyCalculable(binding);
666}
667
668/** ------------------------------------------------------------------------------------------------------------- *
669 * @brief anyBindingRequiresLinearSpace
670 ** ------------------------------------------------------------------------------------------------------------- */
671bool Kernel::anyBindingRequiresLinearSpace() const {
672    for (const Binding & input : mStreamSetInputs) {
673        if (requiresLinearAccess(input)) {
674            return true;
675        }
676    }
677    for (const Binding & output : mStreamSetOutputs) {
678        if (!permitsNonLinearAccess(output)) {
679            return true;
680        }
681    }
682    return false;
683}
684
685/** ------------------------------------------------------------------------------------------------------------- *
686 * @brief generateKernelMethod
687 ** ------------------------------------------------------------------------------------------------------------- */
688void SegmentOrientedKernel::generateKernelMethod(const std::unique_ptr<KernelBuilder> & b) {
689    mTreatUnsafeKernelOperationsAsErrors = false;
690    generateDoSegmentMethod(b);
691}
692
693static inline std::string annotateKernelNameWithDebugFlags(std::string && name) {
694    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
695        name += "_EA";
696    }
697    name += "_O" + std::to_string((int)codegen::OptLevel);
698    return name;
699}
700
701// CONSTRUCTOR
702Kernel::Kernel(std::string && kernelName,
703               Bindings && stream_inputs,
704               Bindings && stream_outputs,
705               Bindings && scalar_parameters,
706               Bindings && scalar_outputs,
707               Bindings && internal_scalars)
708: KernelInterface(annotateKernelNameWithDebugFlags(std::move(kernelName))
709                  , std::move(stream_inputs), std::move(stream_outputs)
710                  , std::move(scalar_parameters), std::move(scalar_outputs)
711                  , std::move(internal_scalars))
712, mCurrentMethod(nullptr)
713, mStride(0)
714, mTreatUnsafeKernelOperationsAsErrors(false)
715, mIsFinal(nullptr)
716, mOutputScalarResult(nullptr)
717, mIsGenerated(false) {
718
719}
720
721Kernel::~Kernel() {
722
723}
724
725// CONSTRUCTOR
726SegmentOrientedKernel::SegmentOrientedKernel(std::string && kernelName,
727                                             Bindings && stream_inputs,
728                                             Bindings && stream_outputs,
729                                             Bindings && scalar_parameters,
730                                             Bindings && scalar_outputs,
731                                             Bindings && internal_scalars)
732: Kernel(std::move(kernelName), std::move(stream_inputs), std::move(stream_outputs), std::move(scalar_parameters), std::move(scalar_outputs), std::move(internal_scalars)) {
733
734}
735
736
737}
Note: See TracBrowser for help on using the repository browser.