source: icGREP/icgrep-devel/icgrep/kernels/kernel_builder.cpp @ 5706

Last change on this file since 5706 was 5706, checked in by nmedfort, 20 months ago

First stage of MultiBlockKernel? and pipeline restructuring

File size: 15.7 KB
Line 
1#include "kernel_builder.h"
2#include <toolchain/toolchain.h>
3#include <kernels/kernel.h>
4#include <kernels/streamset.h>
5#include <llvm/Support/raw_ostream.h>
6
7using namespace llvm;
8using namespace parabix;
9
10using Value = Value;
11
12namespace kernel {
13
14using Port = Kernel::Port;
15
16Value * KernelBuilder::getScalarFieldPtr(llvm::Value * instance, Value * const index) {
17    assert (instance);
18    CreateAssert(instance, "getScalarFieldPtr: instance cannot be null!");
19    return CreateGEP(instance, {getInt32(0), index});
20}
21
22Value * KernelBuilder::getScalarFieldPtr(llvm::Value * instance, const std::string & fieldName) {
23    return getScalarFieldPtr(instance, getInt32(mKernel->getScalarIndex(fieldName)));
24}
25
26llvm::Value * KernelBuilder::getScalarFieldPtr(llvm::Value * index) {
27    return getScalarFieldPtr(mKernel->getInstance(), index);
28}
29
30llvm::Value *KernelBuilder:: getScalarFieldPtr(const std::string & fieldName) {
31    return getScalarFieldPtr(mKernel->getInstance(), fieldName);
32}
33
34Value * KernelBuilder::getScalarField(const std::string & fieldName) {
35    return CreateLoad(getScalarFieldPtr(fieldName), fieldName);
36}
37
38void KernelBuilder::setScalarField(const std::string & fieldName, Value * value) {
39    CreateStore(value, getScalarFieldPtr(fieldName));
40}
41
42Value * KernelBuilder::getStreamHandle(const std::string & name) {
43    Value * const ptr = getScalarField(name + Kernel::BUFFER_PTR_SUFFIX);
44    CreateAssert(ptr, name + " cannot be null!");
45    return ptr;
46}
47
48LoadInst * KernelBuilder::acquireLogicalSegmentNo() {
49    return CreateAtomicLoadAcquire(getScalarFieldPtr(Kernel::LOGICAL_SEGMENT_NO_SCALAR));
50}
51
52void KernelBuilder::releaseLogicalSegmentNo(Value * nextSegNo) {
53    CreateAtomicStoreRelease(nextSegNo, getScalarFieldPtr(Kernel::LOGICAL_SEGMENT_NO_SCALAR));
54}
55
56Value * KernelBuilder::getCycleCountPtr() {
57    return getScalarFieldPtr(Kernel::CYCLECOUNT_SCALAR);
58}
59
60inline const Binding & getBinding(const Kernel * k, const std::string & name) {
61    Port port; unsigned index;
62    std::tie(port, index) = k->getStreamPort(name);
63    if (port == Port::Input) {
64        return k->getStreamInput(index);
65    } else {
66        return k->getStreamOutput(index);
67    }
68}
69
70Value * KernelBuilder::getInternalItemCount(const std::string & name, const std::string & suffix) {
71    const ProcessingRate & rate = getBinding(mKernel, name).getRate();
72    Value * itemCount = nullptr;
73    if (rate.isExactlyRelative()) {
74        Port port; unsigned index;
75        std::tie(port, index) = mKernel->getStreamPort(rate.getReference());
76        if (port == Port::Input) {
77            itemCount = getProcessedItemCount(rate.getReference());
78        } else {
79            itemCount = getProducedItemCount(rate.getReference());
80        }
81        if (rate.getNumerator() != 1) {
82            itemCount = CreateMul(itemCount, ConstantInt::get(itemCount->getType(), rate.getNumerator()));
83        }
84        if (rate.getDenominator() != 1) {
85            itemCount = CreateExactUDiv(itemCount, ConstantInt::get(itemCount->getType(), rate.getDenominator()));
86        }
87    } else {
88        itemCount = getScalarField(name + suffix);
89    }
90    return itemCount;
91}
92
93void KernelBuilder::setInternalItemCount(const std::string & name, const std::string & suffix, llvm::Value * const value) {
94    const ProcessingRate & rate = getBinding(mKernel, name).getRate();
95    if (LLVM_UNLIKELY(rate.isDerived())) {
96        report_fatal_error("Cannot set item count: " + name + " is a Derived rate");
97    }
98    if (codegen::DebugOptionIsSet(codegen::TraceCounts)) {
99        CallPrintIntToStderr(mKernel->getName() + ": " + name + suffix, value);
100    }
101    setScalarField(name + suffix, value);
102}
103
104
105Value * KernelBuilder::getAvailableItemCount(const std::string & name) {
106    const auto & inputs = mKernel->getStreamInputs();
107    for (unsigned i = 0; i < inputs.size(); ++i) {
108        if (inputs[i].getName() == name) {
109            return mKernel->getAvailableItemCount(i);
110        }
111    }
112    return nullptr;
113}
114
115Value * KernelBuilder::getTerminationSignal() {
116    if (mKernel->hasNoTerminateAttribute()) {
117        return getFalse();
118    }
119    return getScalarField(Kernel::TERMINATION_SIGNAL);
120}
121
122void KernelBuilder::setTerminationSignal(llvm::Value * const value) {
123    assert (!mKernel->hasNoTerminateAttribute());
124    assert (value->getType() == getInt1Ty());
125    if (codegen::DebugOptionIsSet(codegen::TraceCounts)) {
126        CallPrintIntToStderr(mKernel->getName() + ": setTerminationSignal", value);
127    }
128    setScalarField(Kernel::TERMINATION_SIGNAL, value);
129}
130
131Value * KernelBuilder::getLinearlyAccessibleItems(const std::string & name, Value * fromPosition, Value * avail, bool reverse) {
132    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
133    return buf->getLinearlyAccessibleItems(this, getStreamHandle(name), fromPosition, avail, reverse);
134}
135
136Value * KernelBuilder::getLinearlyWritableItems(const std::string & name, Value * fromPosition, bool reverse) {
137    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
138    return buf->getLinearlyWritableItems(this, getStreamHandle(name), fromPosition, reverse);
139}
140
141Value * KernelBuilder::copy(const std::string & name, Value * target, Value * source, Value * itemsToCopy, const unsigned alignment) {
142    const StreamSetBuffer * const buf = mKernel->getAnyStreamSetBuffer(name);
143    return buf->copy(this, getStreamHandle(name), target, source, itemsToCopy, alignment);
144}
145
146void KernelBuilder::CreateCopyBack(const std::string & name, llvm::Value * from, llvm::Value * to) {
147    const StreamSetBuffer * const buf = mKernel->getAnyStreamSetBuffer(name);
148    return buf->genCopyBackLogic(this, getStreamHandle(name), from, to, name);
149}
150
151Value * KernelBuilder::getConsumerLock(const std::string & name) {
152    return getScalarField(name + Kernel::CONSUMER_SUFFIX);
153}
154
155void KernelBuilder::setConsumerLock(const std::string & name, Value * value) {
156    setScalarField(name + Kernel::CONSUMER_SUFFIX, value);
157}
158
159inline Value * KernelBuilder::computeBlockIndex(Value * itemCount) {
160    const auto divisor = getBitBlockWidth();
161    if (LLVM_LIKELY((divisor & (divisor - 1)) == 0)) {
162        return CreateLShr(itemCount, std::log2(divisor));
163    } else {
164        return CreateUDiv(itemCount, getSize(divisor));
165    }
166}
167
168Value * KernelBuilder::getInputStreamPtr(const std::string & name, Value * const blockIndex) {
169//    Value * const blockIndex = computeBlockIndex(getProcessedItemCount(name));
170    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
171    return buf->getBlockAddress(this, getStreamHandle(name), blockIndex);
172}
173
174Value * KernelBuilder::getInputStreamBlockPtr(const std::string & name, Value * streamIndex) {
175    const Kernel::StreamPort p = mKernel->getStreamPort(name);
176    if (LLVM_UNLIKELY(p.first == Port::Output)) {
177        report_fatal_error(name + " is not an input stream set");
178    }
179    Value * const addr = mKernel->getStreamSetInputBufferPtr(p.second);
180    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
181    return buf->getStreamBlockPtr(this, getStreamHandle(name), addr, streamIndex, true);
182}
183
184Value * KernelBuilder::loadInputStreamBlock(const std::string & name, Value * streamIndex) {
185    return CreateBlockAlignedLoad(getInputStreamBlockPtr(name, streamIndex));
186}
187
188Value * KernelBuilder::getInputStreamPackPtr(const std::string & name, Value * streamIndex, Value * packIndex) {
189    const Kernel::StreamPort p = mKernel->getStreamPort(name);
190    if (LLVM_UNLIKELY(p.first == Port::Output)) {
191        report_fatal_error(name + " is not an input stream set");
192    }
193    Value * const addr = mKernel->getStreamSetInputBufferPtr(p.second);
194    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
195    return buf->getStreamPackPtr(this, getStreamHandle(name), addr, streamIndex, packIndex, true);
196}
197
198Value * KernelBuilder::loadInputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex) {
199    return CreateBlockAlignedLoad(getInputStreamPackPtr(name, streamIndex, packIndex));
200}
201
202Value * KernelBuilder::getInputStreamSetCount(const std::string & name) {
203    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
204    return buf->getStreamSetCount(this, getStreamHandle(name));
205}
206
207Value * KernelBuilder::getAdjustedInputStreamBlockPtr(Value * blockAdjustment, const std::string & name, Value * streamIndex) {
208    const Kernel::StreamPort p = mKernel->getStreamPort(name);
209    if (LLVM_UNLIKELY(p.first == Port::Output)) {
210        report_fatal_error(name + " is not an input stream set");
211    }
212    Value * const addr = mKernel->getStreamSetInputBufferPtr(p.second);
213    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
214    return buf->getStreamBlockPtr(this, getStreamHandle(name), CreateGEP(addr, blockAdjustment), streamIndex, true);
215}
216
217Value * KernelBuilder::getOutputStreamPtr(const std::string & name, Value * const blockIndex) {
218//    Value * const blockIndex = computeBlockIndex(getProducedItemCount(name));
219    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
220    return buf->getBlockAddress(this, getStreamHandle(name), blockIndex);
221}
222
223Value * KernelBuilder::getOutputStreamBlockPtr(const std::string & name, Value * streamIndex) {
224    const Kernel::StreamPort p = mKernel->getStreamPort(name);
225    if (LLVM_UNLIKELY(p.first == Port::Input)) {
226        report_fatal_error(name + " is not an output stream set");
227    }
228    Value * addr = mKernel->getStreamSetOutputBufferPtr(p.second);
229    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
230    return buf->getStreamBlockPtr(this, getStreamHandle(name), addr, streamIndex, true);
231}
232
233StoreInst * KernelBuilder::storeOutputStreamBlock(const std::string & name, Value * streamIndex, Value * toStore) {
234    return CreateBlockAlignedStore(toStore, getOutputStreamBlockPtr(name, streamIndex));
235}
236
237Value * KernelBuilder::getOutputStreamPackPtr(const std::string & name, Value * streamIndex, Value * packIndex) {
238    const Kernel::StreamPort p = mKernel->getStreamPort(name);
239    if (LLVM_UNLIKELY(p.first == Port::Input)) {
240        report_fatal_error(name + " is not an output stream set");
241    }
242    Value * addr = mKernel->getStreamSetOutputBufferPtr(p.second);
243    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
244    return buf->getStreamPackPtr(this, getStreamHandle(name), addr, streamIndex, packIndex, false);
245}
246
247StoreInst * KernelBuilder::storeOutputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex, Value * toStore) {
248    return CreateBlockAlignedStore(toStore, getOutputStreamPackPtr(name, streamIndex, packIndex));
249}
250
251Value * KernelBuilder::getOutputStreamSetCount(const std::string & name) {
252    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
253    return buf->getStreamSetCount(this, getStreamHandle(name));
254}
255
256Value * KernelBuilder::getRawInputPointer(const std::string & name, Value * absolutePosition) {
257    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
258    return buf->getRawItemPointer(this, getStreamHandle(name), absolutePosition);
259}
260
261Value * KernelBuilder::getRawOutputPointer(const std::string & name, Value * absolutePosition) {
262    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
263    return buf->getRawItemPointer(this, getStreamHandle(name), absolutePosition);
264}
265
266Value * KernelBuilder::getBaseAddress(const std::string & name) {
267    return mKernel->getAnyStreamSetBuffer(name)->getBaseAddress(this, getStreamHandle(name));
268}
269
270void KernelBuilder::setBaseAddress(const std::string & name, Value * const addr) {
271    return mKernel->getAnyStreamSetBuffer(name)->setBaseAddress(this, getStreamHandle(name), addr);
272}
273
274Value * KernelBuilder::getBufferedSize(const std::string & name) {
275    return mKernel->getAnyStreamSetBuffer(name)->getBufferedSize(this, getStreamHandle(name));
276}
277
278void KernelBuilder::setBufferedSize(const std::string & name, Value * size) {
279    mKernel->getAnyStreamSetBuffer(name)->setBufferedSize(this, getStreamHandle(name), size);
280}
281
282
283Value * KernelBuilder::getCapacity(const std::string & name) {
284    return mKernel->getAnyStreamSetBuffer(name)->getCapacity(this, getStreamHandle(name));
285}
286
287void KernelBuilder::setCapacity(const std::string & name, Value * c) {
288    mKernel->getAnyStreamSetBuffer(name)->setCapacity(this, getStreamHandle(name), c);
289}
290
291   
292CallInst * KernelBuilder::createDoSegmentCall(const std::vector<Value *> & args) {
293//    Function * const doSegment = mKernel->getDoSegmentFunction(getModule());
294//    assert (doSegment->getArgumentList().size() == args.size());
295//    return CreateCall(doSegment, args);
296    return mKernel->makeDoSegmentCall(*this, args);
297}
298
299Value * KernelBuilder::getAccumulator(const std::string & accumName) {
300    auto results = mKernel->mOutputScalarResult;
301    if (LLVM_UNLIKELY(results == nullptr)) {
302        report_fatal_error("Cannot get accumulator " + accumName + " until " + mKernel->getName() + " has terminated.");
303    }
304    const auto & outputs = mKernel->getScalarOutputs();
305    const auto n = outputs.size();
306    if (LLVM_UNLIKELY(n == 0)) {
307        report_fatal_error(mKernel->getName() + " has no output scalars.");
308    } else {
309        for (unsigned i = 0; i < n; ++i) {
310            const Binding & b = outputs[i];
311            if (b.getName() == accumName) {
312                if (n == 1) {
313                    return results;
314                } else {
315                    return CreateExtractValue(results, {i});
316                }
317            }
318        }
319        report_fatal_error(mKernel->getName() + " has no output scalar named " + accumName);
320    }
321}
322
323BasicBlock * KernelBuilder::CreateConsumerWait() {
324    const auto consumers = mKernel->getStreamOutputs();
325    BasicBlock * const entry = GetInsertBlock();
326    if (consumers.empty()) {
327        return entry;
328    } else {
329        Function * const parent = entry->getParent();
330        IntegerType * const sizeTy = getSizeTy();
331        ConstantInt * const zero = getInt32(0);
332        ConstantInt * const one = getInt32(1);
333        ConstantInt * const size0 = getSize(0);
334
335        Value * const segNo = acquireLogicalSegmentNo();
336        const auto n = consumers.size();
337        BasicBlock * load[n + 1];
338        BasicBlock * wait[n];
339        for (unsigned i = 0; i < n; ++i) {
340            load[i] = BasicBlock::Create(getContext(), consumers[i].getName() + "Load", parent);
341            wait[i] = BasicBlock::Create(getContext(), consumers[i].getName() + "Wait", parent);
342        }
343        load[n] = BasicBlock::Create(getContext(), "Resume", parent);
344        CreateBr(load[0]);
345        for (unsigned i = 0; i < n; ++i) {
346
347            SetInsertPoint(load[i]);
348            Value * const outputConsumers = getConsumerLock(consumers[i].getName());
349
350            Value * const consumerCount = CreateLoad(CreateGEP(outputConsumers, {zero, zero}));
351            Value * const consumerPtr = CreateLoad(CreateGEP(outputConsumers, {zero, one}));
352            Value * const noConsumers = CreateICmpEQ(consumerCount, size0);
353            CreateUnlikelyCondBr(noConsumers, load[i + 1], wait[i]);
354
355            SetInsertPoint(wait[i]);
356            PHINode * const consumerPhi = CreatePHI(sizeTy, 2);
357            consumerPhi->addIncoming(size0, load[i]);
358
359            Value * const conSegPtr = CreateLoad(CreateGEP(consumerPtr, consumerPhi));
360            Value * const processedSegmentCount = CreateAtomicLoadAcquire(conSegPtr);
361            Value * const ready = CreateICmpEQ(segNo, processedSegmentCount);
362            assert (ready->getType() == getInt1Ty());
363            Value * const nextConsumerIdx = CreateAdd(consumerPhi, CreateZExt(ready, sizeTy));
364            consumerPhi->addIncoming(nextConsumerIdx, wait[i]);
365            Value * const next = CreateICmpEQ(nextConsumerIdx, consumerCount);
366            CreateCondBr(next, load[i + 1], wait[i]);
367        }
368
369        BasicBlock * const exit = load[n];
370        SetInsertPoint(exit);
371        return exit;
372    }
373}
374
375}
Note: See TracBrowser for help on using the repository browser.