source: icGREP/icgrep-devel/icgrep/kernels/kernel_builder.cpp @ 5522

Last change on this file since 5522 was 5522, checked in by cameron, 2 years ago

Fixes for processing rates; multiblock kernel builder

File size: 14.6 KB
Line 
1#include "kernel_builder.h"
2#include <kernels/kernel.h>
3#include <kernels/streamset.h>
4
5using namespace llvm;
6using namespace parabix;
7
8using Value = Value;
9
10namespace kernel {
11
12Value * KernelBuilder::getScalarFieldPtr(llvm::Value * instance, Value * const index) {
13    assert (instance);
14    CreateAssert(instance, "instance cannot be null!");
15    return CreateGEP(instance, {getInt32(0), index});
16}
17
18Value * KernelBuilder::getScalarFieldPtr(llvm::Value * instance, const std::string & fieldName) {
19    return getScalarFieldPtr(instance, getInt32(mKernel->getScalarIndex(fieldName)));
20}
21
22llvm::Value * KernelBuilder::getScalarFieldPtr(llvm::Value * index) {
23    return getScalarFieldPtr(mKernel->getInstance(), index);
24}
25
26llvm::Value *KernelBuilder:: getScalarFieldPtr(const std::string & fieldName) {
27    return getScalarFieldPtr(mKernel->getInstance(), fieldName);
28}
29
30Value * KernelBuilder::getScalarField(const std::string & fieldName) {
31    return CreateLoad(getScalarFieldPtr(fieldName), fieldName);
32}
33
34void KernelBuilder::setScalarField(const std::string & fieldName, Value * value) {
35    CreateStore(value, getScalarFieldPtr(fieldName));
36}
37
38Value * KernelBuilder::getStreamSetBufferPtr(const std::string & name) {
39    Value * const ptr = getScalarField(name + Kernel::BUFFER_PTR_SUFFIX);
40    CreateAssert(ptr, name + " cannot be null!");
41    return ptr;
42}
43
44LoadInst * KernelBuilder::acquireLogicalSegmentNo() {
45    return CreateAtomicLoadAcquire(getScalarFieldPtr(Kernel::LOGICAL_SEGMENT_NO_SCALAR));
46}
47
48void KernelBuilder::releaseLogicalSegmentNo(Value * nextSegNo) {
49    CreateAtomicStoreRelease(nextSegNo, getScalarFieldPtr(Kernel::LOGICAL_SEGMENT_NO_SCALAR));
50}
51
52Value * KernelBuilder::getProducedItemCount(const std::string & name, Value * doFinal) {
53    Kernel::Port port; unsigned index;
54    std::tie(port, index) = mKernel->getStreamPort(name);
55    assert (port == Kernel::Port::Output);
56    const auto & rate = mKernel->getStreamOutput(index).rate;
57    const auto & refSet = rate.referenceStreamSet();
58    if ((refSet != name) && rate.isExact()) {
59        Value * principalCount;
60        std::tie(port, index) = mKernel->getStreamPort(refSet);
61        if (port == Kernel::Port::Input) {
62            principalCount = getProcessedItemCount(refSet);
63        } else {
64            principalCount = getProducedItemCount(refSet);
65        }
66        return rate.CreateRatioCalculation(this, principalCount, doFinal);
67    }
68    return getScalarField(name + Kernel::PRODUCED_ITEM_COUNT_SUFFIX);
69}
70
71Value * KernelBuilder::getProcessedItemCount(const std::string & name) {
72    Kernel::Port port; unsigned index;
73    std::tie(port, index) = mKernel->getStreamPort(name);
74    assert (port == Kernel::Port::Input);
75    const auto & rate = mKernel->getStreamInput(index).rate;
76    const auto & refSet = rate.referenceStreamSet();
77    if ((refSet != name) && rate.isExact()) {
78        Value * const principalCount = getProcessedItemCount(refSet);
79        return rate.CreateRatioCalculation(this, principalCount);
80    }
81    return getScalarField(name + Kernel::PROCESSED_ITEM_COUNT_SUFFIX);
82}
83
84Value * KernelBuilder::getAvailableItemCount(const std::string & name) {
85    const auto & inputs = mKernel->getStreamInputs();
86    for (unsigned i = 0; i < inputs.size(); ++i) {
87        if (inputs[i].name == name) {
88            return mKernel->getAvailableItemCount(i);
89        }
90    }
91    return nullptr;
92}
93
94Value * KernelBuilder::getConsumedItemCount(const std::string & name) {
95    return getScalarField(name + Kernel::CONSUMED_ITEM_COUNT_SUFFIX);
96}
97
98void KernelBuilder::setProducedItemCount(const std::string & name, Value * value) {
99    setScalarField(name + Kernel::PRODUCED_ITEM_COUNT_SUFFIX, value);
100}
101
102void KernelBuilder::setProcessedItemCount(const std::string & name, Value * value) {
103    setScalarField(name + Kernel::PROCESSED_ITEM_COUNT_SUFFIX, value);
104}
105
106void KernelBuilder::setConsumedItemCount(const std::string & name, Value * value) {
107    setScalarField(name + Kernel::CONSUMED_ITEM_COUNT_SUFFIX, value);
108}
109
110Value * KernelBuilder::getTerminationSignal() {
111    return getScalarField(Kernel::TERMINATION_SIGNAL);
112}
113
114void KernelBuilder::setTerminationSignal() {
115    setScalarField(Kernel::TERMINATION_SIGNAL, getTrue());
116}
117
118Value * KernelBuilder::getLinearlyAccessibleItems(const std::string & name, Value * fromPosition) {
119    Kernel::Port port; unsigned index;
120    std::tie(port, index) = mKernel->getStreamPort(name);
121    if (port == Kernel::Port::Input) {
122        const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
123        return buf->getLinearlyAccessibleItems(this, getStreamSetBufferPtr(name), fromPosition);
124    }
125    else {
126        const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
127        return buf->getLinearlyAccessibleItems(this, getStreamSetBufferPtr(name), fromPosition);
128    }
129}
130
131Value * KernelBuilder::getLinearlyAccessibleBlocks(const std::string & name, Value * fromBlock) {
132    Kernel::Port port; unsigned index;
133    std::tie(port, index) = mKernel->getStreamPort(name);
134    if (port == Kernel::Port::Input) {
135        const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
136        return buf->getLinearlyAccessibleBlocks(this, getStreamSetBufferPtr(name), fromBlock);
137    } else {
138        const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
139        return buf->getLinearlyAccessibleBlocks(this, getStreamSetBufferPtr(name), fromBlock);
140    }
141}
142
143Value * KernelBuilder::getLinearlyWritableItems(const std::string & name, Value * fromPosition) {
144    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
145    return buf->getLinearlyWritableItems(this, getStreamSetBufferPtr(name), fromPosition);
146}
147
148Value * KernelBuilder::getLinearlyWritableBlocks(const std::string & name, Value * fromBlock) {
149    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
150    return buf->getLinearlyWritableBlocks(this, getStreamSetBufferPtr(name), fromBlock);
151}
152
153Value * KernelBuilder::getConsumerLock(const std::string & name) {
154    return getScalarField(name + Kernel::CONSUMER_SUFFIX);
155}
156
157void KernelBuilder::setConsumerLock(const std::string & name, Value * value) {
158    setScalarField(name + Kernel::CONSUMER_SUFFIX, value);
159}
160
161inline Value * KernelBuilder::computeBlockIndex(Value * itemCount) {
162    const auto divisor = getBitBlockWidth();
163    if (LLVM_LIKELY((divisor & (divisor - 1)) == 0)) {
164        return CreateLShr(itemCount, std::log2(divisor));
165    } else {
166        return CreateUDiv(itemCount, getSize(divisor));
167    }
168}
169
170Value * KernelBuilder::getInputStreamBlockPtr(const std::string & name, Value * streamIndex) {
171    Value * const blockIndex = computeBlockIndex(getProcessedItemCount(name));
172    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
173    return buf->getStreamBlockPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, true);
174}
175
176Value * KernelBuilder::loadInputStreamBlock(const std::string & name, Value * streamIndex) {
177    return CreateBlockAlignedLoad(getInputStreamBlockPtr(name, streamIndex));
178}
179
180Value * KernelBuilder::getInputStreamPackPtr(const std::string & name, Value * streamIndex, Value * packIndex) {
181    Value * const blockIndex = computeBlockIndex(getProcessedItemCount(name));
182    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
183    return buf->getStreamPackPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, packIndex, true);
184}
185
186Value * KernelBuilder::loadInputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex) {
187    return CreateBlockAlignedLoad(getInputStreamPackPtr(name, streamIndex, packIndex));
188}
189
190Value * KernelBuilder::getInputStreamSetCount(const std::string & name) {
191    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
192    return buf->getStreamSetCount(this, getStreamSetBufferPtr(name));
193}
194
195Value * KernelBuilder::getAdjustedInputStreamBlockPtr(Value * blockAdjustment, const std::string & name, Value * streamIndex) {
196    Value * const blockIndex = CreateAdd(computeBlockIndex(getProcessedItemCount(name)), blockAdjustment);
197    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
198    return buf->getStreamBlockPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, true);
199}
200
201Value * KernelBuilder::getOutputStreamBlockPtr(const std::string & name, Value * streamIndex) {
202    Value * const blockIndex = computeBlockIndex(getProducedItemCount(name));
203    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
204    return buf->getStreamBlockPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, false);
205}
206
207StoreInst * KernelBuilder::storeOutputStreamBlock(const std::string & name, Value * streamIndex, Value * toStore) {
208    return CreateBlockAlignedStore(toStore, getOutputStreamBlockPtr(name, streamIndex));
209}
210
211Value * KernelBuilder::getOutputStreamPackPtr(const std::string & name, Value * streamIndex, Value * packIndex) {
212    Value * const blockIndex = computeBlockIndex(getProducedItemCount(name));
213    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
214    return buf->getStreamPackPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, packIndex, false);
215}
216
217StoreInst * KernelBuilder::storeOutputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex, Value * toStore) {
218    return CreateBlockAlignedStore(toStore, getOutputStreamPackPtr(name, streamIndex, packIndex));
219}
220
221Value * KernelBuilder::getOutputStreamSetCount(const std::string & name) {
222    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
223    return buf->getStreamSetCount(this, getStreamSetBufferPtr(name));
224}
225
226Value * KernelBuilder::getRawInputPointer(const std::string & name, Value * streamIndex, Value * absolutePosition) {
227    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
228    return buf->getRawItemPointer(this, getStreamSetBufferPtr(name), streamIndex, absolutePosition);
229}
230
231Value * KernelBuilder::getRawOutputPointer(const std::string & name, Value * streamIndex, Value * absolutePosition) {
232    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
233    return buf->getRawItemPointer(this, getStreamSetBufferPtr(name), streamIndex, absolutePosition);
234}
235
236Value * KernelBuilder::getBaseAddress(const std::string & name) {
237    return mKernel->getAnyStreamSetBuffer(name)->getBaseAddress(this, getStreamSetBufferPtr(name));
238}
239
240void KernelBuilder::setBaseAddress(const std::string & name, Value * const addr) {
241    return mKernel->getAnyStreamSetBuffer(name)->setBaseAddress(this, getStreamSetBufferPtr(name), addr);
242}
243
244Value * KernelBuilder::getBufferedSize(const std::string & name) {
245    return mKernel->getAnyStreamSetBuffer(name)->getBufferedSize(this, getStreamSetBufferPtr(name));
246}
247
248void KernelBuilder::setBufferedSize(const std::string & name, Value * size) {
249    mKernel->getAnyStreamSetBuffer(name)->setBufferedSize(this, getStreamSetBufferPtr(name), size);
250}
251
252
253Value * KernelBuilder::getCapacity(const std::string & name) {
254    return mKernel->getAnyStreamSetBuffer(name)->getCapacity(this, getStreamSetBufferPtr(name));
255}
256
257void KernelBuilder::setCapacity(const std::string & name, Value * c) {
258    mKernel->getAnyStreamSetBuffer(name)->setCapacity(this, getStreamSetBufferPtr(name), c);
259}
260
261   
262CallInst * KernelBuilder::createDoSegmentCall(const std::vector<Value *> & args) {
263    Function * const doSegment = mKernel->getDoSegmentFunction(getModule());
264    assert (doSegment->getArgumentList().size() == args.size());
265    return CreateCall(doSegment, args);
266}
267
268Value * KernelBuilder::getAccumulator(const std::string & accumName) {
269    auto results = mKernel->mOutputScalarResult;
270    if (LLVM_UNLIKELY(results == nullptr)) {
271        report_fatal_error("Cannot get accumulator " + accumName + " until " + mKernel->getName() + " has terminated.");
272    }
273    const auto & outputs = mKernel->getScalarOutputs();
274    const auto n = outputs.size();
275    if (LLVM_UNLIKELY(n == 0)) {
276        report_fatal_error(mKernel->getName() + " has no output scalars.");
277    } else {
278        for (unsigned i = 0; i < n; ++i) {
279            const Binding & b = outputs[i];
280            if (b.name == accumName) {
281                if (n == 1) {
282                    return results;
283                } else {
284                    return CreateExtractValue(results, {i});
285                }
286            }
287        }
288        report_fatal_error(mKernel->getName() + " has no output scalar named " + accumName);
289    }
290}
291
292BasicBlock * KernelBuilder::CreateConsumerWait() {
293    const auto consumers = mKernel->getStreamOutputs();
294    BasicBlock * const entry = GetInsertBlock();
295    if (consumers.empty()) {
296        return entry;
297    } else {
298        Function * const parent = entry->getParent();
299        IntegerType * const sizeTy = getSizeTy();
300        ConstantInt * const zero = getInt32(0);
301        ConstantInt * const one = getInt32(1);
302        ConstantInt * const size0 = getSize(0);
303
304        Value * const segNo = acquireLogicalSegmentNo();
305        const auto n = consumers.size();
306        BasicBlock * load[n + 1];
307        BasicBlock * wait[n];
308        for (unsigned i = 0; i < n; ++i) {
309            load[i] = BasicBlock::Create(getContext(), consumers[i].name + "Load", parent);
310            wait[i] = BasicBlock::Create(getContext(), consumers[i].name + "Wait", parent);
311        }
312        load[n] = BasicBlock::Create(getContext(), "Resume", parent);
313        CreateBr(load[0]);
314        for (unsigned i = 0; i < n; ++i) {
315
316            SetInsertPoint(load[i]);
317            Value * const outputConsumers = getConsumerLock(consumers[i].name);
318
319            Value * const consumerCount = CreateLoad(CreateGEP(outputConsumers, {zero, zero}));
320            Value * const consumerPtr = CreateLoad(CreateGEP(outputConsumers, {zero, one}));
321            Value * const noConsumers = CreateICmpEQ(consumerCount, size0);
322            CreateUnlikelyCondBr(noConsumers, load[i + 1], wait[i]);
323
324            SetInsertPoint(wait[i]);
325            PHINode * const consumerPhi = CreatePHI(sizeTy, 2);
326            consumerPhi->addIncoming(size0, load[i]);
327
328            Value * const conSegPtr = CreateLoad(CreateGEP(consumerPtr, consumerPhi));
329            Value * const processedSegmentCount = CreateAtomicLoadAcquire(conSegPtr);
330            Value * const ready = CreateICmpEQ(segNo, processedSegmentCount);
331            assert (ready->getType() == getInt1Ty());
332            Value * const nextConsumerIdx = CreateAdd(consumerPhi, CreateZExt(ready, sizeTy));
333            consumerPhi->addIncoming(nextConsumerIdx, wait[i]);
334            Value * const next = CreateICmpEQ(nextConsumerIdx, consumerCount);
335            CreateCondBr(next, load[i + 1], wait[i]);
336        }
337
338        BasicBlock * const exit = load[n];
339        SetInsertPoint(exit);
340        return exit;
341    }
342}
343
344}
Note: See TracBrowser for help on using the repository browser.