source: icGREP/icgrep-devel/icgrep/kernels/kernel_builder.cpp @ 5647

Last change on this file since 5647 was 5647, checked in by nmedfort, 2 years ago

Minor bug fixes and removal of inadvertent check in for StreamSet?.cpp/h

File size: 15.6 KB
Line 
1#include "kernel_builder.h"
2#include <toolchain/toolchain.h>
3#include <kernels/kernel.h>
4#include <kernels/streamset.h>
5
6using namespace llvm;
7using namespace parabix;
8
9using Value = Value;
10
11namespace kernel {
12
13Value * KernelBuilder::getScalarFieldPtr(llvm::Value * instance, Value * const index) {
14    assert (instance);
15    CreateAssert(instance, "instance cannot be null!");
16    return CreateGEP(instance, {getInt32(0), index});
17}
18
19Value * KernelBuilder::getScalarFieldPtr(llvm::Value * instance, const std::string & fieldName) {
20    return getScalarFieldPtr(instance, getInt32(mKernel->getScalarIndex(fieldName)));
21}
22
23llvm::Value * KernelBuilder::getScalarFieldPtr(llvm::Value * index) {
24    return getScalarFieldPtr(mKernel->getInstance(), index);
25}
26
27llvm::Value *KernelBuilder:: getScalarFieldPtr(const std::string & fieldName) {
28    return getScalarFieldPtr(mKernel->getInstance(), fieldName);
29}
30
31Value * KernelBuilder::getScalarField(const std::string & fieldName) {
32    return CreateLoad(getScalarFieldPtr(fieldName), fieldName);
33}
34
35void KernelBuilder::setScalarField(const std::string & fieldName, Value * value) {
36    CreateStore(value, getScalarFieldPtr(fieldName));
37}
38
39Value * KernelBuilder::getStreamSetBufferPtr(const std::string & name) {
40    Value * const ptr = getScalarField(name + Kernel::BUFFER_PTR_SUFFIX);
41    CreateAssert(ptr, name + " cannot be null!");
42    return ptr;
43}
44
45LoadInst * KernelBuilder::acquireLogicalSegmentNo() {
46    return CreateAtomicLoadAcquire(getScalarFieldPtr(Kernel::LOGICAL_SEGMENT_NO_SCALAR));
47}
48
49void KernelBuilder::releaseLogicalSegmentNo(Value * nextSegNo) {
50    CreateAtomicStoreRelease(nextSegNo, getScalarFieldPtr(Kernel::LOGICAL_SEGMENT_NO_SCALAR));
51}
52
53Value * KernelBuilder::getProducedItemCount(const std::string & name, Value * doFinal) {
54    Kernel::Port port; unsigned index;
55    std::tie(port, index) = mKernel->getStreamPort(name);
56    assert (port == Kernel::Port::Output);
57    const auto & rate = mKernel->getStreamOutput(index).rate;
58    const auto & refSet = rate.referenceStreamSet();
59    if ((refSet != name) && rate.isExact()) {
60        Value * principalCount;
61        std::tie(port, index) = mKernel->getStreamPort(refSet);
62        if (port == Kernel::Port::Input) {
63            principalCount = getProcessedItemCount(refSet);
64        } else {
65            principalCount = getProducedItemCount(refSet);
66        }
67        return rate.CreateRatioCalculation(this, principalCount, doFinal);
68    }
69    return getScalarField(name + Kernel::PRODUCED_ITEM_COUNT_SUFFIX);
70}
71
72Value * KernelBuilder::getProcessedItemCount(const std::string & name) {
73    Kernel::Port port; unsigned index;
74    std::tie(port, index) = mKernel->getStreamPort(name);
75    assert (port == Kernel::Port::Input);
76    const auto & rate = mKernel->getStreamInput(index).rate;
77    const auto & refSet = rate.referenceStreamSet();
78    if ((refSet != name) && rate.isExact()) {
79        Value * const principalCount = getProcessedItemCount(refSet);
80        return rate.CreateRatioCalculation(this, principalCount);
81    }
82    return getScalarField(name + Kernel::PROCESSED_ITEM_COUNT_SUFFIX);
83}
84
85Value * KernelBuilder::getAvailableItemCount(const std::string & name) {
86    const auto & inputs = mKernel->getStreamInputs();
87    for (unsigned i = 0; i < inputs.size(); ++i) {
88        if (inputs[i].name == name) {
89            return mKernel->getAvailableItemCount(i);
90        }
91    }
92    return nullptr;
93}
94
95Value * KernelBuilder::getConsumedItemCount(const std::string & name) {
96    return getScalarField(name + Kernel::CONSUMED_ITEM_COUNT_SUFFIX);
97}
98
99void KernelBuilder::setProducedItemCount(const std::string & name, Value * value) {
100    setScalarField(name + Kernel::PRODUCED_ITEM_COUNT_SUFFIX, value);
101    if (codegen::DebugOptionIsSet(codegen::TraceCounts)) {
102        CallPrintIntToStderr(mKernel->getName() + ": " + name + "_producedItemCount", value);
103    }
104}
105
106void KernelBuilder::setProcessedItemCount(const std::string & name, Value * value) {
107    setScalarField(name + Kernel::PROCESSED_ITEM_COUNT_SUFFIX, value);
108    if (codegen::DebugOptionIsSet(codegen::TraceCounts)) {
109        CallPrintIntToStderr(mKernel->getName() + ": " + name + "_processedItemCount", value);
110    }
111}
112
113void KernelBuilder::setConsumedItemCount(const std::string & name, Value * value) {
114    setScalarField(name + Kernel::CONSUMED_ITEM_COUNT_SUFFIX, value);
115    if (codegen::DebugOptionIsSet(codegen::TraceCounts)) {
116        CallPrintIntToStderr(mKernel->getName() + ": " + name + "_consumedItemCount", value);
117    }
118}
119
120Value * KernelBuilder::getTerminationSignal() {
121    return getScalarField(Kernel::TERMINATION_SIGNAL);
122}
123
124void KernelBuilder::setTerminationSignal() {
125    setScalarField(Kernel::TERMINATION_SIGNAL, getTrue());
126    if (codegen::DebugOptionIsSet(codegen::TraceCounts)) {
127        CallPrintIntToStderr(mKernel->getName() + ": setTerminationSignal", getTrue());
128    }
129}
130
131Value * KernelBuilder::getLinearlyAccessibleItems(const std::string & name, Value * fromPosition) {
132    Kernel::Port port; unsigned index;
133    std::tie(port, index) = mKernel->getStreamPort(name);
134    const StreamSetBuffer * buf = nullptr;
135    if (port == Kernel::Port::Input) {       
136        const auto lookAhead = mKernel->getLookAhead(index);
137        if (LLVM_UNLIKELY(lookAhead != 0)) {
138            fromPosition = CreateAdd(ConstantInt::get(fromPosition->getType(), lookAhead), fromPosition);
139        }
140        buf = mKernel->getInputStreamSetBuffer(name);
141    } else {
142        buf = mKernel->getOutputStreamSetBuffer(name);
143    }
144    assert (buf);
145    return buf->getLinearlyAccessibleItems(this, getStreamSetBufferPtr(name), fromPosition);
146}
147
148Value * KernelBuilder::getLinearlyAccessibleBlocks(const std::string & name, Value * fromBlock) {
149    Kernel::Port port; unsigned index;
150    std::tie(port, index) = mKernel->getStreamPort(name);
151    const StreamSetBuffer * buf = nullptr;
152    if (port == Kernel::Port::Input) {       
153        const auto lookAhead = mKernel->getLookAhead(index);
154        if (LLVM_UNLIKELY(lookAhead != 0)) {
155            const auto blocksAhead = (lookAhead + getBitBlockWidth() - 1) / getBitBlockWidth();
156            fromBlock = CreateAdd(ConstantInt::get(fromBlock->getType(), blocksAhead), fromBlock);
157        }
158        buf = mKernel->getInputStreamSetBuffer(name);
159    } else {
160        buf = mKernel->getOutputStreamSetBuffer(name);
161    }
162    assert (buf);
163    return buf->getLinearlyAccessibleBlocks(this, getStreamSetBufferPtr(name), fromBlock);
164}
165
166Value * KernelBuilder::getLinearlyWritableItems(const std::string & name, Value * fromPosition) {
167    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
168    return buf->getLinearlyWritableItems(this, getStreamSetBufferPtr(name), fromPosition);
169}
170
171Value * KernelBuilder::getLinearlyWritableBlocks(const std::string & name, Value * fromBlock) {
172    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
173    return buf->getLinearlyWritableBlocks(this, getStreamSetBufferPtr(name), fromBlock);
174}
175
176Value * KernelBuilder::getConsumerLock(const std::string & name) {
177    return getScalarField(name + Kernel::CONSUMER_SUFFIX);
178}
179
180void KernelBuilder::setConsumerLock(const std::string & name, Value * value) {
181    setScalarField(name + Kernel::CONSUMER_SUFFIX, value);
182}
183
184inline Value * KernelBuilder::computeBlockIndex(Value * itemCount) {
185    const auto divisor = getBitBlockWidth();
186    if (LLVM_LIKELY((divisor & (divisor - 1)) == 0)) {
187        return CreateLShr(itemCount, std::log2(divisor));
188    } else {
189        return CreateUDiv(itemCount, getSize(divisor));
190    }
191}
192
193Value * KernelBuilder::getInputStreamBlockPtr(const std::string & name, Value * streamIndex) {
194    Value * const blockIndex = computeBlockIndex(getProcessedItemCount(name));
195    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
196    return buf->getStreamBlockPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, true);
197}
198
199Value * KernelBuilder::loadInputStreamBlock(const std::string & name, Value * streamIndex) {
200    return CreateBlockAlignedLoad(getInputStreamBlockPtr(name, streamIndex));
201}
202
203Value * KernelBuilder::getInputStreamPackPtr(const std::string & name, Value * streamIndex, Value * packIndex) {
204    Value * const blockIndex = computeBlockIndex(getProcessedItemCount(name));
205    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
206    return buf->getStreamPackPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, packIndex, true);
207}
208
209Value * KernelBuilder::loadInputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex) {
210    return CreateBlockAlignedLoad(getInputStreamPackPtr(name, streamIndex, packIndex));
211}
212
213Value * KernelBuilder::getInputStreamSetCount(const std::string & name) {
214    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
215    return buf->getStreamSetCount(this, getStreamSetBufferPtr(name));
216}
217
218Value * KernelBuilder::getAdjustedInputStreamBlockPtr(Value * blockAdjustment, const std::string & name, Value * streamIndex) {
219    Value * const blockIndex = CreateAdd(computeBlockIndex(getProcessedItemCount(name)), blockAdjustment);
220    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
221    return buf->getStreamBlockPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, true);
222}
223
224Value * KernelBuilder::getOutputStreamBlockPtr(const std::string & name, Value * streamIndex) {
225    Value * const blockIndex = computeBlockIndex(getProducedItemCount(name));
226    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
227    return buf->getStreamBlockPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, false);
228}
229
230StoreInst * KernelBuilder::storeOutputStreamBlock(const std::string & name, Value * streamIndex, Value * toStore) {
231    return CreateBlockAlignedStore(toStore, getOutputStreamBlockPtr(name, streamIndex));
232}
233
234Value * KernelBuilder::getOutputStreamPackPtr(const std::string & name, Value * streamIndex, Value * packIndex) {
235    Value * const blockIndex = computeBlockIndex(getProducedItemCount(name));
236    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
237    return buf->getStreamPackPtr(this, getStreamSetBufferPtr(name), streamIndex, blockIndex, packIndex, false);
238}
239
240StoreInst * KernelBuilder::storeOutputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex, Value * toStore) {
241    return CreateBlockAlignedStore(toStore, getOutputStreamPackPtr(name, streamIndex, packIndex));
242}
243
244Value * KernelBuilder::getOutputStreamSetCount(const std::string & name) {
245    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
246    return buf->getStreamSetCount(this, getStreamSetBufferPtr(name));
247}
248
249Value * KernelBuilder::getRawInputPointer(const std::string & name, Value * streamIndex, Value * absolutePosition) {
250    const StreamSetBuffer * const buf = mKernel->getInputStreamSetBuffer(name);
251    return buf->getRawItemPointer(this, getStreamSetBufferPtr(name), streamIndex, absolutePosition);
252}
253
254Value * KernelBuilder::getRawOutputPointer(const std::string & name, Value * streamIndex, Value * absolutePosition) {
255    const StreamSetBuffer * const buf = mKernel->getOutputStreamSetBuffer(name);
256    return buf->getRawItemPointer(this, getStreamSetBufferPtr(name), streamIndex, absolutePosition);
257}
258
259Value * KernelBuilder::getBaseAddress(const std::string & name) {
260    return mKernel->getAnyStreamSetBuffer(name)->getBaseAddress(this, getStreamSetBufferPtr(name));
261}
262
263void KernelBuilder::setBaseAddress(const std::string & name, Value * const addr) {
264    return mKernel->getAnyStreamSetBuffer(name)->setBaseAddress(this, getStreamSetBufferPtr(name), addr);
265}
266
267Value * KernelBuilder::getBufferedSize(const std::string & name) {
268    return mKernel->getAnyStreamSetBuffer(name)->getBufferedSize(this, getStreamSetBufferPtr(name));
269}
270
271void KernelBuilder::setBufferedSize(const std::string & name, Value * size) {
272    mKernel->getAnyStreamSetBuffer(name)->setBufferedSize(this, getStreamSetBufferPtr(name), size);
273}
274
275
276Value * KernelBuilder::getCapacity(const std::string & name) {
277    return mKernel->getAnyStreamSetBuffer(name)->getCapacity(this, getStreamSetBufferPtr(name));
278}
279
280void KernelBuilder::setCapacity(const std::string & name, Value * c) {
281    mKernel->getAnyStreamSetBuffer(name)->setCapacity(this, getStreamSetBufferPtr(name), c);
282}
283
284   
285CallInst * KernelBuilder::createDoSegmentCall(const std::vector<Value *> & args) {
286    Function * const doSegment = mKernel->getDoSegmentFunction(getModule());
287    assert (doSegment->getArgumentList().size() == args.size());
288    return CreateCall(doSegment, args);
289}
290
291Value * KernelBuilder::getAccumulator(const std::string & accumName) {
292    auto results = mKernel->mOutputScalarResult;
293    if (LLVM_UNLIKELY(results == nullptr)) {
294        report_fatal_error("Cannot get accumulator " + accumName + " until " + mKernel->getName() + " has terminated.");
295    }
296    const auto & outputs = mKernel->getScalarOutputs();
297    const auto n = outputs.size();
298    if (LLVM_UNLIKELY(n == 0)) {
299        report_fatal_error(mKernel->getName() + " has no output scalars.");
300    } else {
301        for (unsigned i = 0; i < n; ++i) {
302            const Binding & b = outputs[i];
303            if (b.name == accumName) {
304                if (n == 1) {
305                    return results;
306                } else {
307                    return CreateExtractValue(results, {i});
308                }
309            }
310        }
311        report_fatal_error(mKernel->getName() + " has no output scalar named " + accumName);
312    }
313}
314
315BasicBlock * KernelBuilder::CreateConsumerWait() {
316    const auto consumers = mKernel->getStreamOutputs();
317    BasicBlock * const entry = GetInsertBlock();
318    if (consumers.empty()) {
319        return entry;
320    } else {
321        Function * const parent = entry->getParent();
322        IntegerType * const sizeTy = getSizeTy();
323        ConstantInt * const zero = getInt32(0);
324        ConstantInt * const one = getInt32(1);
325        ConstantInt * const size0 = getSize(0);
326
327        Value * const segNo = acquireLogicalSegmentNo();
328        const auto n = consumers.size();
329        BasicBlock * load[n + 1];
330        BasicBlock * wait[n];
331        for (unsigned i = 0; i < n; ++i) {
332            load[i] = BasicBlock::Create(getContext(), consumers[i].name + "Load", parent);
333            wait[i] = BasicBlock::Create(getContext(), consumers[i].name + "Wait", parent);
334        }
335        load[n] = BasicBlock::Create(getContext(), "Resume", parent);
336        CreateBr(load[0]);
337        for (unsigned i = 0; i < n; ++i) {
338
339            SetInsertPoint(load[i]);
340            Value * const outputConsumers = getConsumerLock(consumers[i].name);
341
342            Value * const consumerCount = CreateLoad(CreateGEP(outputConsumers, {zero, zero}));
343            Value * const consumerPtr = CreateLoad(CreateGEP(outputConsumers, {zero, one}));
344            Value * const noConsumers = CreateICmpEQ(consumerCount, size0);
345            CreateUnlikelyCondBr(noConsumers, load[i + 1], wait[i]);
346
347            SetInsertPoint(wait[i]);
348            PHINode * const consumerPhi = CreatePHI(sizeTy, 2);
349            consumerPhi->addIncoming(size0, load[i]);
350
351            Value * const conSegPtr = CreateLoad(CreateGEP(consumerPtr, consumerPhi));
352            Value * const processedSegmentCount = CreateAtomicLoadAcquire(conSegPtr);
353            Value * const ready = CreateICmpEQ(segNo, processedSegmentCount);
354            assert (ready->getType() == getInt1Ty());
355            Value * const nextConsumerIdx = CreateAdd(consumerPhi, CreateZExt(ready, sizeTy));
356            consumerPhi->addIncoming(nextConsumerIdx, wait[i]);
357            Value * const next = CreateICmpEQ(nextConsumerIdx, consumerCount);
358            CreateCondBr(next, load[i + 1], wait[i]);
359        }
360
361        BasicBlock * const exit = load[n];
362        SetInsertPoint(exit);
363        return exit;
364    }
365}
366
367}
Note: See TracBrowser for help on using the repository browser.