source: icGREP/icgrep-devel/icgrep/kernels/streamset.cpp @ 6047

Last change on this file since 6047 was 6047, checked in by nmedfort, 17 months ago

Major refactoring of buffer types. Static buffers replace Circular and CircularCopyback?. External buffers unify Source/External?.

File size: 24.5 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5
6#include "streamset.h"
7#include <llvm/IR/Module.h>
8#include <llvm/Support/raw_ostream.h>
9#include <kernels/kernel.h>
10#include <kernels/kernel_builder.h>
11#include <toolchain/toolchain.h>
12#include <llvm/Support/Debug.h>
13#include <llvm/Support/Format.h>
14
15namespace llvm { class Constant; }
16namespace llvm { class Function; }
17
18using namespace parabix;
19using namespace llvm;
20using namespace IDISA;
21
22inline static bool is_power_2(const uint64_t n) {
23    return ((n & (n - 1)) == 0) && n;
24}
25
26Value * StreamSetBuffer::getStreamBlockPtr(IDISA::IDISA_Builder * const b, Value * const handle, Value * streamIndex, Value * blockIndex, const bool /* readOnly */) const {
27    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
28        Value * const count = getStreamSetCount(b, handle);
29        Value * const index = b->CreateZExtOrTrunc(streamIndex, count->getType());
30        Value * const cond = b->CreateICmpULT(index, count);
31        b->CreateAssert(cond, "out-of-bounds stream access");
32    }
33    return b->CreateGEP(getBaseAddress(b, handle), {blockIndex, streamIndex});
34}
35
36Value * StreamSetBuffer::getStreamPackPtr(IDISA::IDISA_Builder * const b, Value * const handle, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool /* readOnly */) const {
37    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
38        Value * const count = getStreamSetCount(b, handle);
39        Value * const index = b->CreateZExtOrTrunc(streamIndex, count->getType());
40        Value * const cond = b->CreateICmpULT(index, count);
41        b->CreateAssert(cond, "out-of-bounds stream access");
42    }
43    return b->CreateGEP(getBaseAddress(b, handle), {blockIndex, streamIndex, packIndex});
44}
45
46Value * StreamSetBuffer::getStreamSetCount(IDISA::IDISA_Builder * const b, Value *) const {
47    size_t count = 1;
48    if (isa<ArrayType>(mBaseType)) {
49        count = mBaseType->getArrayNumElements();
50    }
51    return b->getSize(count);
52}
53
54// External File Buffer
55void ExternalBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) {
56    PointerType * const ptrTy = getPointerType();
57    IntegerType * const sizeTy = b->getSizeTy();
58    StructType * const structTy = StructType::get(b->getContext(), {ptrTy, sizeTy});
59    Value * const handle = b->CreateCacheAlignedAlloca(structTy);
60    mStreamSetHandle = handle;
61    // If mExternalAddress is null, it must be set by a source kernel.
62    Value * ptr = nullptr;
63    Constant * size = nullptr;
64    if (mExternalAddress) {
65        ptr = b->CreatePointerBitCastOrAddrSpaceCast(mExternalAddress, ptrTy);
66        size = ConstantInt::getAllOnesValue(sizeTy);
67    } else {
68        ptr = ConstantPointerNull::get(ptrTy);
69        size = ConstantInt::getNullValue(sizeTy);
70    }
71    b->CreateStore(ptr, b->CreateGEP(handle, {b->getInt32(0), b->getInt32(BaseAddress)}));
72    b->CreateStore(size, b->CreateGEP(handle, {b->getInt32(0), b->getInt32(Capacity)}));
73}
74
75void ExternalBuffer::setBaseAddress(IDISA::IDISA_Builder * const b, llvm::Value * handle, llvm::Value * addr) const {
76    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
77        b->CreateAssert(handle, "handle cannot be null");
78    }
79    Value * const p = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(BaseAddress)});
80    Value * const ptr = b->CreatePointerBitCastOrAddrSpaceCast(addr, getPointerType());
81    b->CreateStore(ptr, p);
82}
83
84Value * ExternalBuffer::getBaseAddress(IDISA::IDISA_Builder * const b, Value * const handle) const {
85    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
86        b->CreateAssert(handle, "handle cannot be null");
87    }
88    Value * const p = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(BaseAddress)});
89    return b->CreateLoad(p);
90}
91
92Value * ExternalBuffer::getOverflowAddress(IDISA::IDISA_Builder * const /* b */, Value * const /* handle */) const {
93    report_fatal_error("getOverflowAddress is not supported by this buffer type");
94}
95
96void ExternalBuffer::setCapacity(IDISA::IDISA_Builder * const b, Value * const handle, Value * const capacity) const {
97    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
98        b->CreateAssert(handle, "handle cannot be null");
99    }
100    Value * const p = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(Capacity)});
101    b->CreateStore(capacity, p);
102}
103
104Value * ExternalBuffer::getCapacity(IDISA::IDISA_Builder * const b, Value * const handle) const {
105    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
106        b->CreateAssert(handle, "handle cannot be null");
107    }
108    Value * const p = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(Capacity)});
109    return b->CreateLoad(p);
110}
111
112void ExternalBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> &) const {
113    // this buffer is not responsible for free-ing th data associated with it
114}
115
116Value * ExternalBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const, Value *, Value *, Value * availItems, const bool reverse) const {
117    // All available items can be accessed.
118    return reverse ? ConstantInt::getAllOnesValue(availItems->getType()) : availItems;
119}
120
121Value * ExternalBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const, Value *, Value * fromPosition, Value *consumed, const bool reverse) const {
122    // Trust that the buffer is large enough to write any amount
123    return reverse ? fromPosition : ConstantInt::getAllOnesValue(fromPosition->getType());
124}
125
126/**
127 * @brief getRawItemPointer
128 *
129 * get a raw pointer the iN field at position absoluteItemPosition of the stream number streamIndex of the stream set.
130 * In the case of a stream whose fields are less than one byte (8 bits) in size, the pointer is to the containing byte.
131 * The type of the pointer is i8* for fields of 8 bits or less, otherwise iN* for N-bit fields.
132 */
133Value * ExternalBuffer::getRawItemPointer(IDISA::IDISA_Builder * const b, Value * const handle, Value * absolutePosition) const {
134    Value * ptr = getBaseAddress(b, handle);
135    Value * relativePosition = absolutePosition;
136    Type * const elemTy = mBaseType->getArrayElementType()->getVectorElementType();
137    const auto bw = elemTy->getPrimitiveSizeInBits();
138    assert (is_power_2(bw));
139    if (bw < 8) {
140        Constant * const fw = ConstantInt::get(relativePosition->getType(), 8 / bw);
141        if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
142            b->CreateAssertZero(b->CreateURem(absolutePosition, fw), "absolutePosition must be byte aligned");
143        }
144        relativePosition = b->CreateUDiv(relativePosition, fw);
145        ptr = b->CreatePointerCast(ptr, b->getInt8PtrTy());
146    } else {
147        ptr = b->CreatePointerCast(ptr, elemTy->getPointerTo());
148    }
149    return b->CreateGEP(ptr, relativePosition);
150}
151
152// Circular Buffer
153void StaticBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) {
154    assert (mCapacity > 0);
155    assert ("allocate buffer was called twice" && !mStreamSetHandle);
156    Type * const ty = getType();
157    const auto blocks = (mCapacity + mOverflow);
158    if (mAddressSpace == 0) {
159        Constant * size = ConstantExpr::getSizeOf(ty);
160        size = ConstantExpr::getMul(size, ConstantInt::get(size->getType(), blocks));
161        mStreamSetHandle = b->CreatePointerCast(b->CreateCacheAlignedMalloc(size), ty->getPointerTo());
162    } else {
163        mStreamSetHandle = b->CreateCacheAlignedAlloca(ty, b->getSize(blocks));
164    }
165}
166
167void StaticBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) const {
168    if (mAddressSpace == 0) {
169        b->CreateFree(mStreamSetHandle);
170    }
171}
172
173inline bool isCapacityGuaranteed(const Value * const index, const size_t capacity) {
174    if (capacity == 0) {
175        return true;
176    } else if (isa<ConstantInt>(index)) {
177        return cast<ConstantInt>(index)->getLimitedValue() < capacity;
178    }
179    return false;
180}
181
182Value * StaticBuffer::modByCapacity(IDISA::IDISA_Builder * const b, Value * const offset) const {
183    assert (offset->getType()->isIntegerTy());
184    if (isCapacityGuaranteed(offset, mCapacity)) {
185        return offset;
186    } else if (mCapacity == 1) {
187        return ConstantInt::getNullValue(offset->getType());
188    } else if (is_power_2(mCapacity)) {
189        return b->CreateAnd(offset, ConstantInt::get(offset->getType(), mCapacity - 1));
190    } else {
191        return b->CreateURem(offset, ConstantInt::get(offset->getType(), mCapacity));
192    }
193}
194
195Value * StaticBuffer::getCapacity(IDISA::IDISA_Builder * const b, Value * const /* handle */) const {
196    return b->getSize(mCapacity * b->getBitBlockWidth());
197}
198
199void StaticBuffer::setCapacity(IDISA::IDISA_Builder * const /* b */, Value * /* handle */, Value * /* c */) const {
200    report_fatal_error("setCapacity is not supported by this buffer type");
201}
202
203Value * StaticBuffer::getBaseAddress(IDISA::IDISA_Builder * const b, Value * const handle) const {
204    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
205        b->CreateAssert(handle, "handle cannot be null");
206    }
207    return handle;
208}
209
210void StaticBuffer::setBaseAddress(IDISA::IDISA_Builder * const /* b */, Value * /* addr */, Value * /* handle */) const {
211    report_fatal_error("setBaseAddress is not supported by this buffer type");
212}
213
214Value * StaticBuffer::getOverflowAddress(IDISA::IDISA_Builder * const b, Value * const handle) const {
215    return b->CreateGEP(getBaseAddress(b, handle), b->getSize(mCapacity));
216}
217
218Value * StaticBuffer::getStreamBlockPtr(IDISA::IDISA_Builder * const b, Value * const handle, Value * streamIndex, Value * blockIndex, const bool /* readOnly */) const {
219    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
220        Value * const count = getStreamSetCount(b, handle);
221        Value * const index = b->CreateZExtOrTrunc(streamIndex, count->getType());
222        Value * const cond = b->CreateICmpULT(index, count);
223        b->CreateAssert(cond, "out-of-bounds stream access");
224    }
225    return b->CreateGEP(getBaseAddress(b, handle), {modByCapacity(b, blockIndex), streamIndex});
226}
227
228Value * StaticBuffer::getStreamPackPtr(IDISA::IDISA_Builder * const b, Value * const handle, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool /* readOnly */) const {
229    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
230        Value * const count = getStreamSetCount(b, handle);
231        Value * const index = b->CreateZExtOrTrunc(streamIndex, count->getType());
232        Value * const cond = b->CreateICmpULT(index, count);
233        b->CreateAssert(cond, "out-of-bounds stream access");
234    }
235    return b->CreateGEP(getBaseAddress(b, handle), {modByCapacity(b, blockIndex), streamIndex, packIndex});
236}
237
238Value * StaticBuffer::getRawItemPointer(IDISA::IDISA_Builder * const b, Value * const handle, Value * absolutePosition) const {
239    Value * ptr = getBaseAddress(b, handle);
240    Value * relativePosition = b->CreateURem(absolutePosition, ConstantInt::get(absolutePosition->getType(), mCapacity * b->getBitBlockWidth()));
241    Type * const elemTy = mBaseType->getArrayElementType()->getVectorElementType();
242    const auto bw = elemTy->getPrimitiveSizeInBits();
243    assert (is_power_2(bw));
244    if (bw < 8) {
245        Constant * const fw = ConstantInt::get(relativePosition->getType(), 8 / bw);
246        relativePosition = b->CreateUDiv(relativePosition, fw);
247        ptr = b->CreatePointerCast(ptr, b->getInt8PtrTy());
248    } else {
249        ptr = b->CreatePointerCast(ptr, elemTy->getPointerTo());
250    }
251    return b->CreateGEP(ptr, relativePosition);
252}
253
254Value * StaticBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const b, Value * const handle, Value * fromPosition, Value * availItems, bool reverse) const {
255    Type * const ty = fromPosition->getType();
256    const auto blockWidth = b->getBitBlockWidth();
257    Value * capacity = getCapacity(b, handle);
258    Value * const itemsFromBase = b->CreateURem(fromPosition, capacity);
259    if (reverse) {
260        Value * const bufAvail = b->CreateSelect(b->CreateIsNull(itemsFromBase), capacity, itemsFromBase);
261        return b->CreateUMin(availItems, bufAvail);
262    } else {
263        if (mOverflow) {
264            capacity = ConstantInt::get(ty, (mCapacity + mOverflow) * blockWidth - 1);
265        }
266        Value * const linearSpace = b->CreateSub(capacity, itemsFromBase);
267        return b->CreateUMin(availItems, linearSpace);
268    }
269}
270
271Value * StaticBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const b, Value * const handle, Value * fromPosition, Value * consumed, bool reverse) const {
272    Type * const ty = fromPosition->getType();
273    const auto blockWidth = b->getBitBlockWidth();
274    Value * capacity = getCapacity(b, handle);
275    fromPosition = b->CreateURem(fromPosition, capacity);
276    if (reverse) {
277        return b->CreateSelect(b->CreateIsNull(fromPosition), capacity, fromPosition);
278    }
279    consumed = b->CreateURem(consumed, capacity);
280    if (mOverflow) {
281        capacity = ConstantInt::get(ty, (mCapacity + mOverflow) * blockWidth - 1);
282    }
283    Value * const limit = b->CreateSelect(b->CreateICmpULE(consumed, fromPosition), capacity, consumed);
284    return b->CreateSub(limit, fromPosition);
285}
286
287
288// Dynamic Buffer
289
290inline StructType * getDynamicBufferStructType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * baseType, const unsigned addrSpace) {
291    IntegerType * sizeTy = b->getSizeTy();
292    PointerType * typePtr = baseType->getPointerTo(addrSpace);
293    return StructType::get(b->getContext(), {typePtr, typePtr, sizeTy, sizeTy});
294}
295
296void DynamicBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) {
297    Type * const structTy = getDynamicBufferStructType(b, mType, mAddressSpace);
298    Value * const handle = b->CreateCacheAlignedAlloca(structTy);
299    Constant * const capacity = b->getSize(mInitialCapacity * b->getBitBlockWidth());
300    // note: when adding extensible stream sets, make sure to set the initial count here.
301    Value * const bufferSize = b->CreateRoundUp(getAllocationSize(b.get(), handle, capacity), b->getSize(b->getCacheAlignment()));;
302    Value * const baseAddressField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(BaseAddress)});
303    Type * const baseAddressPtrTy = baseAddressField->getType()->getPointerElementType();
304    Value * const baseAddress = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufferSize), baseAddressPtrTy);
305    if (codegen::DebugOptionIsSet(codegen::TraceDynamicBuffers)) {
306        b->CallPrintInt("allocated: ", baseAddress);
307        b->CallPrintInt("allocated capacity: ", bufferSize);
308    }
309    b->CreateStore(baseAddress, baseAddressField);
310    Value * const priorAddressField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(PriorBaseAddress)});
311    b->CreateStore(ConstantPointerNull::getNullValue(baseAddressPtrTy), priorAddressField);
312    Value * const capacityField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(Capacity)});
313    b->CreateStore(b->getSize(mInitialCapacity), capacityField);
314    mStreamSetHandle = handle;
315}
316
317void DynamicBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) const {
318    Value * const handle = mStreamSetHandle;
319    /* Free the dynamically allocated buffer, but not the stack-allocated buffer struct. */
320    Value * priorAddressField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(PriorBaseAddress)});
321    b->CreateFree(b->CreateLoad(priorAddressField));
322    Value * baseAddressField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(BaseAddress)});
323    b->CreateFree(b->CreateLoad(baseAddressField));
324}
325
326void DynamicBuffer::setBaseAddress(IDISA::IDISA_Builder * const /* b */, Value * /* addr */, Value * /* handle */) const {
327    report_fatal_error("setBaseAddress is not supported by this buffer type");
328}
329
330Value * DynamicBuffer::getBaseAddress(IDISA::IDISA_Builder * const b, Value * const handle) const {
331    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
332        b->CreateAssert(handle, "handle cannot be null");
333    }
334    Value * const p = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(BaseAddress)});
335    Value * const addr = b->CreateLoad(p);
336    if (codegen::DebugOptionIsSet(codegen::EnableAsserts)) {
337        b->CreateAssert(addr, "base address cannot be 0");
338    }
339    return addr;
340}
341
342Value * DynamicBuffer::getBlockAddress(IDISA::IDISA_Builder * const b, Value * const handle, Value * blockIndex) const {
343    Value * const workingBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(Capacity)}));
344    assert (blockIndex->getType() == workingBlocks->getType());
345    return b->CreateGEP(getBaseAddress(b, handle), b->CreateURem(blockIndex, workingBlocks));
346}
347
348Value * DynamicBuffer::getOverflowAddress(IDISA::IDISA_Builder * const b, Value * const handle) const {
349    Value * const workingBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(Capacity)}));
350    return b->CreateGEP(getBaseAddress(b, handle), workingBlocks);
351}
352
353Value * DynamicBuffer::getRawItemPointer(IDISA::IDISA_Builder * const b, Value * const handle, Value * absolutePosition) const {
354    Constant * blockSize = ConstantInt::get(absolutePosition->getType(), b->getBitBlockWidth());
355    Value * const absBlock = b->CreateUDiv(absolutePosition, blockSize);
356    Value * blockPos = b->CreateURem(absolutePosition, blockSize);
357    Value * blockPtr = getBlockAddress(b, handle, absBlock);
358    Type * const elemTy = mBaseType->getArrayElementType()->getVectorElementType();
359    const auto bw = elemTy->getPrimitiveSizeInBits();
360    assert (is_power_2(bw));
361    if (bw < 8) {
362        blockPos = b->CreateUDiv(blockPos, ConstantInt::get(blockPos->getType(), 8 / bw));
363        blockPtr = b->CreatePointerCast(blockPtr, b->getInt8PtrTy());
364    } else {
365        blockPtr = b->CreatePointerCast(blockPtr, elemTy->getPointerTo());
366    }
367    return b->CreateGEP(blockPtr, blockPos);
368}
369
370Value * DynamicBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const b, Value * const handle, Value * fromPosition, Value * availItems, bool reverse) const {
371    Value * const bufferSize = getCapacity(b, handle);
372    Value * const itemsFromBase = b->CreateURem(fromPosition, bufferSize);
373    if (reverse) {
374        Value * const bufAvail = b->CreateSelect(b->CreateIsNull(itemsFromBase), bufferSize, itemsFromBase);
375        return b->CreateUMin(availItems, bufAvail);
376    } else {
377        Value * capacity = bufferSize;
378        if (mOverflow) {
379            Constant * const overflow = b->getSize(mOverflow * b->getBitBlockWidth() - 1);
380            capacity = b->CreateAdd(bufferSize, overflow);
381        }
382        Value * const linearSpace = b->CreateSub(capacity, itemsFromBase);
383        return b->CreateUMin(availItems, linearSpace);
384    }
385}
386
387Value * DynamicBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const b, Value * const handle, Value * fromPosition, Value * consumed, bool reverse) const {
388    Value * const bufferSize = getCapacity(b, handle);
389    fromPosition = b->CreateURem(fromPosition, bufferSize);
390    if (reverse) {
391        return b->CreateSelect(b->CreateIsNull(fromPosition), bufferSize, fromPosition);
392    }
393    consumed = b->CreateURem(consumed, bufferSize);
394    Value * capacity = bufferSize;
395    if (mOverflow) {
396        Constant * const overflow = b->getSize(mOverflow * b->getBitBlockWidth() - 1);
397        capacity = b->CreateAdd(bufferSize, overflow);
398    }
399    Value * const limit = b->CreateSelect(b->CreateICmpULE(consumed, fromPosition), capacity, consumed);
400    return b->CreateSub(limit, fromPosition);
401}
402
403Value * DynamicBuffer::getCapacity(IDISA::IDISA_Builder * const b, Value * const handle) const {
404    Value * ptr = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(Capacity)});
405    return b->CreateMul(b->CreateLoad(ptr), b->getSize(b->getBitBlockWidth()));
406}
407
408void DynamicBuffer::setCapacity(IDISA::IDISA_Builder * const b, Value * handle, Value * required) const {
409
410    BasicBlock * const entry = b->GetInsertBlock(); assert (entry);
411    BasicBlock * const insertBefore = entry->getNextNode();
412    BasicBlock * const expand = b->CreateBasicBlock("expandDynamicBuffer", insertBefore);
413    BasicBlock * const resume = b->CreateBasicBlock("", insertBefore);
414
415    ConstantInt * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
416
417    Value * const capacityField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(Capacity)});
418    Value * const capacity = b->CreateMul(capacityField, BLOCK_WIDTH);
419    Value * const needsExpansion = b->CreateICmpULT(capacity, required);
420    b->CreateUnlikelyCondBr(needsExpansion, expand, resume);
421
422    b->SetInsertPoint(expand);
423    Value * const newCapacity = b->CreateRoundUp(required, capacity);
424    Value * const bufferSize = getAllocationSize(b, handle, newCapacity);
425    Value * const newBaseAddress = b->CreateCacheAlignedMalloc(bufferSize);
426    Value * const baseAddressField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(BaseAddress)});
427    Value * const currentBaseAddress = b->CreateLoad(baseAddressField);
428    Value * const priorBaseAddressField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(PriorBaseAddress)});
429    Value * const priorBaseAddress = b->CreateLoad(priorBaseAddressField);
430    b->CreateMemCpy(newBaseAddress, currentBaseAddress, capacity, b->getCacheAlignment());
431    b->CreateStore(baseAddressField, newBaseAddress);
432    b->CreateStore(capacityField, b->CreateUDiv(newCapacity, BLOCK_WIDTH));
433    b->CreateStore(priorBaseAddressField, currentBaseAddress);
434    b->CreateFree(priorBaseAddress);
435
436    b->CreateBr(resume);
437    b->SetInsertPoint(resume);
438}
439
440Value * DynamicBuffer::getAllocationSize(IDISA::IDISA_Builder * const b, Value * handle, Value * const requiredItemCapacity) const {
441    Value * itemCapacity = requiredItemCapacity;
442    if (mOverflow) {
443        Constant * const overflowSize =  b->getSize(mOverflow * b->getBitBlockWidth());
444        itemCapacity = b->CreateAdd(requiredItemCapacity, overflowSize);
445    }
446    Value * const numOfStreams = getStreamSetCount(b, handle);
447    Value * bufferSize = b->CreateMul(itemCapacity, numOfStreams);
448    const auto fieldWidth = mBaseType->getArrayElementType()->getScalarSizeInBits();
449    if (LLVM_LIKELY(fieldWidth < 8)) {
450        bufferSize = b->CreateUDiv(bufferSize, b->getSize(8 / fieldWidth));
451    } else if (LLVM_UNLIKELY(fieldWidth > 8)) {
452        bufferSize = b->CreateMul(bufferSize, b->getSize(fieldWidth / 8));
453    }
454    return bufferSize;
455}
456
457ExternalBuffer::ExternalBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * const type,
458                               Value * const externalAddress, const unsigned AddressSpace)
459: StreamSetBuffer(BufferKind::ExternalBuffer, b, type, AddressSpace)
460, mExternalAddress(externalAddress) {
461    mUniqueID = "E";
462    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
463}
464
465StaticBuffer::StaticBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * const type,
466                           const size_t capacity, const size_t overflowBlocks, const unsigned AddressSpace)
467: StreamSetBuffer(BufferKind::StaticBuffer, b, type, AddressSpace)
468, mCapacity(capacity)
469, mOverflow(overflowBlocks) {
470    mUniqueID = "S" + std::to_string(capacity);
471    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
472}
473
474DynamicBuffer::DynamicBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * const type,
475                             const size_t initialCapacity, const size_t overflowBlocks, const unsigned AddressSpace)
476: StreamSetBuffer(BufferKind::DynamicBuffer, b, type, AddressSpace)
477, mInitialCapacity(initialCapacity)
478, mOverflow(overflowBlocks) {
479    mUniqueID = "D";
480    if (overflowBlocks != 0) mUniqueID += std::to_string(overflowBlocks);
481    if (AddressSpace != 0) mUniqueID += "@" + std::to_string(AddressSpace);
482}
483
484// Helper routines
485ArrayType * resolveStreamSetType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type) {
486    unsigned numElements = 1;
487    if (LLVM_LIKELY(type->isArrayTy())) {
488        numElements = type->getArrayNumElements();
489        type = type->getArrayElementType();
490    }
491    if (LLVM_LIKELY(type->isVectorTy() && type->getVectorNumElements() == 0)) {
492        type = type->getVectorElementType();
493        if (LLVM_LIKELY(type->isIntegerTy())) {
494            const auto fieldWidth = cast<IntegerType>(type)->getBitWidth();
495            type = b->getBitBlockType();
496            if (fieldWidth != 1) {
497                type = ArrayType::get(type, fieldWidth);
498            }
499            return ArrayType::get(type, numElements);
500        }
501    }
502    std::string tmp;
503    raw_string_ostream out(tmp);
504    type->print(out);
505    out << " is an unvalid stream set buffer type.";
506    report_fatal_error(out.str());
507}
508
509
510inline StreamSetBuffer::StreamSetBuffer(BufferKind k, const std::unique_ptr<kernel::KernelBuilder> & b, Type * baseType, unsigned AddressSpace)
511: mBufferKind(k)
512, mType(resolveStreamSetType(b, baseType))
513, mAddressSpace(AddressSpace)
514, mStreamSetHandle(nullptr)
515, mBaseType(baseType)
516, mProducer(nullptr) {
517
518}
519
520StreamSetBuffer::~StreamSetBuffer() { }
Note: See TracBrowser for help on using the repository browser.