source: icGREP/icgrep-devel/icgrep/kernels/streamset.cpp @ 5611

Last change on this file since 5611 was 5597, checked in by nmedfort, 23 months ago

Modified stream set buffers to use heap memory.

File size: 40.3 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5
6#include "streamset.h"
7#include <llvm/IR/Module.h>
8#include <llvm/Support/raw_ostream.h>
9#include <kernels/kernel.h>
10#include <kernels/kernel_builder.h>
11#include <toolchain/toolchain.h>
12#include <llvm/Support/Debug.h>
13#include <llvm/Support/Format.h>
14
15namespace llvm { class Constant; }
16namespace llvm { class Function; }
17
18using namespace parabix;
19using namespace llvm;
20using namespace IDISA;
21
22
23Type * StreamSetBuffer::getStreamSetBlockType() const { return mType;}
24
25ArrayType * resolveStreamSetType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type);
26
27StructType * resolveExpandableStreamSetType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type);
28
29void StreamSetBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
30    if (LLVM_LIKELY(mStreamSetBufferPtr == nullptr)) {
31        Type * const ty = getType();
32        if (mAddressSpace == 0) {
33            Constant * size = ConstantExpr::getSizeOf(ty);
34            size = ConstantExpr::getMul(size, ConstantInt::get(size->getType(), mBufferBlocks));
35            mStreamSetBufferPtr = iBuilder->CreatePointerCast(iBuilder->CreateCacheAlignedMalloc(size), ty->getPointerTo());
36        } else {
37            mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(ty, iBuilder->getSize(mBufferBlocks));
38        }
39        iBuilder->CreateAlignedStore(Constant::getNullValue(ty), mStreamSetBufferPtr, iBuilder->getCacheAlignment());
40    } else {
41        report_fatal_error("StreamSetBuffer::allocateBuffer() was called twice on the same stream set");
42    }
43}
44
45void StreamSetBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) const {
46    if (mAddressSpace == 0) {
47        iBuilder->CreateFree(mStreamSetBufferPtr);
48    }
49}
50
51Value * StreamSetBuffer::getStreamBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, const bool /* readOnly */) const {
52    if (codegen::EnableAsserts) {
53        Value * const count = getStreamSetCount(iBuilder, self);
54        Value * const index = iBuilder->CreateZExtOrTrunc(streamIndex, count->getType());
55        Value * const cond = iBuilder->CreateICmpULT(index, count);
56        iBuilder->CreateAssert(cond, "StreamSetBuffer: out-of-bounds stream access");
57    }
58    return iBuilder->CreateGEP(getStreamSetBlockPtr(iBuilder, self, blockIndex), {iBuilder->getInt32(0), streamIndex});
59}
60
61Value * StreamSetBuffer::getStreamPackPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool /* readOnly */) const {
62    if (codegen::EnableAsserts) {
63        Value * const count = getStreamSetCount(iBuilder, self);
64        Value * const index = iBuilder->CreateZExtOrTrunc(streamIndex, count->getType());
65        Value * const cond = iBuilder->CreateICmpULT(index, count);
66        iBuilder->CreateAssert(cond, "StreamSetBuffer: out-of-bounds stream access");
67    }
68    return iBuilder->CreateGEP(getStreamSetBlockPtr(iBuilder, self, blockIndex), {iBuilder->getInt32(0), streamIndex, packIndex});
69}
70
71void StreamSetBuffer::setBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * /* self */, Value * /* addr */) const {
72    report_fatal_error("setBaseAddress is not supported by this buffer type");
73}
74
75Value * StreamSetBuffer::getBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * /* self */) const {
76    report_fatal_error("getBufferedSize is not supported by this buffer type");
77}
78
79void StreamSetBuffer::setBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * /* self */, llvm::Value * /* size */) const {
80    report_fatal_error("setBufferedSize is not supported by this buffer type");
81}
82
83Value * StreamSetBuffer::getCapacity(IDISA::IDISA_Builder * const iBuilder, Value * /* self */) const {
84    report_fatal_error("getCapacity is not supported by this buffer type");
85}
86
87void StreamSetBuffer::setCapacity(IDISA::IDISA_Builder * const iBuilder, Value * /* self */, llvm::Value * /* c */) const {
88    report_fatal_error("setCapacity is not supported by this buffer type");
89}
90
91inline bool StreamSetBuffer::isCapacityGuaranteed(const Value * const index, const size_t capacity) const {
92    if (LLVM_UNLIKELY(isa<ConstantInt>(index))) {
93        if (LLVM_LIKELY(cast<ConstantInt>(index)->getLimitedValue() < capacity)) {
94            return true;
95        }
96    }
97    return false;
98}
99
100Value * StreamSetBuffer::getStreamSetCount(IDISA::IDISA_Builder * const iBuilder, Value *) const {
101    size_t count = 1;
102    if (isa<ArrayType>(mBaseType)) {
103        count = mBaseType->getArrayNumElements();
104    }
105    return iBuilder->getSize(count);
106}
107
108inline Value * StreamSetBuffer::modByBufferBlocks(IDISA::IDISA_Builder * const iBuilder, Value * const offset) const {
109    assert (offset->getType()->isIntegerTy());
110    if (isCapacityGuaranteed(offset, mBufferBlocks)) {
111        return offset;
112    } else if (mBufferBlocks == 1) {
113        return ConstantInt::getNullValue(iBuilder->getSizeTy());
114    } else if ((mBufferBlocks & (mBufferBlocks - 1)) == 0) { // is power of 2
115        return iBuilder->CreateAnd(offset, ConstantInt::get(offset->getType(), mBufferBlocks - 1));
116    } else {
117        return iBuilder->CreateURem(offset, ConstantInt::get(offset->getType(), mBufferBlocks));
118    }
119}
120
121/**
122 * @brief getRawItemPointer
123 *
124 * get a raw pointer the iN field at position absoluteItemPosition of the stream number streamIndex of the stream set.
125 * In the case of a stream whose fields are less than one byte (8 bits) in size, the pointer is to the containing byte.
126 * The type of the pointer is i8* for fields of 8 bits or less, otherwise iN* for N-bit fields.
127 */
128Value * StreamSetBuffer::getRawItemPointer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * absolutePosition) const {
129    Value * ptr = iBuilder->CreateGEP(getBaseAddress(iBuilder, self), {iBuilder->getInt32(0), streamIndex});
130    Value * relativePosition = absolutePosition;
131    const auto bw = mBaseType->getArrayElementType()->getScalarSizeInBits();
132    if (bw < 8) {
133        assert (bw  == 1 || bw == 2 || bw == 4);
134        relativePosition = iBuilder->CreateUDiv(relativePosition, ConstantInt::get(relativePosition->getType(), 8 / bw));
135        ptr = iBuilder->CreatePointerCast(ptr, iBuilder->getInt8PtrTy());
136    } else {
137        ptr = iBuilder->CreatePointerCast(ptr, iBuilder->getIntNTy(bw)->getPointerTo());
138    }
139    return iBuilder->CreateGEP(ptr, relativePosition);
140}
141
142Value * StreamSetBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
143    if (isa<ArrayType>(mType) && dyn_cast<ArrayType>(mType)->getNumElements() > 1) {
144        Constant * stride = iBuilder->getSize(iBuilder->getStride());
145        return iBuilder->CreateSub(stride, iBuilder->CreateURem(fromPosition, stride));
146    } else {
147        Constant * bufSize = iBuilder->getSize(mBufferBlocks * iBuilder->getStride());
148        return iBuilder->CreateSub(bufSize, iBuilder->CreateURem(fromPosition, bufSize, "linearItems"));
149    }
150}
151
152Value * StreamSetBuffer::getLinearlyAccessibleBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
153    Constant * bufBlocks = iBuilder->getSize(mBufferBlocks);
154    return iBuilder->CreateSub(bufBlocks, iBuilder->CreateURem(fromBlock, bufBlocks), "linearBlocks");
155}
156
157Value * StreamSetBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
158    return getLinearlyAccessibleItems(iBuilder, self, fromPosition);
159}
160
161Value * StreamSetBuffer::getLinearlyWritableBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
162    return getLinearlyAccessibleBlocks(iBuilder, self, fromBlock);
163}
164
165Value * StreamSetBuffer::getBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
166    iBuilder->CreateAssert(self, "StreamSetBuffer base address cannot be 0");
167    return self;
168}
169
170void StreamSetBuffer::createBlockCopy(IDISA::IDISA_Builder * const iBuilder, Value * targetBlockPtr, Value * sourceBlockPtr, Value * blocksToCopy) const {
171    Type * i8ptr = iBuilder->getInt8PtrTy();
172    unsigned alignment = iBuilder->getBitBlockWidth() / 8;
173    size_t numStreams = 1;
174    if (isa<ArrayType>(mBaseType)) {
175        numStreams = mBaseType->getArrayNumElements();
176    }
177    const auto fieldWidth = mBaseType->getArrayElementType()->getScalarSizeInBits();
178    Value * blockCopyBytes = iBuilder->CreateMul(blocksToCopy, iBuilder->getSize(iBuilder->getBitBlockWidth() * numStreams * fieldWidth/8));
179    iBuilder->CreateMemMove(iBuilder->CreateBitCast(targetBlockPtr, i8ptr), iBuilder->CreateBitCast(sourceBlockPtr, i8ptr), blockCopyBytes, alignment);
180}
181
182void StreamSetBuffer::createBlockAlignedCopy(IDISA::IDISA_Builder * const iBuilder, Value * targetBlockPtr, Value * sourceBlockPtr, Value * itemsToCopy) const {
183    Type * const int8PtrTy = iBuilder->getInt8PtrTy();
184    const unsigned alignment = iBuilder->getBitBlockWidth() / 8;
185    Constant * const blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
186    size_t numStreams = 1;
187    if (isa<ArrayType>(mBaseType)) {
188        numStreams = mBaseType->getArrayNumElements();
189    }
190    const auto fieldWidth = mBaseType->getArrayElementType()->getScalarSizeInBits();
191    if (numStreams == 1) {
192        Value * copyBits = iBuilder->CreateMul(itemsToCopy, iBuilder->getSize(fieldWidth));
193        Value * copyBytes = iBuilder->CreateLShr(iBuilder->CreateAdd(copyBits, iBuilder->getSize(7)), iBuilder->getSize(3));
194        iBuilder->CreateMemMove(iBuilder->CreateBitCast(targetBlockPtr, int8PtrTy), iBuilder->CreateBitCast(sourceBlockPtr, int8PtrTy), copyBytes, alignment);
195    } else {
196        Value * blocksToCopy = iBuilder->CreateUDiv(itemsToCopy, blockSize);
197        Value * partialItems = iBuilder->CreateURem(itemsToCopy, blockSize);
198        Value * partialBlockTargetPtr = iBuilder->CreateGEP(targetBlockPtr, blocksToCopy);
199        Value * partialBlockSourcePtr = iBuilder->CreateGEP(sourceBlockPtr, blocksToCopy);
200        Value * blockCopyBytes = iBuilder->CreateMul(blocksToCopy, iBuilder->getSize(iBuilder->getBitBlockWidth() * numStreams * fieldWidth/8));
201        iBuilder->CreateMemMove(iBuilder->CreateBitCast(targetBlockPtr, int8PtrTy), iBuilder->CreateBitCast(sourceBlockPtr, int8PtrTy), blockCopyBytes, alignment);
202        Value * partialCopyBitsPerStream = iBuilder->CreateMul(partialItems, iBuilder->getSize(fieldWidth));
203        Value * partialCopyBytesPerStream = iBuilder->CreateLShr(iBuilder->CreateAdd(partialCopyBitsPerStream, iBuilder->getSize(7)), iBuilder->getSize(3));
204        for (unsigned strm = 0; strm < numStreams; strm++) {
205            Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
206            Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
207            strmTargetPtr = iBuilder->CreateBitCast(strmTargetPtr, int8PtrTy);
208            strmSourcePtr = iBuilder->CreateBitCast(strmSourcePtr, int8PtrTy);
209            iBuilder->CreateMemMove(strmTargetPtr, strmSourcePtr, partialCopyBytesPerStream, alignment);
210        }
211    }
212}
213
214// Source File Buffer
215
216Type * SourceBuffer::getStreamSetBlockType() const {
217    return cast<PointerType>(mType->getStructElementType(int(SourceBuffer::Field::BaseAddress)))->getElementType();
218}
219
220
221Value * SourceBuffer::getBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
222    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::BufferedSize))});
223    return iBuilder->CreateLoad(ptr);
224}
225
226void SourceBuffer::setBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * self, llvm::Value * size) const {
227    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::BufferedSize))});
228    iBuilder->CreateStore(size, ptr);
229}
230
231Value * SourceBuffer::getCapacity(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
232    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::Capacity))});
233    return iBuilder->CreateLoad(ptr);
234}
235
236void SourceBuffer::setCapacity(IDISA::IDISA_Builder * const iBuilder, Value * self, llvm::Value * c) const {
237    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::Capacity))});
238    iBuilder->CreateStore(c, ptr);
239}
240
241void SourceBuffer::setBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * addr) const {
242    Value * const ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::BaseAddress))});
243
244    iBuilder->CreateStore(iBuilder->CreatePointerCast(addr, ptr->getType()->getPointerElementType()), ptr);
245}
246
247Value * SourceBuffer::getBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * const self) const {
248    iBuilder->CreateAssert(self, "SourceBuffer: instance cannot be null");
249    Value * const ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::BaseAddress))});
250    Value * const addr = iBuilder->CreateLoad(ptr);
251    iBuilder->CreateAssert(addr, "SourceBuffer: base address cannot be 0");
252    return addr;
253}
254
255Value * SourceBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
256    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), blockIndex);
257}
258
259Value * SourceBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
260    return iBuilder->CreateSub(getCapacity(iBuilder, self), fromPosition);
261}
262
263Value * SourceBuffer::getLinearlyAccessibleBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
264    return iBuilder->CreateSub(iBuilder->CreateUDiv(getCapacity(iBuilder, self), iBuilder->getSize(iBuilder->getBitBlockWidth())), fromBlock);
265}
266
267void SourceBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
268    if (LLVM_LIKELY(mStreamSetBufferPtr == nullptr)) {
269        Type * const ty = getType();
270        mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(ty, iBuilder->getSize(mBufferBlocks));
271        iBuilder->CreateAlignedStore(Constant::getNullValue(ty), mStreamSetBufferPtr, iBuilder->getCacheAlignment());
272    } else {
273        report_fatal_error("StreamSetBuffer::allocateBuffer() was called twice on the same stream set");
274    }
275}
276
277void SourceBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) const {
278
279}
280
281// External File Buffer
282void ExternalBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> &) {
283    report_fatal_error("External buffers cannot be allocated.");
284}
285
286void ExternalBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> &) const {
287
288}
289
290Value * ExternalBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
291    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), blockIndex);
292}
293
294Value * ExternalBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const, Value *, Value *) const {
295    report_fatal_error("External buffers: getLinearlyAccessibleItems is not supported.");
296}
297
298// Circular Buffer
299Value * CircularBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * const self, Value * const blockIndex) const {
300    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), modByBufferBlocks(iBuilder, blockIndex));
301}
302
303Value * CircularBuffer::getRawItemPointer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * absolutePosition) const {
304    Value * ptr = iBuilder->CreateGEP(getBaseAddress(iBuilder, self), {iBuilder->getInt32(0), streamIndex});
305    Value * relativePosition = iBuilder->CreateURem(absolutePosition, ConstantInt::get(absolutePosition->getType(), mBufferBlocks * iBuilder->getBitBlockWidth()));
306    const auto bw = mBaseType->getArrayElementType()->getScalarSizeInBits();
307    if (bw < 8) {
308        assert (bw  == 1 || bw == 2 || bw == 4);
309        relativePosition = iBuilder->CreateUDiv(relativePosition, ConstantInt::get(relativePosition->getType(), 8 / bw));
310        ptr = iBuilder->CreatePointerCast(ptr, iBuilder->getInt8PtrTy());
311    } else {
312        ptr = iBuilder->CreatePointerCast(ptr, iBuilder->getIntNTy(bw)->getPointerTo());
313    }
314    return iBuilder->CreateGEP(ptr, relativePosition);
315}
316
317// CircularCopybackBuffer Buffer
318void CircularCopybackBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
319    Type * const ty = getType();
320    Constant * size = ConstantExpr::getSizeOf(ty);
321    size = ConstantExpr::getMul(size, ConstantInt::get(size->getType(), mBufferBlocks + mOverflowBlocks));
322    mStreamSetBufferPtr = iBuilder->CreatePointerCast(iBuilder->CreateCacheAlignedMalloc(size), ty->getPointerTo());
323}
324
325void CircularCopybackBuffer::createCopyBack(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * overFlowItems) const {
326    Value * overFlowAreaPtr = iBuilder->CreateGEP(self, iBuilder->getSize(mBufferBlocks));
327    createBlockAlignedCopy(iBuilder, self, overFlowAreaPtr, overFlowItems);
328}
329
330Value * CircularCopybackBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
331    return iBuilder->CreateAdd(getLinearlyAccessibleItems(iBuilder, self, fromPosition), iBuilder->getSize(mOverflowBlocks * iBuilder->getBitBlockWidth()));
332}
333
334Value * CircularCopybackBuffer::getLinearlyWritableBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
335    return iBuilder->CreateAdd(getLinearlyAccessibleBlocks(iBuilder, self, fromBlock), iBuilder->getSize(mOverflowBlocks));
336}
337
338// SwizzledCopybackBuffer Buffer
339
340void SwizzledCopybackBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
341    Type * const ty = getType();
342    Constant * size = ConstantExpr::getSizeOf(ty);
343    size = ConstantExpr::getMul(size, ConstantInt::get(size->getType(), mBufferBlocks + mOverflowBlocks));
344    mStreamSetBufferPtr = iBuilder->CreatePointerCast(iBuilder->CreateCacheAlignedMalloc(size), ty->getPointerTo());
345}
346
347void SwizzledCopybackBuffer::createBlockAlignedCopy(IDISA::IDISA_Builder * const iBuilder, Value * targetBlockPtr, Value * sourceBlockPtr, Value * itemsToCopy) const {
348    Type * int8PtrTy = iBuilder->getInt8PtrTy();
349    DataLayout DL(iBuilder->getModule());
350    IntegerType * const intAddrTy = iBuilder->getIntPtrTy(DL);
351
352    Constant * blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
353    Function * f = iBuilder->GetInsertBlock()->getParent();
354    BasicBlock * wholeBlockCopy = BasicBlock::Create(iBuilder->getContext(), "wholeBlockCopy", f, 0);
355    BasicBlock * partialBlockCopy = BasicBlock::Create(iBuilder->getContext(), "partialBlockCopy", f, 0);
356    BasicBlock * copyDone = BasicBlock::Create(iBuilder->getContext(), "copyDone", f, 0);
357    const unsigned numStreams = getType()->getArrayNumElements();
358    const unsigned swizzleFactor = iBuilder->getBitBlockWidth()/mFieldWidth;
359    const auto elemTy = getType()->getArrayElementType();
360    const unsigned fieldWidth = isa<ArrayType>(elemTy) ? elemTy->getArrayNumElements() : 1;
361    Value * blocksToCopy = iBuilder->CreateUDiv(itemsToCopy, blockSize);
362    Value * partialItems = iBuilder->CreateURem(itemsToCopy, blockSize);
363    Value * partialBlockTargetPtr = iBuilder->CreateGEP(targetBlockPtr, blocksToCopy);
364    Value * partialBlockSourcePtr = iBuilder->CreateGEP(sourceBlockPtr, blocksToCopy);
365    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(blocksToCopy, iBuilder->getSize(0)), wholeBlockCopy, partialBlockCopy);
366
367    iBuilder->SetInsertPoint(wholeBlockCopy);
368    const unsigned alignment = iBuilder->getBitBlockWidth() / 8;
369    Value * copyLength = iBuilder->CreateSub(iBuilder->CreatePtrToInt(partialBlockTargetPtr, intAddrTy), iBuilder->CreatePtrToInt(targetBlockPtr, intAddrTy));
370    iBuilder->CreateMemMove(iBuilder->CreatePointerCast(targetBlockPtr, int8PtrTy), iBuilder->CreatePointerCast(sourceBlockPtr, int8PtrTy), copyLength, alignment);
371    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(partialItems, iBuilder->getSize(0)), partialBlockCopy, copyDone);
372    iBuilder->SetInsertPoint(partialBlockCopy);
373    Value * copyBits = iBuilder->CreateMul(itemsToCopy, iBuilder->getSize(fieldWidth * swizzleFactor));
374    Value * copyBytes = iBuilder->CreateLShr(iBuilder->CreateAdd(copyBits, iBuilder->getSize(7)), iBuilder->getSize(3));
375    for (unsigned strm = 0; strm < numStreams; strm += swizzleFactor) {
376        Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
377        Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
378        iBuilder->CreateMemMove(iBuilder->CreatePointerCast(strmTargetPtr, int8PtrTy), iBuilder->CreatePointerCast(strmSourcePtr, int8PtrTy), copyBytes, alignment);
379    }
380    iBuilder->CreateBr(copyDone);
381
382    iBuilder->SetInsertPoint(copyDone);
383}
384
385void SwizzledCopybackBuffer::createCopyBack(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * overFlowItems) const {
386    Value * overFlowAreaPtr = iBuilder->CreateGEP(self, iBuilder->getSize(mBufferBlocks));
387    createBlockAlignedCopy(iBuilder, self, overFlowAreaPtr, overFlowItems);
388}
389
390Value * SwizzledCopybackBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
391    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), modByBufferBlocks(iBuilder, blockIndex));
392}
393
394Value * SwizzledCopybackBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
395    return iBuilder->CreateAdd(getLinearlyAccessibleItems(iBuilder, self, fromPosition), iBuilder->getSize(mOverflowBlocks * iBuilder->getBitBlockWidth()));
396}
397
398Value * SwizzledCopybackBuffer::getLinearlyWritableBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
399    return iBuilder->CreateAdd(getLinearlyAccessibleBlocks(iBuilder, self, fromBlock), iBuilder->getSize(mOverflowBlocks));
400}
401
402// Expandable Buffer
403
404void ExpandableBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
405    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType());
406    Value * const capacityPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
407    iBuilder->CreateStore(iBuilder->getSize(mInitialCapacity), capacityPtr);
408    Type * const bufferType = getType()->getStructElementType(1)->getPointerElementType();
409    Constant * const bufferWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(bufferType), iBuilder->getSizeTy(), false);
410    Constant * const size = ConstantExpr::getMul(iBuilder->getSize(mBufferBlocks * mInitialCapacity), bufferWidth);
411    const auto alignment = std::max(iBuilder->getCacheAlignment(), iBuilder->getBitBlockWidth() / 8);
412    Value * const ptr = iBuilder->CreateAlignedMalloc(size, alignment);
413    iBuilder->CreateMemZero(ptr, size, bufferType->getPrimitiveSizeInBits() / 8);
414    Value * const streamSetPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
415    iBuilder->CreateStore(iBuilder->CreatePointerCast(ptr, bufferType->getPointerTo()), streamSetPtr);
416}
417
418std::pair<Value *, Value *> ExpandableBuffer::getInternalStreamBuffer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, const bool readOnly) const {
419
420    // ENTRY
421    Value * const capacityPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
422    Value * const capacity = iBuilder->CreateLoad(capacityPtr);
423    Value * const streamSetPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
424    Value * const streamSet = iBuilder->CreateLoad(streamSetPtr);
425    blockIndex = modByBufferBlocks(iBuilder, blockIndex);
426
427    assert (streamIndex->getType() == capacity->getType());
428    Value * const cond = iBuilder->CreateICmpULT(streamIndex, capacity);
429
430    // Are we guaranteed that we can access this stream?
431    if (readOnly || isCapacityGuaranteed(streamIndex, mInitialCapacity)) {
432        iBuilder->CreateAssert(cond, "ExpandableBuffer: out-of-bounds stream access");
433        Value * offset = iBuilder->CreateAdd(iBuilder->CreateMul(blockIndex, capacity), streamIndex);
434        return {streamSet, offset};
435    }
436
437    BasicBlock * const entry = iBuilder->GetInsertBlock();
438    BasicBlock * const expand = BasicBlock::Create(iBuilder->getContext(), "expand", entry->getParent());
439    BasicBlock * const resume = BasicBlock::Create(iBuilder->getContext(), "resume", entry->getParent());
440
441    iBuilder->CreateLikelyCondBr(cond, resume, expand);
442
443    // EXPAND
444    iBuilder->SetInsertPoint(expand);
445
446    Type * elementType = getType()->getStructElementType(1)->getPointerElementType();
447    Constant * const vectorWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(elementType), capacity->getType(), false);
448
449    Value * newCapacity = iBuilder->CreateAdd(streamIndex, iBuilder->getSize(1));
450    newCapacity = iBuilder->CreateCeilLog2(newCapacity);
451    newCapacity = iBuilder->CreateShl(iBuilder->getSize(1), newCapacity, "newCapacity");
452
453    std::string tmp;
454    raw_string_ostream out(tmp);
455    out << "__expand";
456    elementType->print(out);
457    std::string name = out.str();
458
459    Module * const m = iBuilder->getModule();
460    Function * expandFunction = m->getFunction(name);
461
462    if (expandFunction == nullptr) {
463
464        const auto ip = iBuilder->saveIP();
465
466        FunctionType * fty = FunctionType::get(elementType->getPointerTo(), {elementType->getPointerTo(), iBuilder->getSizeTy(), iBuilder->getSizeTy()}, false);
467        expandFunction = Function::Create(fty, GlobalValue::PrivateLinkage, name, m);
468
469        auto args = expandFunction->arg_begin();
470        Value * streamSet = &*args++;
471        Value * capacity = &*args++;
472        Value * newCapacity = &*args;
473
474        BasicBlock * entry = BasicBlock::Create(iBuilder->getContext(), "entry", expandFunction);
475        iBuilder->SetInsertPoint(entry);
476
477        Value * size = iBuilder->CreateMul(newCapacity, iBuilder->getSize(mBufferBlocks));
478        const auto memAlign = std::max(iBuilder->getCacheAlignment(), iBuilder->getBitBlockWidth() / 8);
479
480        Value * newStreamSet = iBuilder->CreatePointerCast(iBuilder->CreateAlignedMalloc(iBuilder->CreateMul(size, vectorWidth), memAlign), elementType->getPointerTo());
481        Value * const diffCapacity = iBuilder->CreateMul(iBuilder->CreateSub(newCapacity, capacity), vectorWidth);
482
483        const auto alignment = elementType->getPrimitiveSizeInBits() / 8;
484        for (unsigned i = 0; i < mBufferBlocks; ++i) {
485            ConstantInt * const offset = iBuilder->getSize(i);
486            Value * srcOffset = iBuilder->CreateMul(capacity, offset);
487            Value * srcPtr = iBuilder->CreateGEP(streamSet, srcOffset);
488            Value * destOffset = iBuilder->CreateMul(newCapacity, offset);
489            Value * destPtr = iBuilder->CreateGEP(newStreamSet, destOffset);
490            iBuilder->CreateMemCpy(destPtr, srcPtr, iBuilder->CreateMul(capacity, vectorWidth), alignment);
491            Value * destZeroOffset = iBuilder->CreateAdd(destOffset, capacity);
492            Value * destZeroPtr = iBuilder->CreateGEP(newStreamSet, destZeroOffset);
493            iBuilder->CreateMemZero(destZeroPtr, diffCapacity, alignment);
494        }
495
496        iBuilder->CreateFree(streamSet);
497
498        iBuilder->CreateRet(newStreamSet);
499
500        iBuilder->restoreIP(ip);
501    }
502
503    Value * newStreamSet = iBuilder->CreateCall(expandFunction, {streamSet, capacity, newCapacity});
504    iBuilder->CreateStore(newStreamSet, streamSetPtr);
505    iBuilder->CreateStore(newCapacity, capacityPtr);
506
507    iBuilder->CreateBr(resume);
508
509    // RESUME
510    iBuilder->SetInsertPoint(resume);
511
512    PHINode * phiStreamSet = iBuilder->CreatePHI(streamSet->getType(), 2);
513    phiStreamSet->addIncoming(streamSet, entry);
514    phiStreamSet->addIncoming(newStreamSet, expand);
515
516    PHINode * phiCapacity = iBuilder->CreatePHI(capacity->getType(), 2);
517    phiCapacity->addIncoming(capacity, entry);
518    phiCapacity->addIncoming(newCapacity, expand);
519
520    Value * offset = iBuilder->CreateAdd(iBuilder->CreateMul(blockIndex, phiCapacity), streamIndex);
521
522    return {phiStreamSet, offset};
523}
524
525Value * ExpandableBuffer::getStreamBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, const bool readOnly) const {
526    Value * ptr, * offset;
527    std::tie(ptr, offset) = getInternalStreamBuffer(iBuilder, self, streamIndex, blockIndex, readOnly);
528    return iBuilder->CreateGEP(ptr, offset);
529}
530
531Value * ExpandableBuffer::getStreamPackPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool readOnly) const {
532    Value * ptr, * offset;
533    std::tie(ptr, offset) = getInternalStreamBuffer(iBuilder, self, streamIndex, blockIndex, readOnly);
534    return iBuilder->CreateGEP(ptr, {offset, packIndex});
535}
536
537Value * ExpandableBuffer::getStreamSetCount(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
538    return iBuilder->CreateLoad(iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)}));
539}
540
541Value * ExpandableBuffer::getBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
542    iBuilder->CreateAssert(self, "ExpandableBuffer: instance cannot be null");
543    Value * const baseAddr = iBuilder->CreateLoad(iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)}));
544    iBuilder->CreateAssert(self, "ExpandableBuffer: base address cannot be 0");
545    return baseAddr;
546}
547
548void ExpandableBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) const {
549    b->CreateFree(getBaseAddress(b.get(), mStreamSetBufferPtr));
550}
551
552Value * ExpandableBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value *, Value *) const {
553    report_fatal_error("Expandable buffers: getStreamSetBlockPtr is not supported.");
554}
555
556Value * ExpandableBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value *) const {
557    report_fatal_error("Expandable buffers: getLinearlyAccessibleItems is not supported.");
558}
559
560SourceBuffer::SourceBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, unsigned MemoryAddressSpace, unsigned StructAddressSpace)
561: StreamSetBuffer(BufferKind::SourceBuffer, type, StructType::get(resolveStreamSetType(b, type)->getPointerTo(MemoryAddressSpace), b->getSizeTy(), b->getSizeTy(), nullptr), 0, StructAddressSpace) {
562    mUniqueID = "B";
563    if (MemoryAddressSpace != 0 || StructAddressSpace != 0) {
564        mUniqueID += "@" + std::to_string(MemoryAddressSpace) + ":" + std::to_string(StructAddressSpace);
565    }
566}
567
568ExternalBuffer::ExternalBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, llvm::Value * addr, unsigned AddressSpace)
569: StreamSetBuffer(BufferKind::ExternalBuffer, type, resolveStreamSetType(b, type), 0, AddressSpace) {
570    mUniqueID = "E";
571    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
572    mStreamSetBufferPtr = b->CreatePointerBitCastOrAddrSpaceCast(addr, getPointerType());
573}
574
575CircularBuffer::CircularBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
576: StreamSetBuffer(BufferKind::CircularBuffer, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
577    mUniqueID = "C" + std::to_string(bufferBlocks);
578    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
579}
580
581CircularBuffer::CircularBuffer(const BufferKind k, const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
582: StreamSetBuffer(k, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
583
584}
585
586CircularCopybackBuffer::CircularCopybackBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned AddressSpace)
587: CircularBuffer(BufferKind::CircularCopybackBuffer, b, type, bufferBlocks, AddressSpace)
588, mOverflowBlocks(overflowBlocks) {
589    mUniqueID = "CC" + std::to_string(bufferBlocks);
590    if (mOverflowBlocks != 1) mUniqueID += "_" + std::to_string(mOverflowBlocks);
591    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
592}
593
594ExpandableBuffer::ExpandableBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
595: StreamSetBuffer(BufferKind::ExpandableBuffer, type, resolveExpandableStreamSetType(b, type), bufferBlocks, AddressSpace)
596, mInitialCapacity(type->getArrayNumElements()) {
597    mUniqueID = "XP" + std::to_string(bufferBlocks);
598    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
599}
600
601SwizzledCopybackBuffer::SwizzledCopybackBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned fieldwidth, unsigned AddressSpace)
602: StreamSetBuffer(BufferKind::SwizzledCopybackBuffer, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks), mFieldWidth(fieldwidth) {
603    mUniqueID = "SW" + std::to_string(fieldwidth) + ":" + std::to_string(bufferBlocks);
604    if (mOverflowBlocks != 1) {
605        mUniqueID += "_" + std::to_string(mOverflowBlocks);
606    }
607    if (AddressSpace > 0) {
608        mUniqueID += "@" + std::to_string(AddressSpace);
609    }
610}
611
612Value * DynamicBuffer::getBaseAddress(IDISA::IDISA_Builder * const b, Value * const handle) const {
613    b->CreateAssert(handle, "DynamicBuffer: instance cannot be null");
614    Value * const p = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::BaseAddress))});
615    Value * const addr = b->CreateLoad(p);
616    b->CreateAssert(addr, "DynamicBuffer: base address cannot be 0");
617    return addr;
618}
619
620Value * DynamicBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const b, Value * handle, Value * blockIndex) const {
621    Value * const wkgBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::WorkingBlocks))}));
622    return b->CreateGEP(getBaseAddress(b, handle), b->CreateURem(blockIndex, wkgBlocks));
623}
624
625Value * DynamicBuffer::getRawItemPointer(IDISA::IDISA_Builder * const b, Value * handle, Value * streamIndex, Value * absolutePosition) const {
626    Value * absBlock = b->CreateUDiv(absolutePosition, b->getSize(b->getBitBlockWidth()));
627    Value * blockPos = b->CreateURem(absolutePosition, b->getSize(b->getBitBlockWidth()));
628    Value * blockPtr = b->CreateGEP(getStreamSetBlockPtr(b, handle, absBlock), {b->getInt32(0), streamIndex});
629    const auto bw = mBaseType->getArrayElementType()->getScalarSizeInBits();
630    if (bw < 8) {
631        assert (bw  == 1 || bw == 2 || bw == 4);
632        blockPos = b->CreateUDiv(blockPos, ConstantInt::get(blockPos->getType(), 8 / bw));
633        blockPtr = b->CreatePointerCast(blockPtr, b->getInt8PtrTy());
634    } else {
635        blockPtr = b->CreatePointerCast(blockPtr, b->getIntNTy(bw)->getPointerTo());
636    }
637    return b->CreateGEP(blockPtr, blockPos);
638}
639
640
641Value * DynamicBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const b, Value * handle, Value * fromPosition) const {
642    Constant * blockSize = b->getSize(b->getBitBlockWidth());
643    if (isa<ArrayType>(mType) && dyn_cast<ArrayType>(mType)->getNumElements() > 1) {
644        return b->CreateSub(blockSize, b->CreateURem(fromPosition, blockSize));
645    } else {
646        Value * const bufBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::WorkingBlocks))}));
647        Value * bufSize = b->CreateMul(bufBlocks, blockSize);
648        return b->CreateSub(bufSize, b->CreateURem(fromPosition, bufSize, "linearItems"));
649    }
650}
651
652Value * DynamicBuffer::getLinearlyAccessibleBlocks(IDISA::IDISA_Builder * const b, Value * handle, Value * fromBlock) const {
653    Value * const bufBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::WorkingBlocks))}));
654    return b->CreateSub(bufBlocks, b->CreateURem(fromBlock, bufBlocks), "linearBlocks");
655}
656
657void DynamicBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) {
658    Value * handle = b->CreateCacheAlignedAlloca(mBufferStructType);
659    size_t numStreams = 1;
660    if (isa<ArrayType>(mBaseType)) {
661        numStreams = mBaseType->getArrayNumElements();
662    }
663    const auto fieldWidth = mBaseType->getArrayElementType()->getScalarSizeInBits();
664    Value * bufSize = b->getSize((mBufferBlocks + mOverflowBlocks) * b->getBitBlockWidth() * numStreams * fieldWidth/8);
665    bufSize = b->CreateRoundUp(bufSize, b->getSize(b->getCacheAlignment()));
666    Value * bufBasePtrField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::BaseAddress))});
667    Value * bufPtr = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufSize), bufBasePtrField->getType()->getPointerElementType());
668    b->CreateStore(bufPtr, bufBasePtrField);
669    b->CreateStore(bufSize, b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::AllocatedCapacity))}));
670    b->CreateStore(b->getSize(mBufferBlocks), b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::WorkingBlocks))}));
671    b->CreateStore(b->getSize(-1), b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::Length))}));
672    b->CreateStore(b->getSize(0), b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::ProducedPosition))}));
673    b->CreateStore(b->getSize(0), b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::ConsumedPosition))}));
674    mStreamSetBufferPtr = handle;
675}
676
677void DynamicBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) const {
678    /* Free the dynamically allocated buffer, but not the stack-allocated buffer struct. */
679    b->CreateFree(b->CreateLoad(b->CreateGEP(mStreamSetBufferPtr, {b->getInt32(0), b->getInt32(int(Field::BaseAddress))})));
680}
681
682
683DynamicBuffer::DynamicBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t initialCapacity, size_t overflow, unsigned swizzle, unsigned addrSpace)
684: StreamSetBuffer(BufferKind::DynamicBuffer, type, resolveStreamSetType(b, type), initialCapacity, addrSpace)
685, mBufferStructType(StructType::get(resolveStreamSetType(b, type)->getPointerTo(addrSpace), 
686                                    b->getSizeTy(), b->getSizeTy(), b->getSizeTy(), b->getSizeTy(), b->getSizeTy(), nullptr))
687, mSwizzleFactor(swizzle)
688, mOverflowBlocks(overflow)
689{
690    mUniqueID = "DB";
691    if (swizzle != 1) {
692        mUniqueID += "s" + std::to_string(swizzle);
693    }
694        if (overflow != 0) {
695        mUniqueID += "o" + std::to_string(overflow);
696    }
697    if (addrSpace != 0) {
698        mUniqueID += "@" + std::to_string(addrSpace);
699    }
700}
701
702
703inline StreamSetBuffer::StreamSetBuffer(BufferKind k, Type * baseType, Type * resolvedType, unsigned BufferBlocks, unsigned AddressSpace)
704: mBufferKind(k)
705, mType(resolvedType)
706, mBufferBlocks(BufferBlocks)
707, mAddressSpace(AddressSpace)
708, mStreamSetBufferPtr(nullptr)
709, mBaseType(baseType)
710, mProducer(nullptr) {
711
712}
713
714StreamSetBuffer::~StreamSetBuffer() { }
715
716// Helper routines
717ArrayType * resolveStreamSetType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type) {
718    unsigned numElements = 1;
719    if (LLVM_LIKELY(type->isArrayTy())) {
720        numElements = type->getArrayNumElements();
721        type = type->getArrayElementType();
722    }
723    if (LLVM_LIKELY(type->isVectorTy() && type->getVectorNumElements() == 0)) {
724        type = type->getVectorElementType();
725        if (LLVM_LIKELY(type->isIntegerTy())) {
726            const auto fieldWidth = cast<IntegerType>(type)->getBitWidth();
727            type = b->getBitBlockType();
728            if (fieldWidth != 1) {
729                type = ArrayType::get(type, fieldWidth);
730            }
731            return ArrayType::get(type, numElements);
732        }
733    }
734    std::string tmp;
735    raw_string_ostream out(tmp);
736    type->print(out);
737    out << " is an unvalid stream set buffer type.";
738    report_fatal_error(out.str());
739}
740
741StructType * resolveExpandableStreamSetType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type) {
742    if (LLVM_LIKELY(type->isArrayTy())) {
743        type = type->getArrayElementType();
744    }
745    if (LLVM_LIKELY(type->isVectorTy() && type->getVectorNumElements() == 0)) {
746        type = type->getVectorElementType();
747        if (LLVM_LIKELY(type->isIntegerTy())) {
748            const auto fieldWidth = cast<IntegerType>(type)->getBitWidth();
749            type = b->getBitBlockType();
750            if (fieldWidth != 1) {
751                type = ArrayType::get(type, fieldWidth);
752            }
753            return StructType::get(b->getSizeTy(), type->getPointerTo(), nullptr);
754        }
755    }
756    std::string tmp;
757    raw_string_ostream out(tmp);
758    type->print(out);
759    out << " is an unvalid stream set buffer type.";
760    report_fatal_error(out.str());
761}
Note: See TracBrowser for help on using the repository browser.