source: icGREP/icgrep-devel/icgrep/kernels/streamset.cpp @ 5429

Last change on this file since 5429 was 5429, checked in by nmedfort, 2 years ago

Refactored source kernels. icgrep from stdin should now be able to handle any file size.

File size: 26.7 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5
6#include "streamset.h"
7#include <IR_Gen/idisa_builder.h>  // for IDISA_Builder
8#include <llvm/IR/BasicBlock.h>    // for BasicBlock
9#include <llvm/IR/Constants.h>     // for ConstantInt
10#include <llvm/IR/DataLayout.h>    // for DataLayout
11#include <llvm/IR/DerivedTypes.h>  // for IntegerType (ptr only), PointerType
12#include <llvm/IR/Module.h>        // for Module
13#include <llvm/IR/Value.h>         // for Value
14#include <llvm/Support/raw_ostream.h>
15#include <llvm/IR/CFG.h>
16#include <kernels/kernel.h>
17#include <toolchain/toolchain.h>
18
19namespace llvm { class Constant; }
20namespace llvm { class Function; }
21
22using namespace parabix;
23using namespace llvm;
24using namespace IDISA;
25
26ArrayType * resolveStreamSetType(IDISA_Builder * const b, Type * type);
27
28StructType * resolveExpandableStreamSetType(IDISA_Builder * const b, Type * type);
29
30void StreamSetBuffer::allocateBuffer() {
31    if (LLVM_LIKELY(mStreamSetBufferPtr == nullptr)) {
32        Type * const ty = getType();
33        mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(ty, iBuilder->getSize(mBufferBlocks));
34        iBuilder->CreateAlignedStore(Constant::getNullValue(ty), mStreamSetBufferPtr, iBuilder->getCacheAlignment());
35    } else {
36        report_fatal_error("StreamSetBuffer::allocateBuffer() was called twice on the same stream set");
37    }
38}
39
40Value * StreamSetBuffer::getStreamBlockPtr(Value * self, Value * streamIndex, Value * blockIndex, const bool /* readOnly */) const {
41    iBuilder->CreateAssert(iBuilder->CreateICmpULT(streamIndex, getStreamSetCount(self)), "StreamSetBuffer: out-of-bounds stream access");
42    return iBuilder->CreateGEP(getStreamSetBlockPtr(self, blockIndex), {iBuilder->getInt32(0), streamIndex});
43}
44
45Value * StreamSetBuffer::getStreamPackPtr(Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool /* readOnly */) const {
46    iBuilder->CreateAssert(iBuilder->CreateICmpULT(streamIndex, getStreamSetCount(self)), "StreamSetBuffer: out-of-bounds stream access");
47    return iBuilder->CreateGEP(getStreamSetBlockPtr(self, blockIndex), {iBuilder->getInt32(0), streamIndex, packIndex});
48}
49
50void StreamSetBuffer::setBaseAddress(Value * /* self */, Value * /* addr */) const {
51    report_fatal_error("setBaseAddress is not supported by this buffer type");
52}
53
54Value * StreamSetBuffer::getBufferedSize(Value * /* self */) const {
55    report_fatal_error("getBufferedSize is not supported by this buffer type");
56}
57
58void StreamSetBuffer::setBufferedSize(Value * /* self */, llvm::Value * /* size */) const {
59    report_fatal_error("setBufferedSize is not supported by this buffer type");
60}
61
62inline bool StreamSetBuffer::isCapacityGuaranteed(const Value * const index, const size_t capacity) const {
63    if (LLVM_UNLIKELY(isa<ConstantInt>(index))) {
64        if (LLVM_LIKELY(cast<ConstantInt>(index)->getLimitedValue() < capacity)) {
65            return true;
66        }
67    }
68    return false;
69}
70
71Value * StreamSetBuffer::getStreamSetCount(Value *) const {
72    uint64_t count = 1;
73    if (isa<ArrayType>(mBaseType)) {
74        count = mBaseType->getArrayNumElements();
75    }
76    return iBuilder->getSize(count);
77}
78
79inline Value * StreamSetBuffer::modByBufferBlocks(Value * const offset) const {
80    assert (offset->getType()->isIntegerTy());
81    if (isCapacityGuaranteed(offset, mBufferBlocks)) {
82        return offset;
83    } else if (mBufferBlocks == 1) {
84        return ConstantInt::getNullValue(iBuilder->getSizeTy());
85    } else if ((mBufferBlocks & (mBufferBlocks - 1)) == 0) { // is power of 2
86        return iBuilder->CreateAnd(offset, ConstantInt::get(offset->getType(), mBufferBlocks - 1));
87    } else {
88        return iBuilder->CreateURem(offset, ConstantInt::get(offset->getType(), mBufferBlocks));
89    }
90}
91
92/**
93 * @brief getRawItemPointer
94 *
95 * get a raw pointer the iN field at position absoluteItemPosition of the stream number streamIndex of the stream set.
96 * In the case of a stream whose fields are less than one byte (8 bits) in size, the pointer is to the containing byte.
97 * The type of the pointer is i8* for fields of 8 bits or less, otherwise iN* for N-bit fields.
98 */
99Value * StreamSetBuffer::getRawItemPointer(Value * self, Value * streamIndex, Value * absolutePosition) const {
100    Value * ptr = getBaseAddress(self);
101    if (!isa<ConstantInt>(streamIndex) || !cast<ConstantInt>(streamIndex)->isZero()) {
102        ptr = iBuilder->CreateGEP(ptr, {iBuilder->getInt32(0), streamIndex});
103    }
104    IntegerType * const ty = cast<IntegerType>(mBaseType->getArrayElementType()->getVectorElementType());
105    ptr = iBuilder->CreatePointerCast(ptr, ty->getPointerTo());
106    if (LLVM_UNLIKELY(ty->getBitWidth() < 8)) {
107        const auto bw = ty->getBitWidth();
108        if (LLVM_LIKELY((bw & (bw - 1)) == 0)) { // is power of 2
109            absolutePosition = iBuilder->CreateUDiv(absolutePosition, ConstantInt::get(absolutePosition->getType(), 8 / bw));
110        } else {
111            absolutePosition = iBuilder->CreateMul(absolutePosition, ConstantInt::get(absolutePosition->getType(), bw));
112            absolutePosition = iBuilder->CreateUDiv(absolutePosition, ConstantInt::get(absolutePosition->getType(), 8));
113        }
114    }
115    return iBuilder->CreateGEP(ptr, absolutePosition);
116}
117
118Value * StreamSetBuffer::getLinearlyAccessibleItems(Value * self, Value * fromPosition) const {
119    if (isa<ArrayType>(mType) && dyn_cast<ArrayType>(mType)->getNumElements() > 1) {
120        Constant * stride = iBuilder->getSize(iBuilder->getStride());
121        return iBuilder->CreateSub(stride, iBuilder->CreateURem(fromPosition, stride));
122    } else {
123        Constant * bufSize = iBuilder->getSize(mBufferBlocks * iBuilder->getStride());
124        return iBuilder->CreateSub(bufSize, iBuilder->CreateURem(fromPosition, bufSize));
125    }
126}
127
128Value * StreamSetBuffer::getLinearlyAccessibleBlocks(Value * self, Value * fromBlock) const {
129    Constant * bufBlocks = iBuilder->getSize(mBufferBlocks);
130    return iBuilder->CreateSub(bufBlocks, iBuilder->CreateURem(fromBlock, bufBlocks));
131}
132
133void StreamSetBuffer::reserveBytes(Value * self, llvm::Value *requested) const {
134    report_fatal_error("reserve() can only be used with ExtensibleBuffers");
135}
136
137Value * StreamSetBuffer::getBaseAddress(Value * self) const {
138    return self;
139}
140
141void StreamSetBuffer::releaseBuffer(Value * /* self */) const {
142    /* do nothing: memory is stack allocated */
143}
144
145// Single Block Buffer
146
147// For a single block buffer, the block pointer is always the buffer base pointer.
148Value * SingleBlockBuffer::getStreamSetBlockPtr(Value * self, Value *) const {
149    return self;
150}
151
152// Source File Buffer
153Value * SourceBuffer::getBufferedSize(Value * self) const {
154    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
155    return iBuilder->CreateLoad(ptr);
156}
157
158void SourceBuffer::setBufferedSize(Value * self, llvm::Value * size) const {
159    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
160    iBuilder->CreateStore(size, ptr);
161}
162
163void SourceBuffer::setBaseAddress(Value * self, Value * addr) const {
164    Value * const ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
165    iBuilder->CreateStore(iBuilder->CreatePointerCast(addr, ptr->getType()->getPointerElementType()), ptr);
166}
167
168Value * SourceBuffer::getBaseAddress(Value * const self) const {
169    Value * const ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
170    Value * const addr = iBuilder->CreateLoad(ptr);
171    return addr;
172}
173
174Value * SourceBuffer::getStreamSetBlockPtr(Value * self, Value * blockIndex) const {
175    return iBuilder->CreateGEP(getBaseAddress(self), blockIndex);
176}
177
178Value * SourceBuffer::getLinearlyAccessibleItems(Value * self, Value *) const {
179    report_fatal_error("External buffers: getLinearlyAccessibleItems is not supported.");
180}
181
182// External File Buffer
183void ExternalBuffer::allocateBuffer() {
184    report_fatal_error("External buffers cannot be allocated.");
185}
186
187Value * ExternalBuffer::getStreamSetBlockPtr(Value * self, Value * blockIndex) const {
188    return iBuilder->CreateGEP(getBaseAddress(self), blockIndex);
189}
190
191Value * ExternalBuffer::getLinearlyAccessibleItems(Value *, Value *) const {
192    report_fatal_error("External buffers: getLinearlyAccessibleItems is not supported.");
193}
194
195// Circular Buffer
196Value * CircularBuffer::getStreamSetBlockPtr(Value * const self, Value * const blockIndex) const {
197    return iBuilder->CreateGEP(getBaseAddress(self), modByBufferBlocks(blockIndex));
198}
199
200// CircularCopybackBuffer Buffer
201void CircularCopybackBuffer::allocateBuffer() {
202    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType(), iBuilder->getSize(mBufferBlocks + mOverflowBlocks));
203}
204
205void CircularCopybackBuffer::createCopyBack(Value * self, Value * overFlowItems) const {
206    Type * size_ty = iBuilder->getSizeTy();
207    Type * i8ptr = iBuilder->getInt8PtrTy();
208    Constant * blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
209    Function * f = iBuilder->GetInsertBlock()->getParent();
210    BasicBlock * wholeBlockCopy = BasicBlock::Create(iBuilder->getContext(), "wholeBlockCopy", f, 0);
211    BasicBlock * partialBlockCopy = BasicBlock::Create(iBuilder->getContext(), "partialBlockCopy", f, 0);
212    BasicBlock * copyBackDone = BasicBlock::Create(iBuilder->getContext(), "copyBackDone", f, 0);
213    unsigned numStreams = getType()->getArrayNumElements();
214    auto elemTy = getType()->getArrayElementType();
215    unsigned fieldWidth = isa<ArrayType>(elemTy) ? elemTy->getArrayNumElements() : 1;
216    Value * overFlowAreaPtr = iBuilder->CreateGEP(self, iBuilder->getSize(mBufferBlocks));
217    Value * overFlowBlocks = iBuilder->CreateUDiv(overFlowItems, blockSize);
218    Value * partialItems = iBuilder->CreateURem(overFlowItems, blockSize);
219    Value * partialBlockTargetPtr = iBuilder->CreateGEP(self, overFlowBlocks);
220    Value * partialBlockSourcePtr = iBuilder->CreateGEP(overFlowAreaPtr, overFlowBlocks);
221    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(overFlowBlocks, iBuilder->getSize(0)), wholeBlockCopy, partialBlockCopy);
222    iBuilder->SetInsertPoint(wholeBlockCopy);
223    unsigned alignment = iBuilder->getBitBlockWidth() / 8;
224    Value * copyLength = iBuilder->CreateSub(iBuilder->CreatePtrToInt(partialBlockTargetPtr, size_ty), iBuilder->CreatePtrToInt(self, size_ty));
225    iBuilder->CreateMemMove(iBuilder->CreateBitCast(self, i8ptr), iBuilder->CreateBitCast(overFlowAreaPtr, i8ptr), copyLength, alignment);
226    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(partialItems, iBuilder->getSize(0)), partialBlockCopy, copyBackDone);
227    iBuilder->SetInsertPoint(partialBlockCopy);
228    Value * copyBits = iBuilder->CreateMul(overFlowItems, iBuilder->getSize(fieldWidth));
229    Value * copyBytes = iBuilder->CreateLShr(iBuilder->CreateAdd(copyBits, iBuilder->getSize(7)), iBuilder->getSize(3));
230    for (unsigned strm = 0; strm < numStreams; strm++) {
231        Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
232        Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
233        iBuilder->CreateMemMove(iBuilder->CreateBitCast(strmTargetPtr, i8ptr), iBuilder->CreateBitCast(strmSourcePtr, i8ptr), copyBytes, alignment);
234    }
235    iBuilder->CreateBr(copyBackDone);
236    iBuilder->SetInsertPoint(copyBackDone);
237}
238
239Value * CircularCopybackBuffer::getStreamSetBlockPtr(Value * self, Value * blockIndex) const {
240    return iBuilder->CreateGEP(getBaseAddress(self), modByBufferBlocks(blockIndex));
241}
242
243// SwizzledCopybackBuffer Buffer
244
245void SwizzledCopybackBuffer::allocateBuffer() {
246    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType(), iBuilder->getSize(mBufferBlocks + mOverflowBlocks));
247}
248
249void SwizzledCopybackBuffer::createCopyBack(Value * self, Value * overFlowItems) const {
250    Type * size_ty = iBuilder->getSizeTy();
251    Type * i8ptr = iBuilder->getInt8PtrTy();
252    Constant * blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
253    Function * f = iBuilder->GetInsertBlock()->getParent();
254    BasicBlock * wholeBlockCopy = BasicBlock::Create(iBuilder->getContext(), "wholeBlockCopy", f, 0);
255    BasicBlock * partialBlockCopy = BasicBlock::Create(iBuilder->getContext(), "partialBlockCopy", f, 0);
256    BasicBlock * copyBackDone = BasicBlock::Create(iBuilder->getContext(), "copyBackDone", f, 0);
257    unsigned numStreams = getType()->getArrayNumElements();
258    unsigned swizzleFactor = iBuilder->getBitBlockWidth()/mFieldWidth;
259    auto elemTy = getType()->getArrayElementType();
260    unsigned fieldWidth = isa<ArrayType>(elemTy) ? elemTy->getArrayNumElements() : 1;
261    Value * overFlowAreaPtr = iBuilder->CreateGEP(self, iBuilder->getSize(mBufferBlocks));
262    Value * overFlowBlocks = iBuilder->CreateUDiv(overFlowItems, blockSize);
263    Value * partialItems = iBuilder->CreateURem(overFlowItems, blockSize);
264    Value * partialBlockTargetPtr = iBuilder->CreateGEP(self, overFlowBlocks);
265    Value * partialBlockSourcePtr = iBuilder->CreateGEP(overFlowAreaPtr, overFlowBlocks);
266    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(overFlowBlocks, iBuilder->getSize(0)), wholeBlockCopy, partialBlockCopy);
267    iBuilder->SetInsertPoint(wholeBlockCopy);
268    unsigned alignment = iBuilder->getBitBlockWidth() / 8;
269    Value * copyLength = iBuilder->CreateSub(iBuilder->CreatePtrToInt(partialBlockTargetPtr, size_ty), iBuilder->CreatePtrToInt(self, size_ty));
270    iBuilder->CreateMemMove(iBuilder->CreateBitCast(self, i8ptr), iBuilder->CreateBitCast(overFlowAreaPtr, i8ptr), copyLength, alignment);
271    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(partialItems, iBuilder->getSize(0)), partialBlockCopy, copyBackDone);
272    iBuilder->SetInsertPoint(partialBlockCopy);
273    Value * copyBits = iBuilder->CreateMul(overFlowItems, iBuilder->getSize(fieldWidth * swizzleFactor));
274    Value * copyBytes = iBuilder->CreateLShr(iBuilder->CreateAdd(copyBits, iBuilder->getSize(7)), iBuilder->getSize(3));
275    for (unsigned strm = 0; strm < numStreams; strm += swizzleFactor) {
276        Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
277        Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
278        iBuilder->CreateMemMove(iBuilder->CreateBitCast(strmTargetPtr, i8ptr), iBuilder->CreateBitCast(strmSourcePtr, i8ptr), copyBytes, alignment);
279    }
280    iBuilder->CreateBr(copyBackDone);
281    iBuilder->SetInsertPoint(copyBackDone);
282}
283
284Value * SwizzledCopybackBuffer::getStreamSetBlockPtr(Value * self, Value * blockIndex) const {
285    return iBuilder->CreateGEP(getBaseAddress(self), modByBufferBlocks(blockIndex));
286}
287
288SwizzledCopybackBuffer::SwizzledCopybackBuffer(IDISA::IDISA_Builder * b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned fieldwidth, unsigned AddressSpace)
289: StreamSetBuffer(BufferKind::SwizzledCopybackBuffer, b, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks), mFieldWidth(fieldwidth) {
290    mUniqueID = "SW" + std::to_string(fieldwidth) + ":" + std::to_string(bufferBlocks);
291    if (mOverflowBlocks != 1) {
292        mUniqueID += "_" + std::to_string(mOverflowBlocks);
293    }
294    if (AddressSpace > 0) {
295        mUniqueID += "@" + std::to_string(AddressSpace);
296    }
297}
298
299// Expandable Buffer
300
301void ExpandableBuffer::allocateBuffer() {
302    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType());
303    Value * const capacityPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
304    iBuilder->CreateStore(iBuilder->getSize(mInitialCapacity), capacityPtr);
305    Type * const bufferType = getType()->getStructElementType(1)->getPointerElementType();
306    Constant * const bufferWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(bufferType), iBuilder->getSizeTy(), false);
307    Constant * const size = ConstantExpr::getMul(iBuilder->getSize(mBufferBlocks * mInitialCapacity), bufferWidth);
308    Value * const ptr = iBuilder->CreateAlignedMalloc(size, iBuilder->getCacheAlignment());
309    iBuilder->CreateMemZero(ptr, size, bufferType->getPrimitiveSizeInBits() / 8);
310    Value * const streamSetPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
311    iBuilder->CreateStore(iBuilder->CreatePointerCast(ptr, bufferType->getPointerTo()), streamSetPtr);
312}
313
314std::pair<Value *, Value *> ExpandableBuffer::getInternalStreamBuffer(Value * self, Value * streamIndex, Value * blockIndex, const bool readOnly) const {
315
316    // ENTRY
317    Value * const capacityPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
318    Value * const capacity = iBuilder->CreateLoad(capacityPtr);
319    Value * const streamSetPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
320    Value * const streamSet = iBuilder->CreateLoad(streamSetPtr);
321    blockIndex = modByBufferBlocks(blockIndex);
322
323    assert (streamIndex->getType() == capacity->getType());
324    Value * const cond = iBuilder->CreateICmpULT(streamIndex, capacity);
325
326    // Are we guaranteed that we can access this stream?
327    if (readOnly || isCapacityGuaranteed(streamIndex, mInitialCapacity)) {
328        iBuilder->CreateAssert(cond, "ExpandableBuffer: out-of-bounds stream access");
329        Value * offset = iBuilder->CreateAdd(iBuilder->CreateMul(blockIndex, capacity), streamIndex);
330        return {streamSet, offset};
331    }
332
333    BasicBlock * const entry = iBuilder->GetInsertBlock();
334    BasicBlock * const expand = BasicBlock::Create(iBuilder->getContext(), "expand", entry->getParent());
335    BasicBlock * const resume = BasicBlock::Create(iBuilder->getContext(), "resume", entry->getParent());
336
337    iBuilder->CreateLikelyCondBr(cond, resume, expand);
338
339    // EXPAND
340    iBuilder->SetInsertPoint(expand);
341
342    Type * elementType = getType()->getStructElementType(1)->getPointerElementType();
343    Constant * const vectorWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(elementType), capacity->getType(), false);
344
345    Value * newCapacity = iBuilder->CreateAdd(streamIndex, iBuilder->getSize(1));
346    newCapacity = iBuilder->CreateCeilLog2(newCapacity);
347    newCapacity = iBuilder->CreateShl(iBuilder->getSize(1), newCapacity, "newCapacity");
348
349    std::string tmp;
350    raw_string_ostream out(tmp);
351    out << "__expand";
352    elementType->print(out);
353    std::string name = out.str();
354
355    Module * const m = iBuilder->getModule();
356    Function * expandFunction = m->getFunction(name);
357
358    if (expandFunction == nullptr) {
359
360        const auto ip = iBuilder->saveIP();
361
362        FunctionType * fty = FunctionType::get(elementType->getPointerTo(), {elementType->getPointerTo(), iBuilder->getSizeTy(), iBuilder->getSizeTy()}, false);
363        expandFunction = Function::Create(fty, GlobalValue::PrivateLinkage, name, m);
364
365        auto args = expandFunction->arg_begin();
366        Value * streamSet = &*args++;
367        Value * capacity = &*args++;
368        Value * newCapacity = &*args;
369
370        BasicBlock * entry = BasicBlock::Create(iBuilder->getContext(), "entry", expandFunction);
371        iBuilder->SetInsertPoint(entry);
372
373        Value * size = iBuilder->CreateMul(newCapacity, iBuilder->getSize(mBufferBlocks));
374        Value * newStreamSet = iBuilder->CreatePointerCast(iBuilder->CreateAlignedMalloc(iBuilder->CreateMul(size, vectorWidth), iBuilder->getCacheAlignment()), elementType->getPointerTo());
375        Value * const diffCapacity = iBuilder->CreateMul(iBuilder->CreateSub(newCapacity, capacity), vectorWidth);
376
377        const auto alignment = elementType->getPrimitiveSizeInBits() / 8;
378        for (unsigned i = 0; i < mBufferBlocks; ++i) {
379            ConstantInt * const offset = iBuilder->getSize(i);
380            Value * srcOffset = iBuilder->CreateMul(capacity, offset);
381            Value * srcPtr = iBuilder->CreateGEP(streamSet, srcOffset);
382            Value * destOffset = iBuilder->CreateMul(newCapacity, offset);
383            Value * destPtr = iBuilder->CreateGEP(newStreamSet, destOffset);
384            iBuilder->CreateMemCpy(destPtr, srcPtr, iBuilder->CreateMul(capacity, vectorWidth), alignment);
385            Value * destZeroOffset = iBuilder->CreateAdd(destOffset, capacity);
386            Value * destZeroPtr = iBuilder->CreateGEP(newStreamSet, destZeroOffset);
387            iBuilder->CreateMemZero(destZeroPtr, diffCapacity, alignment);
388        }
389
390        iBuilder->CreateAlignedFree(streamSet);
391
392        iBuilder->CreateRet(newStreamSet);
393
394        iBuilder->restoreIP(ip);
395    }
396
397    Value * newStreamSet = iBuilder->CreateCall(expandFunction, {streamSet, capacity, newCapacity});
398    iBuilder->CreateStore(newStreamSet, streamSetPtr);
399    iBuilder->CreateStore(newCapacity, capacityPtr);
400
401    iBuilder->CreateBr(resume);
402
403    // RESUME
404    iBuilder->SetInsertPoint(resume);
405
406    PHINode * phiStreamSet = iBuilder->CreatePHI(streamSet->getType(), 2);
407    phiStreamSet->addIncoming(streamSet, entry);
408    phiStreamSet->addIncoming(newStreamSet, expand);
409
410    PHINode * phiCapacity = iBuilder->CreatePHI(capacity->getType(), 2);
411    phiCapacity->addIncoming(capacity, entry);
412    phiCapacity->addIncoming(newCapacity, expand);
413
414    Value * offset = iBuilder->CreateAdd(iBuilder->CreateMul(blockIndex, phiCapacity), streamIndex);
415
416    return {phiStreamSet, offset};
417}
418
419Value * ExpandableBuffer::getStreamBlockPtr(Value * self, Value * streamIndex, Value * blockIndex, const bool readOnly) const {
420    Value * ptr, * offset;
421    std::tie(ptr, offset) = getInternalStreamBuffer(self, streamIndex, blockIndex, readOnly);
422    return iBuilder->CreateGEP(ptr, offset);
423}
424
425Value * ExpandableBuffer::getStreamPackPtr(Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool readOnly) const {
426    Value * ptr, * offset;
427    std::tie(ptr, offset) = getInternalStreamBuffer(self, streamIndex, blockIndex, readOnly);
428    return iBuilder->CreateGEP(ptr, {offset, packIndex});
429}
430
431Value * ExpandableBuffer::getStreamSetCount(Value * self) const {
432    return iBuilder->CreateLoad(iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)}));
433}
434
435Value * ExpandableBuffer::getBaseAddress(Value * self) const {
436    return iBuilder->CreateLoad(iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)}));
437}
438
439void ExpandableBuffer::releaseBuffer(Value * self) const {
440    iBuilder->CreateAlignedFree(getBaseAddress(self));
441}
442
443Value * ExpandableBuffer::getStreamSetBlockPtr(Value *, Value *) const {
444    report_fatal_error("Expandable buffers: getStreamSetBlockPtr is not supported.");
445}
446
447Value * ExpandableBuffer::getLinearlyAccessibleItems(Value * self, Value *) const {
448    report_fatal_error("Expandable buffers: getLinearlyAccessibleItems is not supported.");
449}
450
451// Constructors
452SingleBlockBuffer::SingleBlockBuffer(IDISA::IDISA_Builder * b, Type * type)
453: StreamSetBuffer(BufferKind::BlockBuffer, b, type, resolveStreamSetType(b, type), 1, 0) {
454    mUniqueID = "S";
455
456}
457
458SourceBuffer::SourceBuffer(IDISA::IDISA_Builder * b, Type * type, unsigned AddressSpace)
459: StreamSetBuffer(BufferKind::SourceBuffer, b, type, StructType::get(resolveStreamSetType(b, type)->getPointerTo(), b->getSizeTy(), nullptr), 0, AddressSpace) {
460    mUniqueID = "M"; // + std::to_string(bufferBlocks);
461    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
462}
463
464ExternalBuffer::ExternalBuffer(IDISA::IDISA_Builder * b, Type * type, llvm::Value * addr, unsigned AddressSpace)
465: StreamSetBuffer(BufferKind::ExternalBuffer, b, type, resolveStreamSetType(b, type), 0, AddressSpace) {
466    mUniqueID = "E";
467    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
468    mStreamSetBufferPtr = b->CreatePointerBitCastOrAddrSpaceCast(addr, getPointerType());
469}
470
471CircularBuffer::CircularBuffer(IDISA::IDISA_Builder * b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
472: StreamSetBuffer(BufferKind::CircularBuffer, b, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
473    mUniqueID = "C" + std::to_string(bufferBlocks);
474    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
475}
476
477CircularCopybackBuffer::CircularCopybackBuffer(IDISA::IDISA_Builder * b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned AddressSpace)
478: StreamSetBuffer(BufferKind::CircularCopybackBuffer, b, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks) {
479    mUniqueID = "CC" + std::to_string(bufferBlocks);
480    if (mOverflowBlocks != 1) mUniqueID += "_" + std::to_string(mOverflowBlocks);
481    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
482}
483
484ExpandableBuffer::ExpandableBuffer(IDISA::IDISA_Builder * b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
485: StreamSetBuffer(BufferKind::ExpandableBuffer, b, type, resolveExpandableStreamSetType(b, type), bufferBlocks, AddressSpace)
486, mInitialCapacity(type->getArrayNumElements()) {
487    mUniqueID = "XP" + std::to_string(bufferBlocks);
488    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
489}
490
491inline StreamSetBuffer::StreamSetBuffer(BufferKind k, IDISA::IDISA_Builder * b, Type * baseType, Type * resolvedType, unsigned blocks, unsigned AddressSpace)
492: mBufferKind(k)
493, iBuilder(b)
494, mType(resolvedType)
495, mBufferBlocks(blocks)
496, mAddressSpace(AddressSpace)
497, mStreamSetBufferPtr(nullptr)
498, mBaseType(baseType)
499, mProducer(nullptr) {
500
501}
502
503StreamSetBuffer::~StreamSetBuffer() { }
504
505// Helper routines
506ArrayType * resolveStreamSetType(IDISA_Builder * const b, Type * type) {
507    unsigned numElements = 1;
508    if (LLVM_LIKELY(type->isArrayTy())) {
509        numElements = type->getArrayNumElements();
510        type = type->getArrayElementType();
511    }
512    if (LLVM_LIKELY(type->isVectorTy() && type->getVectorNumElements() == 0)) {
513        type = type->getVectorElementType();
514        if (LLVM_LIKELY(type->isIntegerTy())) {
515            const auto fieldWidth = cast<IntegerType>(type)->getBitWidth();
516            type = b->getBitBlockType();
517            if (fieldWidth != 1) {
518                type = ArrayType::get(type, fieldWidth);
519            }
520            return ArrayType::get(type, numElements);
521        }
522    }
523    std::string tmp;
524    raw_string_ostream out(tmp);
525    type->print(out);
526    out << " is an unvalid stream set buffer type.";
527    report_fatal_error(out.str());
528}
529
530StructType * resolveExpandableStreamSetType(IDISA_Builder * const b, Type * type) {
531    if (LLVM_LIKELY(type->isArrayTy())) {
532        type = type->getArrayElementType();
533    }
534    if (LLVM_LIKELY(type->isVectorTy() && type->getVectorNumElements() == 0)) {
535        type = type->getVectorElementType();
536        if (LLVM_LIKELY(type->isIntegerTy())) {
537            const auto fieldWidth = cast<IntegerType>(type)->getBitWidth();
538            type = b->getBitBlockType();
539            if (fieldWidth != 1) {
540                type = ArrayType::get(type, fieldWidth);
541            }
542            return StructType::get(b->getSizeTy(), type->getPointerTo(), nullptr);
543        }
544    }
545    std::string tmp;
546    raw_string_ostream out(tmp);
547    type->print(out);
548    out << " is an unvalid stream set buffer type.";
549    report_fatal_error(out.str());
550}
Note: See TracBrowser for help on using the repository browser.