source: icGREP/icgrep-devel/icgrep/kernels/streamset.cpp @ 5395

Last change on this file since 5395 was 5395, checked in by cameron, 3 years ago

Uniquify kernel names with buffer types/sizes; update u8u16 to use ParabixDriver?

File size: 28.0 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5
6#include "streamset.h"
7#include <IR_Gen/idisa_builder.h>  // for IDISA_Builder
8#include <llvm/IR/BasicBlock.h>    // for BasicBlock
9#include <llvm/IR/Constants.h>     // for ConstantInt
10#include <llvm/IR/DataLayout.h>    // for DataLayout
11#include <llvm/IR/DerivedTypes.h>  // for IntegerType (ptr only), PointerType
12#include <llvm/IR/Module.h>        // for Module
13#include <llvm/IR/Value.h>         // for Value
14#include <llvm/Support/raw_ostream.h>
15#include <llvm/IR/CFG.h>
16
17namespace llvm { class Constant; }
18namespace llvm { class Function; }
19
20using namespace parabix;
21using namespace llvm;
22using namespace IDISA;
23
24ArrayType * resolveStreamSetType(IDISA_Builder * const b, Type * type);
25
26StructType * resolveExpandableStreamSetType(IDISA_Builder * const b, Type * type);
27
28void StreamSetBuffer::allocateBuffer() {
29    Type * const ty = getType();
30    ConstantInt * blocks = iBuilder->getSize(mBufferBlocks);
31    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(ty, iBuilder->getSize(mBufferBlocks));
32    Constant * width = ConstantExpr::getMul(ConstantExpr::getSizeOf(ty), blocks);
33    iBuilder->CreateMemZero(mStreamSetBufferPtr, width, iBuilder->getCacheAlignment());
34}
35
36Value * StreamSetBuffer::getStreamBlockPtr(Value * self, Value * streamIndex, Value * blockIndex, const bool /* readOnly */) const {
37    iBuilder->CreateAssert(iBuilder->CreateICmpULT(streamIndex, getStreamSetCount(self)), "StreamSetBuffer: out-of-bounds stream access");
38    return iBuilder->CreateGEP(getStreamSetBlockPtr(getBaseAddress(self), blockIndex), {iBuilder->getInt32(0), streamIndex});
39}
40
41Value * StreamSetBuffer::getStreamPackPtr(Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool /* readOnly */) const {
42    iBuilder->CreateAssert(iBuilder->CreateICmpULT(streamIndex, getStreamSetCount(self)), "StreamSetBuffer: out-of-bounds stream access");
43    return iBuilder->CreateGEP(getStreamSetBlockPtr(getBaseAddress(self), blockIndex), {iBuilder->getInt32(0), streamIndex, packIndex});
44}
45
46inline bool StreamSetBuffer::isCapacityGuaranteed(const Value * const index, const size_t capacity) const {
47    if (LLVM_UNLIKELY(isa<ConstantInt>(index))) {
48        if (LLVM_LIKELY(cast<ConstantInt>(index)->getLimitedValue() < capacity)) {
49            return true;
50        }
51    }
52    return false;
53}
54
55Value * StreamSetBuffer::getStreamSetCount(Value *) const {
56    uint64_t count = 1;
57    if (isa<ArrayType>(mBaseType)) {
58        count = mBaseType->getArrayNumElements();
59    }
60    return iBuilder->getSize(count);
61}
62
63inline Value * StreamSetBuffer::modByBufferBlocks(Value * const offset) const {
64    assert (offset->getType()->isIntegerTy());
65    if (isCapacityGuaranteed(offset, mBufferBlocks)) {
66        return offset;
67    } else if (mBufferBlocks == 1) {
68        return ConstantInt::getNullValue(iBuilder->getSizeTy());
69    } else if ((mBufferBlocks & (mBufferBlocks - 1)) == 0) { // is power of 2
70        return iBuilder->CreateAnd(offset, ConstantInt::get(offset->getType(), mBufferBlocks - 1));
71    } else {
72        return iBuilder->CreateURem(offset, ConstantInt::get(offset->getType(), mBufferBlocks));
73    }
74}
75
76/**
77 * @brief getRawItemPointer
78 *
79 * get a raw pointer the iN field at position absoluteItemPosition of the stream number streamIndex of the stream set.
80 * In the case of a stream whose fields are less than one byte (8 bits) in size, the pointer is to the containing byte.
81 * The type of the pointer is i8* for fields of 8 bits or less, otherwise iN* for N-bit fields.
82 */
83Value * StreamSetBuffer::getRawItemPointer(Value * self, Value * streamIndex, Value * absolutePosition) const {
84    Value * ptr = getBaseAddress(self);
85    if (isa<ConstantInt>(streamIndex) && cast<ConstantInt>(streamIndex)->isZero()) {
86        ptr = iBuilder->CreateGEP(ptr, {iBuilder->getInt32(0), streamIndex});
87    }
88    IntegerType * const ty = cast<IntegerType>(mBaseType->getArrayElementType()->getVectorElementType());
89    ptr = iBuilder->CreatePointerCast(ptr, ty->getPointerTo());
90    if (LLVM_UNLIKELY(ty->getBitWidth() < 8)) {
91        const auto bw = ty->getBitWidth();
92        if (LLVM_LIKELY((bw & (bw - 1)) == 0)) { // is power of 2
93            absolutePosition = iBuilder->CreateUDiv(absolutePosition, ConstantInt::get(absolutePosition->getType(), 8 / bw));
94        } else {
95            absolutePosition = iBuilder->CreateMul(absolutePosition, ConstantInt::get(absolutePosition->getType(), bw));
96            absolutePosition = iBuilder->CreateUDiv(absolutePosition, ConstantInt::get(absolutePosition->getType(), 8));
97        }
98    }
99    return iBuilder->CreateGEP(ptr, absolutePosition);
100}
101
102Value * StreamSetBuffer::getLinearlyAccessibleItems(Value * self, Value * fromPosition) const {
103    if (isa<ArrayType>(mType) && dyn_cast<ArrayType>(mType)->getNumElements() > 1) {
104        Constant * stride = iBuilder->getSize(iBuilder->getStride());
105        return iBuilder->CreateSub(stride, iBuilder->CreateURem(fromPosition, stride));
106    } else {
107        Constant * bufSize = iBuilder->getSize(mBufferBlocks * iBuilder->getStride());
108        return iBuilder->CreateSub(bufSize, iBuilder->CreateURem(fromPosition, bufSize));
109    }
110}
111
112Value * StreamSetBuffer::getLinearlyAccessibleBlocks(Value * self, Value * fromBlock) const {
113    Constant * bufBlocks = iBuilder->getSize(mBufferBlocks);
114    return iBuilder->CreateSub(bufBlocks, iBuilder->CreateURem(fromBlock, bufBlocks));
115}
116
117void StreamSetBuffer::reserveBytes(Value * self, llvm::Value *requested) const {
118    report_fatal_error("reserve() can only be used with ExtensibleBuffers");
119}
120
121Value * StreamSetBuffer::getBaseAddress(Value * self) const {
122    return self;
123}
124
125void StreamSetBuffer::releaseBuffer(Value * /* self */) const {
126    /* do nothing: memory is stack allocated */
127}
128
129// Single Block Buffer
130
131// For a single block buffer, the block pointer is always the buffer base pointer.
132Value * SingleBlockBuffer::getStreamSetBlockPtr(Value * self, Value *) const {
133    return self;
134}
135
136// External File Buffer
137void ExternalFileBuffer::setStreamSetBuffer(Value * ptr) {
138    mStreamSetBufferPtr = iBuilder->CreatePointerBitCastOrAddrSpaceCast(ptr, getPointerType());
139}
140
141void ExternalFileBuffer::allocateBuffer() {
142    report_fatal_error("External buffers cannot be allocated.");
143}
144
145Value * ExternalFileBuffer::getStreamSetBlockPtr(Value * self, Value * blockIndex) const {
146    return iBuilder->CreateGEP(self, blockIndex);
147}
148
149Value * ExternalFileBuffer::getLinearlyAccessibleItems(Value * self, Value *) const {
150    report_fatal_error("External buffers: getLinearlyAccessibleItems is not supported.");
151}
152
153// ExtensibleBuffer
154Value * ExtensibleBuffer::getLinearlyAccessibleItems(Value * self, Value * fromPosition) const {
155    Value * capacityPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
156    Value * capacity = iBuilder->CreateLoad(capacityPtr);
157    return iBuilder->CreateSub(capacity, fromPosition);
158}
159
160void ExtensibleBuffer::allocateBuffer() {
161    Type * ty = getType();
162    Value * instance = iBuilder->CreateCacheAlignedAlloca(ty);
163    Value * const capacityPtr = iBuilder->CreateGEP(instance, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
164    Constant * initialSize = ConstantExpr::getSizeOf(ty->getStructElementType(1)->getPointerElementType());
165    initialSize = ConstantExpr::getMul(initialSize, iBuilder->getSize(mBufferBlocks));
166    initialSize = ConstantExpr::getIntegerCast(initialSize, iBuilder->getSizeTy(), false);
167    iBuilder->CreateStore(initialSize, capacityPtr);
168    Value * addr = iBuilder->CreateAnonymousMMap(initialSize);
169    Value * const addrPtr = iBuilder->CreateGEP(instance, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
170    addr = iBuilder->CreatePointerCast(addr, addrPtr->getType()->getPointerElementType());
171    iBuilder->CreateStore(addr, addrPtr);
172    mStreamSetBufferPtr = instance;
173}
174
175Value * ExtensibleBuffer::getStreamSetBlockPtr(Value * self, Value * blockIndex) const {
176    return iBuilder->CreateGEP(self, blockIndex);
177}
178
179void ExtensibleBuffer::reserveBytes(Value * const self, llvm::Value * const requiredSize) const {
180    Value * const capacityPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
181    Value * const currentSize = iBuilder->CreateLoad(capacityPtr);
182    BasicBlock * const entry = iBuilder->GetInsertBlock();
183    BasicBlock * const expand = BasicBlock::Create(iBuilder->getContext(), "expand", entry->getParent());
184    BasicBlock * const resume = BasicBlock::Create(iBuilder->getContext(), "resume", entry->getParent());
185    iBuilder->CreateLikelyCondBr(iBuilder->CreateICmpULT(requiredSize, currentSize), resume, expand);
186    iBuilder->SetInsertPoint(expand);
187    Value * const reservedSize = iBuilder->CreateShl(requiredSize, 1);
188    Value * const baseAddrPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
189    Value * const baseAddr = iBuilder->CreateLoad(baseAddrPtr);
190    Value * newAddr = iBuilder->CreateMRemap(baseAddr, currentSize, reservedSize);
191    newAddr = iBuilder->CreatePointerCast(newAddr, baseAddr->getType());
192    iBuilder->CreateStore(reservedSize, capacityPtr);
193    iBuilder->CreateStore(newAddr, baseAddrPtr);
194    iBuilder->CreateBr(resume);
195    iBuilder->SetInsertPoint(resume);
196}
197
198Value * ExtensibleBuffer::getBaseAddress(Value * const self) const {
199    return iBuilder->CreateLoad(iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)}));
200}
201
202void ExtensibleBuffer::releaseBuffer(Value * self) const {
203    Value * const sizePtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
204    Value * size = iBuilder->CreateLoad(sizePtr);
205    iBuilder->CreateMUnmap(getBaseAddress(self), size);
206}
207
208// Circular Buffer
209
210Value * CircularBuffer::getStreamSetBlockPtr(Value * const self, Value * const blockIndex) const {
211    return iBuilder->CreateGEP(self, modByBufferBlocks(blockIndex));
212}
213
214// CircularCopybackBuffer Buffer
215
216void CircularCopybackBuffer::allocateBuffer() {
217    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType(), iBuilder->getSize(mBufferBlocks + mOverflowBlocks));
218}
219
220void CircularCopybackBuffer::createCopyBack(Value * self, Value * overFlowItems) const {
221    Type * size_ty = iBuilder->getSizeTy();
222    Type * i8ptr = iBuilder->getInt8PtrTy();
223    Constant * blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
224    Function * f = iBuilder->GetInsertBlock()->getParent();
225    BasicBlock * wholeBlockCopy = BasicBlock::Create(iBuilder->getContext(), "wholeBlockCopy", f, 0);
226    BasicBlock * partialBlockCopy = BasicBlock::Create(iBuilder->getContext(), "partialBlockCopy", f, 0);
227    BasicBlock * copyBackDone = BasicBlock::Create(iBuilder->getContext(), "copyBackDone", f, 0);
228    unsigned numStreams = getType()->getArrayNumElements();
229    auto elemTy = getType()->getArrayElementType();
230    unsigned fieldWidth = isa<ArrayType>(elemTy) ? elemTy->getArrayNumElements() : 1;
231    Value * overFlowAreaPtr = iBuilder->CreateGEP(self, iBuilder->getSize(mBufferBlocks));
232    Value * overFlowBlocks = iBuilder->CreateUDiv(overFlowItems, blockSize);
233    Value * partialItems = iBuilder->CreateURem(overFlowItems, blockSize);
234    Value * partialBlockTargetPtr = iBuilder->CreateGEP(self, overFlowBlocks);
235    Value * partialBlockSourcePtr = iBuilder->CreateGEP(overFlowAreaPtr, overFlowBlocks);
236    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(overFlowBlocks, iBuilder->getSize(0)), wholeBlockCopy, partialBlockCopy);
237    iBuilder->SetInsertPoint(wholeBlockCopy);
238    unsigned alignment = iBuilder->getBitBlockWidth() / 8;
239    Value * copyLength = iBuilder->CreateSub(iBuilder->CreatePtrToInt(partialBlockTargetPtr, size_ty), iBuilder->CreatePtrToInt(self, size_ty));
240    iBuilder->CreateMemMove(iBuilder->CreateBitCast(self, i8ptr), iBuilder->CreateBitCast(overFlowAreaPtr, i8ptr), copyLength, alignment);
241    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(partialItems, iBuilder->getSize(0)), partialBlockCopy, copyBackDone);
242    iBuilder->SetInsertPoint(partialBlockCopy);
243    Value * copyBits = iBuilder->CreateMul(overFlowItems, iBuilder->getSize(fieldWidth));
244    Value * copyBytes = iBuilder->CreateLShr(iBuilder->CreateAdd(copyBits, iBuilder->getSize(7)), iBuilder->getSize(3));
245    for (unsigned strm = 0; strm < numStreams; strm++) {
246        Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
247        Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
248        iBuilder->CreateMemMove(iBuilder->CreateBitCast(strmTargetPtr, i8ptr), iBuilder->CreateBitCast(strmSourcePtr, i8ptr), copyBytes, alignment);
249    }
250    iBuilder->CreateBr(copyBackDone);
251    iBuilder->SetInsertPoint(copyBackDone);
252}
253
254Value * CircularCopybackBuffer::getStreamSetBlockPtr(Value * self, Value * blockIndex) const {
255    return iBuilder->CreateGEP(self, modByBufferBlocks(blockIndex));
256}
257
258// SwizzledCopybackBuffer Buffer
259
260void SwizzledCopybackBuffer::allocateBuffer() {
261    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType(), iBuilder->getSize(mBufferBlocks + mOverflowBlocks));
262}
263
264void SwizzledCopybackBuffer::createCopyBack(Value * self, Value * overFlowItems) const {
265    Type * size_ty = iBuilder->getSizeTy();
266    Type * i8ptr = iBuilder->getInt8PtrTy();
267    Constant * blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
268    Function * f = iBuilder->GetInsertBlock()->getParent();
269    BasicBlock * wholeBlockCopy = BasicBlock::Create(iBuilder->getContext(), "wholeBlockCopy", f, 0);
270    BasicBlock * partialBlockCopy = BasicBlock::Create(iBuilder->getContext(), "partialBlockCopy", f, 0);
271    BasicBlock * copyBackDone = BasicBlock::Create(iBuilder->getContext(), "copyBackDone", f, 0);
272    unsigned numStreams = getType()->getArrayNumElements();
273    unsigned swizzleFactor = iBuilder->getBitBlockWidth()/mFieldWidth;
274    auto elemTy = getType()->getArrayElementType();
275    unsigned fieldWidth = isa<ArrayType>(elemTy) ? elemTy->getArrayNumElements() : 1;
276    Value * overFlowAreaPtr = iBuilder->CreateGEP(self, iBuilder->getSize(mBufferBlocks));
277    Value * overFlowBlocks = iBuilder->CreateUDiv(overFlowItems, blockSize);
278    Value * partialItems = iBuilder->CreateURem(overFlowItems, blockSize);
279    Value * partialBlockTargetPtr = iBuilder->CreateGEP(self, overFlowBlocks);
280    Value * partialBlockSourcePtr = iBuilder->CreateGEP(overFlowAreaPtr, overFlowBlocks);
281    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(overFlowBlocks, iBuilder->getSize(0)), wholeBlockCopy, partialBlockCopy);
282    iBuilder->SetInsertPoint(wholeBlockCopy);
283    unsigned alignment = iBuilder->getBitBlockWidth() / 8;
284    Value * copyLength = iBuilder->CreateSub(iBuilder->CreatePtrToInt(partialBlockTargetPtr, size_ty), iBuilder->CreatePtrToInt(self, size_ty));
285    iBuilder->CreateMemMove(iBuilder->CreateBitCast(self, i8ptr), iBuilder->CreateBitCast(overFlowAreaPtr, i8ptr), copyLength, alignment);
286    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(partialItems, iBuilder->getSize(0)), partialBlockCopy, copyBackDone);
287    iBuilder->SetInsertPoint(partialBlockCopy);
288    Value * copyBits = iBuilder->CreateMul(overFlowItems, iBuilder->getSize(fieldWidth * swizzleFactor));
289    Value * copyBytes = iBuilder->CreateLShr(iBuilder->CreateAdd(copyBits, iBuilder->getSize(7)), iBuilder->getSize(3));
290    for (unsigned strm = 0; strm < numStreams; strm += swizzleFactor) {
291        Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
292        Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
293        iBuilder->CreateMemMove(iBuilder->CreateBitCast(strmTargetPtr, i8ptr), iBuilder->CreateBitCast(strmSourcePtr, i8ptr), copyBytes, alignment);
294    }
295    iBuilder->CreateBr(copyBackDone);
296    iBuilder->SetInsertPoint(copyBackDone);
297}
298
299Value * SwizzledCopybackBuffer::getStreamSetBlockPtr(Value * self, Value * blockIndex) const {
300    return iBuilder->CreateGEP(self, modByBufferBlocks(blockIndex));
301}
302
303SwizzledCopybackBuffer::SwizzledCopybackBuffer(IDISA::IDISA_Builder * b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned fieldwidth, unsigned AddressSpace)
304: StreamSetBuffer(BufferKind::SwizzledCopybackBuffer, b, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks), mFieldWidth(fieldwidth) {
305    mUniqueID = "SW" + std::to_string(fieldwidth) + ":" + std::to_string(bufferBlocks);
306    if (mOverflowBlocks != 1) mUniqueID += "_" + std::to_string(mOverflowBlocks);
307    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
308
309}
310
311// Expandable Buffer
312
313void ExpandableBuffer::allocateBuffer() {
314    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType());
315    Value * const capacityPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
316    iBuilder->CreateStore(iBuilder->getSize(mInitialCapacity), capacityPtr);
317    Type * const bufferType = getType()->getStructElementType(1)->getPointerElementType();
318    Constant * const bufferWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(bufferType), iBuilder->getSizeTy(), false);
319    Constant * const size = ConstantExpr::getMul(iBuilder->getSize(mBufferBlocks * mInitialCapacity), bufferWidth);
320    Value * const ptr = iBuilder->CreateAlignedMalloc(size, iBuilder->getCacheAlignment());
321    iBuilder->CreateMemZero(ptr, size, bufferType->getPrimitiveSizeInBits() / 8);
322    Value * const streamSetPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
323    iBuilder->CreateStore(iBuilder->CreatePointerCast(ptr, bufferType->getPointerTo()), streamSetPtr);
324}
325
326std::pair<Value *, Value *> ExpandableBuffer::getInternalStreamBuffer(Value * self, Value * streamIndex, Value * blockIndex, const bool readOnly) const {
327
328    // ENTRY
329    Value * const capacityPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
330    Value * const capacity = iBuilder->CreateLoad(capacityPtr);
331    Value * const streamSetPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
332    Value * const streamSet = iBuilder->CreateLoad(streamSetPtr);
333    blockIndex = modByBufferBlocks(blockIndex);
334
335    assert (streamIndex->getType() == capacity->getType());
336    Value * const cond = iBuilder->CreateICmpULT(streamIndex, capacity);
337
338    // Are we guaranteed that we can access this stream?
339    if (readOnly || isCapacityGuaranteed(streamIndex, mInitialCapacity)) {
340        iBuilder->CreateAssert(cond, "ExpandableBuffer: out-of-bounds stream access");
341        Value * offset = iBuilder->CreateAdd(iBuilder->CreateMul(blockIndex, capacity), streamIndex);
342        return {streamSet, offset};
343    }
344
345    BasicBlock * const entry = iBuilder->GetInsertBlock();
346    BasicBlock * const expand = BasicBlock::Create(iBuilder->getContext(), "expand", entry->getParent());
347    BasicBlock * const resume = BasicBlock::Create(iBuilder->getContext(), "resume", entry->getParent());
348
349    iBuilder->CreateLikelyCondBr(cond, resume, expand);
350
351    // EXPAND
352    iBuilder->SetInsertPoint(expand);
353
354    Type * elementType = getType()->getStructElementType(1)->getPointerElementType();
355    Constant * const vectorWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(elementType), capacity->getType(), false);
356
357    Value * newCapacity = iBuilder->CreateAdd(streamIndex, iBuilder->getSize(1));
358    newCapacity = iBuilder->CreateCeilLog2(newCapacity);
359    newCapacity = iBuilder->CreateShl(iBuilder->getSize(1), newCapacity, "newCapacity");
360
361    std::string tmp;
362    raw_string_ostream out(tmp);
363    out << "__expand";
364    elementType->print(out);
365    std::string name = out.str();
366
367    Module * const m = iBuilder->getModule();
368    Function * expandFunction = m->getFunction(name);
369
370    if (expandFunction == nullptr) {
371
372        const auto ip = iBuilder->saveIP();
373
374        FunctionType * fty = FunctionType::get(elementType->getPointerTo(), {elementType->getPointerTo(), iBuilder->getSizeTy(), iBuilder->getSizeTy()}, false);
375        expandFunction = Function::Create(fty, GlobalValue::PrivateLinkage, name, m);
376
377        auto args = expandFunction->arg_begin();
378        Value * streamSet = &*args++;
379        Value * capacity = &*args++;
380        Value * newCapacity = &*args;
381
382        BasicBlock * entry = BasicBlock::Create(iBuilder->getContext(), "entry", expandFunction);
383        iBuilder->SetInsertPoint(entry);
384
385        Value * size = iBuilder->CreateMul(newCapacity, iBuilder->getSize(mBufferBlocks));
386        Value * newStreamSet = iBuilder->CreatePointerCast(iBuilder->CreateAlignedMalloc(iBuilder->CreateMul(size, vectorWidth), iBuilder->getCacheAlignment()), elementType->getPointerTo());
387        Value * const diffCapacity = iBuilder->CreateMul(iBuilder->CreateSub(newCapacity, capacity), vectorWidth);
388
389        const auto alignment = elementType->getPrimitiveSizeInBits() / 8;
390        for (unsigned i = 0; i < mBufferBlocks; ++i) {
391            ConstantInt * const offset = iBuilder->getSize(i);
392            Value * srcOffset = iBuilder->CreateMul(capacity, offset);
393            Value * srcPtr = iBuilder->CreateGEP(streamSet, srcOffset);
394            Value * destOffset = iBuilder->CreateMul(newCapacity, offset);
395            Value * destPtr = iBuilder->CreateGEP(newStreamSet, destOffset);
396            iBuilder->CreateMemCpy(destPtr, srcPtr, iBuilder->CreateMul(capacity, vectorWidth), alignment);
397            Value * destZeroOffset = iBuilder->CreateAdd(destOffset, capacity);
398            Value * destZeroPtr = iBuilder->CreateGEP(newStreamSet, destZeroOffset);
399            iBuilder->CreateMemZero(destZeroPtr, diffCapacity, alignment);
400        }
401
402        iBuilder->CreateAlignedFree(streamSet);
403
404        iBuilder->CreateRet(newStreamSet);
405
406        iBuilder->restoreIP(ip);
407    }
408
409    Value * newStreamSet = iBuilder->CreateCall(expandFunction, {streamSet, capacity, newCapacity});
410    iBuilder->CreateStore(newStreamSet, streamSetPtr);
411    iBuilder->CreateStore(newCapacity, capacityPtr);
412
413    iBuilder->CreateBr(resume);
414
415    // RESUME
416    iBuilder->SetInsertPoint(resume);
417
418    PHINode * phiStreamSet = iBuilder->CreatePHI(streamSet->getType(), 2);
419    phiStreamSet->addIncoming(streamSet, entry);
420    phiStreamSet->addIncoming(newStreamSet, expand);
421
422    PHINode * phiCapacity = iBuilder->CreatePHI(capacity->getType(), 2);
423    phiCapacity->addIncoming(capacity, entry);
424    phiCapacity->addIncoming(newCapacity, expand);
425
426    Value * offset = iBuilder->CreateAdd(iBuilder->CreateMul(blockIndex, phiCapacity), streamIndex);
427
428    return {phiStreamSet, offset};
429}
430
431Value * ExpandableBuffer::getStreamBlockPtr(Value * self, Value * streamIndex, Value * blockIndex, const bool readOnly) const {
432    Value * ptr, * offset;
433    std::tie(ptr, offset) = getInternalStreamBuffer(self, streamIndex, blockIndex, readOnly);
434    return iBuilder->CreateGEP(ptr, offset);
435}
436
437Value * ExpandableBuffer::getStreamPackPtr(Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool readOnly) const {
438    Value * ptr, * offset;
439    std::tie(ptr, offset) = getInternalStreamBuffer(self, streamIndex, blockIndex, readOnly);
440    return iBuilder->CreateGEP(ptr, {offset, packIndex});
441}
442
443Value * ExpandableBuffer::getStreamSetCount(Value * self) const {
444    return iBuilder->CreateLoad(iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)}));
445}
446
447Value * ExpandableBuffer::getBaseAddress(Value * self) const {
448    return iBuilder->CreateLoad(iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)}));
449}
450
451void ExpandableBuffer::releaseBuffer(Value * self) const {
452    iBuilder->CreateAlignedFree(getBaseAddress(self));
453}
454
455Value * ExpandableBuffer::getStreamSetBlockPtr(Value *, Value *) const {
456    report_fatal_error("Expandable buffers: getStreamSetBlockPtr is not supported.");
457}
458
459Value * ExpandableBuffer::getLinearlyAccessibleItems(Value * self, Value *) const {
460    report_fatal_error("Expandable buffers: getLinearlyAccessibleItems is not supported.");
461}
462
463// Constructors
464SingleBlockBuffer::SingleBlockBuffer(IDISA::IDISA_Builder * b, Type * type)
465: StreamSetBuffer(BufferKind::BlockBuffer, b, type, resolveStreamSetType(b, type), 1, 0) {
466    mUniqueID = "S";
467
468}
469
470ExternalFileBuffer::ExternalFileBuffer(IDISA::IDISA_Builder * b, Type * type, unsigned AddressSpace)
471: StreamSetBuffer(BufferKind::ExternalFileBuffer, b, type, resolveStreamSetType(b, type), 0, AddressSpace) {
472    mUniqueID = "E";
473    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
474}
475
476ExtensibleBuffer::ExtensibleBuffer(IDISA::IDISA_Builder * b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
477: StreamSetBuffer(BufferKind::ExtensibleBuffer, b, type, StructType::get(b->getSizeTy(), resolveStreamSetType(b, type)->getPointerTo(), nullptr), bufferBlocks, AddressSpace) {
478    mUniqueID = "XT" + std::to_string(bufferBlocks);
479    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
480}
481
482CircularBuffer::CircularBuffer(IDISA::IDISA_Builder * b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
483: StreamSetBuffer(BufferKind::CircularBuffer, b, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
484    mUniqueID = "C" + std::to_string(bufferBlocks);
485    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
486
487}
488
489CircularCopybackBuffer::CircularCopybackBuffer(IDISA::IDISA_Builder * b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned AddressSpace)
490: StreamSetBuffer(BufferKind::CircularCopybackBuffer, b, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks) {
491    mUniqueID = "CC" + std::to_string(bufferBlocks);
492    if (mOverflowBlocks != 1) mUniqueID += "_" + std::to_string(mOverflowBlocks);
493    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
494}
495
496ExpandableBuffer::ExpandableBuffer(IDISA::IDISA_Builder * b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
497: StreamSetBuffer(BufferKind::ExpandableBuffer, b, type, resolveExpandableStreamSetType(b, type), bufferBlocks, AddressSpace)
498, mInitialCapacity(type->getArrayNumElements()) {
499    mUniqueID = "XP" + std::to_string(bufferBlocks);
500    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
501}
502
503inline StreamSetBuffer::StreamSetBuffer(BufferKind k, IDISA::IDISA_Builder * b, Type * baseType, Type * resolvedType, unsigned blocks, unsigned AddressSpace)
504: mBufferKind(k)
505, iBuilder(b)
506, mType(resolvedType)
507, mBufferBlocks(blocks)
508, mAddressSpace(AddressSpace)
509, mStreamSetBufferPtr(nullptr)
510, mBaseType(baseType) {
511
512}
513
514StreamSetBuffer::~StreamSetBuffer() { }
515
516// Helper routines
517ArrayType * resolveStreamSetType(IDISA_Builder * const b, Type * type) {
518    unsigned numElements = 1;
519    if (LLVM_LIKELY(type->isArrayTy())) {
520        numElements = type->getArrayNumElements();
521        type = type->getArrayElementType();
522    }
523    if (LLVM_LIKELY(type->isVectorTy() && type->getVectorNumElements() == 0)) {
524        type = type->getVectorElementType();
525        if (LLVM_LIKELY(type->isIntegerTy())) {
526            const auto fieldWidth = cast<IntegerType>(type)->getBitWidth();
527            type = b->getBitBlockType();
528            if (fieldWidth != 1) {
529                type = ArrayType::get(type, fieldWidth);
530            }
531            return ArrayType::get(type, numElements);
532        }
533    }
534    std::string tmp;
535    raw_string_ostream out(tmp);
536    type->print(out);
537    out << " is an unvalid stream set buffer type.";
538    report_fatal_error(out.str());
539}
540
541StructType * resolveExpandableStreamSetType(IDISA_Builder * const b, Type * type) {
542    if (LLVM_LIKELY(type->isArrayTy())) {
543        type = type->getArrayElementType();
544    }
545    if (LLVM_LIKELY(type->isVectorTy() && type->getVectorNumElements() == 0)) {
546        type = type->getVectorElementType();
547        if (LLVM_LIKELY(type->isIntegerTy())) {
548            const auto fieldWidth = cast<IntegerType>(type)->getBitWidth();
549            type = b->getBitBlockType();
550            if (fieldWidth != 1) {
551                type = ArrayType::get(type, fieldWidth);
552            }
553            return StructType::get(b->getSizeTy(), type->getPointerTo(), nullptr);
554        }
555    }
556    std::string tmp;
557    raw_string_ostream out(tmp);
558    type->print(out);
559    out << " is an unvalid stream set buffer type.";
560    report_fatal_error(out.str());
561}
Note: See TracBrowser for help on using the repository browser.