source: icGREP/icgrep-devel/icgrep/kernels/streamset.cpp @ 5191

Last change on this file since 5191 was 5191, checked in by cameron, 3 years ago

mmap and posix_memalign buffering options for u8u16

File size: 12.8 KB
RevLine 
[5044]1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5
6   
7#include <kernels/streamset.h>
8#include <vector>
9#include <IDISA/idisa_builder.h>
10#include <llvm/IR/Type.h>
[5191]11
[5100]12using namespace parabix;
[5044]13
[5135]14enum SS_struct_index {iProducer_pos = 0, iConsumer_pos = 1, iEnd_of_input = 2, iBuffer_ptr = 3};
15
[5185]16llvm::Value * parabix::getProducerPosPtr(IDISA::IDISA_Builder * b, Value * bufferStructPtr) {
17    return b->CreateGEP(bufferStructPtr, {b->getInt32(0), b->getInt32(iProducer_pos)});
18}
19
20llvm::Value * parabix::getConsumerPosPtr(IDISA::IDISA_Builder * b, Value * bufferStructPtr) {
21    return b->CreateGEP(bufferStructPtr, {b->getInt32(0), b->getInt32(iConsumer_pos)});
22}
23
24llvm::Value * parabix::hasEndOfInputPtr(IDISA::IDISA_Builder * b, Value * bufferStructPtr) {
25    return b->CreateGEP(bufferStructPtr, {b->getInt32(0), b->getInt32(iEnd_of_input)});
26}
27
28llvm::Value * parabix::getStreamSetBufferPtr(IDISA::IDISA_Builder * b, Value * bufferStructPtr) {
29    return b->CreateLoad(b->CreateGEP(bufferStructPtr, {b->getInt32(0), b->getInt32(iBuffer_ptr)}));
30}
31
[5044]32llvm::Type * StreamSetType::getStreamSetBlockType(IDISA::IDISA_Builder * iBuilder) {
33    llvm::Type * streamType = mFieldWidth == 1 ? iBuilder->getBitBlockType() : ArrayType::get(iBuilder->getBitBlockType(), mFieldWidth);
34    return ArrayType::get(streamType, mStreamCount);
35}
[5076]36
[5123]37llvm::PointerType * StreamSetBuffer::getStreamBufferPointerType() {
[5184]38    return PointerType::get(mStreamSetType.getStreamSetBlockType(iBuilder), mAddrSpace);
[5044]39}
40
[5135]41llvm::PointerType * StreamSetBuffer::getStreamSetStructPointerType() {
42    return PointerType::get(mStreamSetStructType, 0);
43}
44
[5185]45llvm::Value * StreamSetBuffer::getProducerPosPtr(Value * bufferStructPtr) {
46    return iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iProducer_pos)});
[5135]47}
48
[5185]49void StreamSetBuffer::setProducerPos(Value * bufferStructPtr, llvm::Value * pos){
50    iBuilder->CreateStore(pos, iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iProducer_pos)}));
[5135]51}
52
[5185]53llvm::Value * StreamSetBuffer::getConsumerPosPtr(Value * bufferStructPtr) {
54    return iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iConsumer_pos)});
[5135]55}
56
[5185]57void StreamSetBuffer::setConsumerPos(Value * bufferStructPtr, Value * pos){
58    iBuilder->CreateStore(pos, iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iConsumer_pos)}));
[5135]59}
60
[5185]61llvm::Value * StreamSetBuffer::hasEndOfInputPtr(Value * bufferStructPtr) {
62    return iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iEnd_of_input)});
[5135]63}
64
[5185]65void StreamSetBuffer::setEndOfInput(Value * bufferStructPtr){
[5188]66    iBuilder->CreateStore(ConstantInt::get(iBuilder->getInt1Ty(), 1), iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iEnd_of_input)}));
[5135]67}
68
[5185]69
[5135]70llvm::Value * StreamSetBuffer::getStreamSetStructPtr(){
71    return mStreamSetStructPtr;
72}
[5100]73
[5185]74llvm::Value * StreamSetBuffer::allocateBuffer() {
[5139]75    Type * const size_ty = iBuilder->getSizeTy();
[5188]76    Type * const int1ty = iBuilder->getInt1Ty();
[5185]77    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(mStreamSetType.getStreamSetBlockType(iBuilder), ConstantInt::get(iBuilder->getSizeTy(), mBufferBlocks));
78    mStreamSetStructPtr = iBuilder->CreateCacheAlignedAlloca(mStreamSetStructType);
79    //iBuilder->CallPrintInt("mStreamSetBufferPtr", iBuilder->CreatePtrToInt(mStreamSetBufferPtr, iBuilder->getInt64Ty()));
80    //iBuilder->CallPrintInt("mStreamSetStructPtr", iBuilder->CreatePtrToInt(mStreamSetStructPtr, iBuilder->getInt64Ty()));
[5139]81    iBuilder->CreateStore(ConstantInt::get(size_ty, 0), iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iProducer_pos)}));
82    iBuilder->CreateStore(ConstantInt::get(size_ty, 0), iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iConsumer_pos)}));
[5188]83    iBuilder->CreateStore(ConstantInt::get(int1ty, 0), iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iEnd_of_input)}));
[5142]84    iBuilder->CreateStore(mStreamSetBufferPtr, iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)}));
[5185]85   
[5044]86    return mStreamSetBufferPtr;
87}
88
[5191]89llvm::Value * StreamSetBuffer::getStreamSetBlockPointer(llvm::Value * bufferStructPtr, llvm::Value * blockNo) {
90    Value * handle = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)});
91    return iBuilder->CreateGEP(iBuilder->CreateLoad(handle), {blockNo});
92}
93
94
95
96
[5185]97// Single Block Buffer
[5100]98// For a single block buffer, the block pointer is always the buffer base pointer.
[5185]99llvm::Value * SingleBlockBuffer::getStreamSetBlockPointer(llvm::Value * bufferStructPtr, llvm::Value * blockNo) {
100    Value * handle = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)});
[5142]101    return iBuilder->CreateLoad(handle);
[5044]102}
103
[5100]104
[5191]105// External File Buffer
[5100]106
[5142]107void ExternalFileBuffer::setStreamSetBuffer(llvm::Value * ptr, Value * fileSize) {
[5190]108   
[5139]109    Type * const size_ty = iBuilder->getSizeTy();
[5188]110    Type * const int1ty = iBuilder->getInt1Ty();
[5190]111   
[5135]112    PointerType * t = getStreamBufferPointerType();   
[5122]113    mStreamSetBufferPtr = iBuilder->CreatePointerBitCastOrAddrSpaceCast(ptr, t);
[5190]114   
[5185]115    mStreamSetStructPtr = iBuilder->CreateCacheAlignedAlloca(mStreamSetStructType);
[5142]116    iBuilder->CreateStore(fileSize, iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iProducer_pos)}));
[5139]117    iBuilder->CreateStore(ConstantInt::get(size_ty, 0), iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iConsumer_pos)}));
[5188]118    iBuilder->CreateStore(ConstantInt::get(int1ty, 1), iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iEnd_of_input)}));
[5135]119    iBuilder->CreateStore(mStreamSetBufferPtr, iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)}));
[5122]120}
121
[5190]122void ExternalFileBuffer::setEmptyBuffer(llvm::Value * ptr) {
123   
124    Type * const size_ty = iBuilder->getSizeTy();
125    Type * const int1ty = iBuilder->getInt1Ty();
126   
127    PointerType * t = getStreamBufferPointerType();   
128    mStreamSetBufferPtr = iBuilder->CreatePointerBitCastOrAddrSpaceCast(ptr, t);
129   
130    mStreamSetStructPtr = iBuilder->CreateCacheAlignedAlloca(mStreamSetStructType);
131    iBuilder->CreateStore(ConstantInt::get(size_ty, 0), iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iProducer_pos)}));
132    iBuilder->CreateStore(ConstantInt::get(size_ty, 0), iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iConsumer_pos)}));
133    iBuilder->CreateStore(ConstantInt::get(int1ty,0), iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iEnd_of_input)}));
134    iBuilder->CreateStore(mStreamSetBufferPtr, iBuilder->CreateGEP(mStreamSetStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)}));
135}
136
[5142]137llvm::Value * ExternalFileBuffer::allocateBuffer() {
[5100]138    throw std::runtime_error("External buffers cannot be allocated.");
139}
140
[5185]141llvm::Value * ExternalFileBuffer::getStreamSetBlockPointer(llvm::Value * bufferStructPtr, llvm::Value * blockNo) {
142    Value * handle = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)});
[5135]143    return iBuilder->CreateGEP(iBuilder->CreateLoad(handle), {blockNo});
[5100]144}
145
146// Circular Stack Allocated Buffer
147
[5185]148llvm::Value * CircularBuffer::getStreamSetBlockPointer(llvm::Value * bufferStructPtr, llvm::Value * blockNo) {
149    Value * handle = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)});
150    // Circular access is based on blockNo mod mBufferBlocks.  For power of 2 buffer sizes (required), we
151    // use bitwise masking to efficiently compute the mod function  (blockNo & (mBufferBlocks - 1)
152    Value * bufPtr = iBuilder->CreateLoad(handle);
153    //iBuilder->CallPrintInt("CircularBuffer bufPtr", iBuilder->CreatePtrToInt(bufPtr, iBuilder->getSizeTy()));
154    return iBuilder->CreateGEP(bufPtr, {iBuilder->CreateAnd(blockNo, ConstantInt::get(iBuilder->getSizeTy(), mBufferBlocks-1))});
[5100]155}
156
[5191]157llvm::Value * LinearCopybackBuffer::getStreamSetBlockPointer(llvm::Value * bufferStructPtr, llvm::Value * blockNo) {
[5185]158    Constant * blockWidth = ConstantInt::get(iBuilder->getSizeTy(), iBuilder->getStride());
159    Value * consumerPos_ptr = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iConsumer_pos)});
160    Value * consumerPos = iBuilder->CreateLoad(consumerPos_ptr);
161    Value * consumerBlock = iBuilder->CreateUDiv(consumerPos, blockWidth);
162    Value * handle = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)});
163    Value * bufPtr = iBuilder->CreateLoad(handle);
164    //iBuilder->CallPrintInt("LinearBuffer bufPtr", iBuilder->CreatePtrToInt(bufPtr, iBuilder->getSizeTy()));
165    return iBuilder->CreateGEP(bufPtr, {iBuilder->CreateSub(blockNo, consumerBlock)});
[5100]166}
167
[5191]168void LinearCopybackBuffer::setConsumerPos(Value * bufferStructPtr, Value * new_consumer_pos) {
[5185]169    Type * const i1 = iBuilder->getInt1Ty();
170    Type * const i8 = iBuilder->getInt8Ty();
171    Type * const i32 = iBuilder->getInt32Ty();
172    Type * const i8_ptr = PointerType::get(i8, mAddrSpace);
173    Module * M = iBuilder->getModule();
[5190]174    Function * memmoveFunc = cast<Function>(M->getOrInsertFunction("llvm.memmove.p0i8.p0i8.i" + std::to_string(sizeof(size_t) * 8), 
[5185]175                                                                  iBuilder->getVoidTy(), i8_ptr, i8_ptr, iBuilder->getSizeTy(), i32, i1, nullptr));
176    Function * current = iBuilder->GetInsertBlock()->getParent();
177    BasicBlock * copyBackBody = BasicBlock::Create(M->getContext(), "copy_back", current, 0);
178    BasicBlock * setConsumerPosExit = BasicBlock::Create(M->getContext(), "setConsumerPos_done", current, 0);
179    Constant * blockWidth = ConstantInt::get(iBuilder->getSizeTy(), iBuilder->getStride());
180    Constant * one = ConstantInt::get(iBuilder->getSizeTy(), 1);
181    Value * consumerPos_ptr = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iConsumer_pos)});
182    Value * consumerPos = iBuilder->CreateLoad(consumerPos_ptr);
183    Value * consumerBlock = iBuilder->CreateUDiv(consumerPos, blockWidth);
184    // Ensure that the new consumer position is no less than the current position.
185    new_consumer_pos = iBuilder->CreateSelect(iBuilder->CreateICmpULT(new_consumer_pos, consumerPos), consumerPos, new_consumer_pos);
186    Value * producerPos = iBuilder->CreateLoad(iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iProducer_pos)}));
187    // Ensure that the new consumer position is no greater than the current producer position.
188    Value * new_pos_lt_producer_pos = iBuilder->CreateICmpULT(new_consumer_pos, producerPos);
189    new_consumer_pos = iBuilder->CreateSelect(new_pos_lt_producer_pos, new_consumer_pos, producerPos);
190    // Now, the new_consumer_pos is at most = to the producer_pos; if =, we're done.
191    iBuilder->CreateCondBr(new_pos_lt_producer_pos, copyBackBody, setConsumerPosExit);
192    iBuilder->SetInsertPoint(copyBackBody);
193   
194    Value * new_consumer_block = iBuilder->CreateUDiv(new_consumer_pos, blockWidth);
195   
196    Value * lastProducerBlock = iBuilder->CreateUDiv(iBuilder->CreateSub(producerPos, one), blockWidth);
197    //iBuilder->CallPrintInt("new_consumer_block", new_consumer_block);
198    //iBuilder->CallPrintInt("lastProducerBlock", lastProducerBlock);
[5100]199
[5185]200    Value * copyBlocks = iBuilder->CreateAdd(iBuilder->CreateSub(lastProducerBlock, new_consumer_block), one);
201    Constant * blockBytes = ConstantInt::get(iBuilder->getSizeTy(), mStreamSetType.StreamCount() * mStreamSetType.StreamFieldWidth() * iBuilder->getStride()/8);
202    Value * copyLength = iBuilder->CreateMul(copyBlocks, blockBytes);
[5190]203    //iBuilder->CallPrintInt("memmove copyLength", copyLength);
[5185]204    // Must copy back one full block for each of the streams in the stream set.
205    Value * handle = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)});
206    Value * bufferPtr = iBuilder->CreateLoad(handle);
[5190]207    //iBuilder->CallPrintInt("memmove bufferPtr", iBuilder->CreatePtrToInt(bufferPtr, iBuilder->getSizeTy()));
[5185]208
209    Value * copyFrom = iBuilder->CreateGEP(bufferPtr, {iBuilder->CreateSub(new_consumer_block, consumerBlock)});
[5190]210    //iBuilder->CallPrintInt("memmove copyFrom", iBuilder->CreatePtrToInt(copyFrom, iBuilder->getSizeTy()));
[5185]211    Value * alignment = ConstantInt::get(iBuilder->getInt32Ty(), iBuilder->getBitBlockWidth()/8);
212   
[5190]213    iBuilder->CreateCall(memmoveFunc, {iBuilder->CreateBitCast(bufferPtr, i8_ptr), iBuilder->CreateBitCast(copyFrom, i8_ptr), copyLength, alignment, ConstantInt::getNullValue(i1)});
[5185]214    iBuilder->CreateBr(setConsumerPosExit);
215    // Copy back done, store the new consumer position.
216    iBuilder->SetInsertPoint(setConsumerPosExit);
217    iBuilder->CreateStore(new_consumer_pos, consumerPos_ptr);
218}   
Note: See TracBrowser for help on using the repository browser.