Ignore:
Timestamp:
Oct 25, 2017, 4:57:58 PM (20 months ago)
Author:
nmedfort
Message:

First stage of MultiBlockKernel? and pipeline restructuring

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/kernels/streamset.cpp

    r5650 r5706  
    2828
    2929void StreamSetBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
     30    assert (mBufferBlocks > 0);
    3031    if (LLVM_LIKELY(mStreamSetBufferPtr == nullptr)) {
    3132        Type * const ty = getType();
     
    4950}
    5051
    51 Value * StreamSetBuffer::getStreamBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, const bool /* readOnly */) const {
     52Value * StreamSetBuffer::getStreamBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * addr, Value * streamIndex, const bool /* readOnly */) const {
    5253    if (codegen::EnableAsserts) {
    5354        Value * const count = getStreamSetCount(iBuilder, self);
     
    5657        iBuilder->CreateAssert(cond, "StreamSetBuffer: out-of-bounds stream access");
    5758    }
    58     return iBuilder->CreateGEP(getStreamSetBlockPtr(iBuilder, self, blockIndex), {iBuilder->getInt32(0), streamIndex});
    59 }
    60 
    61 Value * StreamSetBuffer::getStreamPackPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool /* readOnly */) const {
     59    return iBuilder->CreateGEP(addr, {iBuilder->getInt32(0), streamIndex});
     60}
     61
     62Value * StreamSetBuffer::getStreamPackPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * addr, Value * streamIndex, Value * packIndex, const bool /* readOnly */) const {
    6263    if (codegen::EnableAsserts) {
    6364        Value * const count = getStreamSetCount(iBuilder, self);
     
    6667        iBuilder->CreateAssert(cond, "StreamSetBuffer: out-of-bounds stream access");
    6768    }
    68     return iBuilder->CreateGEP(getStreamSetBlockPtr(iBuilder, self, blockIndex), {iBuilder->getInt32(0), streamIndex, packIndex});
    69 }
    70 
    71 void StreamSetBuffer::setBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * /* self */, Value * /* addr */) const {
     69    return iBuilder->CreateGEP(addr, {iBuilder->getInt32(0), streamIndex, packIndex});
     70}
     71
     72void StreamSetBuffer::setBaseAddress(IDISA::IDISA_Builder * const /* iBuilder */, Value * /* self */, Value * /* addr */) const {
    7273    report_fatal_error("setBaseAddress is not supported by this buffer type");
    7374}
    7475
    75 Value * StreamSetBuffer::getBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * /* self */) const {
     76Value * StreamSetBuffer::getBufferedSize(IDISA::IDISA_Builder * const /* iBuilder */, Value * /* self */) const {
    7677    report_fatal_error("getBufferedSize is not supported by this buffer type");
    7778}
    7879
    79 void StreamSetBuffer::setBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * /* self */, llvm::Value * /* size */) const {
     80void StreamSetBuffer::setBufferedSize(IDISA::IDISA_Builder * const /* iBuilder */, Value * /* self */, llvm::Value * /* size */) const {
    8081    report_fatal_error("setBufferedSize is not supported by this buffer type");
    8182}
    8283
    83 Value * StreamSetBuffer::getCapacity(IDISA::IDISA_Builder * const iBuilder, Value * /* self */) const {
    84     report_fatal_error("getCapacity is not supported by this buffer type");
    85 }
    86 
    87 void StreamSetBuffer::setCapacity(IDISA::IDISA_Builder * const iBuilder, Value * /* self */, llvm::Value * /* c */) const {
     84Value * StreamSetBuffer::getCapacity(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
     85    return getBufferedSize(iBuilder, self);
     86}
     87
     88void StreamSetBuffer::setCapacity(IDISA::IDISA_Builder * const /* iBuilder */, Value * /* self */, llvm::Value * /* c */) const {
    8889    report_fatal_error("setCapacity is not supported by this buffer type");
    8990}
     
    126127 * The type of the pointer is i8* for fields of 8 bits or less, otherwise iN* for N-bit fields.
    127128 */
    128 Value * StreamSetBuffer::getRawItemPointer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * absolutePosition) const {
    129     Value * ptr = iBuilder->CreateGEP(getBaseAddress(iBuilder, self), {iBuilder->getInt32(0), streamIndex});
     129Value * StreamSetBuffer::getRawItemPointer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * absolutePosition) const {
     130    Value * ptr = getBaseAddress(iBuilder, self);
    130131    Value * relativePosition = absolutePosition;
    131132    const auto bw = mBaseType->getArrayElementType()->getScalarSizeInBits();
     
    141142
    142143Value * StreamSetBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const b, Value * self, Value * fromPosition, Value * availItems, bool reverse) const {
    143     Constant * bufSize = b->getSize(mBufferBlocks * b->getStride());
     144    Constant * bufSize = ConstantInt::get(fromPosition->getType(), mBufferBlocks * b->getStride());
    144145    Value * itemsFromBase = b->CreateURem(fromPosition, bufSize);
    145146    if (reverse) {
    146147        Value * bufAvail = b->CreateSelect(b->CreateICmpEQ(itemsFromBase, b->getSize(0)), bufSize, itemsFromBase);
    147148        return b->CreateSelect(b->CreateICmpULT(bufAvail, availItems), bufAvail, availItems);
    148     }
    149     else {
     149    } else {
    150150        Value * linearSpace = b->CreateSub(bufSize, itemsFromBase, "linearSpace");
    151151        return b->CreateSelect(b->CreateICmpULT(availItems, linearSpace), availItems, linearSpace);
     
    154154
    155155Value * StreamSetBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition, bool reverse) const {
    156     Constant * bufSize = iBuilder->getSize(mBufferBlocks * iBuilder->getStride());
     156    Constant * bufSize = ConstantInt::get(fromPosition->getType(), mBufferBlocks * iBuilder->getStride());
    157157    Value * bufRem = iBuilder->CreateURem(fromPosition, bufSize);
    158158    if (reverse) {
    159159        return iBuilder->CreateSelect(iBuilder->CreateICmpEQ(bufRem, iBuilder->getSize(0)), bufSize, bufRem);
    160160    }
    161     else return iBuilder->CreateSub(bufSize, bufRem, "linearSpace");
     161    return iBuilder->CreateSub(bufSize, bufRem, "linearSpace");
    162162}
    163163
     
    179179}
    180180
     181Value * StreamSetBuffer::copy(IDISA::IDISA_Builder * const b, Value * self, Value * const target, Value * const source, Value * itemsToCopy, const unsigned alignment) const {
     182    Type * ty = getBaseType();
     183    if (LLVM_LIKELY(isa<ArrayType>(ty))) {
     184        ty = ty->getArrayElementType();
     185    }
     186    if (LLVM_LIKELY(isa<VectorType>(ty))) {
     187        ty = ty->getVectorElementType();
     188    }
     189    const auto itemWidth = ty->getScalarSizeInBits();
     190    assert (itemWidth > 0);
     191    Value * const m = b->CreateMul(getStreamSetCount(b, self), b->getSize(itemWidth / 8));
     192    Value * const bytesToCopy = b->CreateMul(itemsToCopy, m);
     193
     194    // TODO: lz4d s2p reads misaligned data into the source stream. The stream binding should indicate alignment.
     195    // alignment ? alignment : b->getBitBlockWidth() / 8
     196    b->CreateMemCpy(target, source, bytesToCopy, 1);
     197    return bytesToCopy;
     198}
     199
    181200void StreamSetBuffer::createBlockAlignedCopy(IDISA::IDISA_Builder * const iBuilder, Value * targetBlockPtr, Value * sourceBlockPtr, Value * itemsToCopy) const {
    182     Type * const int8PtrTy = iBuilder->getInt8PtrTy();
    183201    const unsigned alignment = iBuilder->getBitBlockWidth() / 8;
    184     Constant * const blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
     202    Constant * const blockSize = ConstantInt::get(itemsToCopy->getType(), iBuilder->getBitBlockWidth());
    185203    size_t numStreams = 1;
    186204    if (isa<ArrayType>(mBaseType)) {
     
    191209        Value * copyBits = iBuilder->CreateMul(itemsToCopy, iBuilder->getSize(fieldWidth));
    192210        Value * copyBytes = iBuilder->CreateLShr(iBuilder->CreateAdd(copyBits, iBuilder->getSize(7)), iBuilder->getSize(3));
    193         iBuilder->CreateMemMove(iBuilder->CreateBitCast(targetBlockPtr, int8PtrTy), iBuilder->CreateBitCast(sourceBlockPtr, int8PtrTy), copyBytes, alignment);
     211        iBuilder->CreateMemMove(targetBlockPtr, sourceBlockPtr, copyBytes, alignment);
    194212    } else {
    195213        Value * blocksToCopy = iBuilder->CreateUDiv(itemsToCopy, blockSize);
     
    198216        Value * partialBlockSourcePtr = iBuilder->CreateGEP(sourceBlockPtr, blocksToCopy);
    199217        Value * blockCopyBytes = iBuilder->CreateMul(blocksToCopy, iBuilder->getSize(iBuilder->getBitBlockWidth() * numStreams * fieldWidth/8));
    200         iBuilder->CreateMemMove(iBuilder->CreateBitCast(targetBlockPtr, int8PtrTy), iBuilder->CreateBitCast(sourceBlockPtr, int8PtrTy), blockCopyBytes, alignment);
     218        iBuilder->CreateMemMove(targetBlockPtr, sourceBlockPtr, blockCopyBytes, alignment);
    201219        Value * partialCopyBitsPerStream = iBuilder->CreateMul(partialItems, iBuilder->getSize(fieldWidth));
    202220        Value * partialCopyBytesPerStream = iBuilder->CreateLShr(iBuilder->CreateAdd(partialCopyBitsPerStream, iBuilder->getSize(7)), iBuilder->getSize(3));
    203         for (unsigned strm = 0; strm < numStreams; strm++) {
    204             Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
    205             Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
    206             strmTargetPtr = iBuilder->CreateBitCast(strmTargetPtr, int8PtrTy);
    207             strmSourcePtr = iBuilder->CreateBitCast(strmSourcePtr, int8PtrTy);
     221        for (unsigned i = 0; i < numStreams; i++) {
     222            Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(i)});
     223            Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(i)});
    208224            iBuilder->CreateMemMove(strmTargetPtr, strmSourcePtr, partialCopyBytesPerStream, alignment);
    209225        }
     
    211227}
    212228
    213 void StreamSetBuffer::genCopyBackLogic(IDISA::IDISA_Builder * const b, Value * handle, Value * priorProduced, Value * newProduced, const std::string Name) {
     229void StreamSetBuffer::genCopyBackLogic(IDISA::IDISA_Builder * const b, Value * handle, Value * priorProduced, Value * newProduced, const std::string Name) const {
    214230    report_fatal_error("Copy back not supported for this buffer type:" + Name);
    215231}
     
    244260void SourceBuffer::setBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * addr) const {
    245261    Value * const ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::BaseAddress))});
    246 
    247262    iBuilder->CreateStore(iBuilder->CreatePointerCast(addr, ptr->getType()->getPointerElementType()), ptr);
    248263}
     
    256271}
    257272
    258 Value * SourceBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
    259     return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), blockIndex);
     273Value * SourceBuffer::getBlockAddress(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
     274    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), blockIndex );
    260275}
    261276
    262277Value * SourceBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition, Value * availItems, bool reverse) const {
    263278    if (reverse) report_fatal_error("SourceBuffer cannot be accessed in reverse");
    264     Value * maxAvail = iBuilder->CreateSub(getCapacity(iBuilder, self), fromPosition);
     279    Value * maxAvail = iBuilder->CreateSub(getBufferedSize(iBuilder, self), fromPosition);
    265280    return iBuilder->CreateSelect(iBuilder->CreateICmpULT(availItems, maxAvail), availItems, maxAvail);
    266281}
     
    293308}
    294309
    295 Value * ExternalBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
     310Value * ExternalBuffer::getBlockAddress(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
    296311    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), blockIndex);
    297312}
    298313
    299 // All available items can be accessed.
    300 Value * ExternalBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const, Value *, Value *, Value * availItems, bool) const {
    301     return availItems;
     314Value * ExternalBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const, Value *, Value *, Value * availItems, const bool reverse) const {
     315    // All available items can be accessed.
     316    return reverse ? ConstantInt::getAllOnesValue(availItems->getType()) : availItems;
     317}
     318
     319Value * ExternalBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const, Value *, Value * fromPosition, const bool reverse) const {
     320    // Trust that the buffer is large enough to write any amount
     321    return reverse ? fromPosition : ConstantInt::getAllOnesValue(fromPosition->getType());
    302322}
    303323
    304324// Circular Buffer
    305 Value * CircularBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * const self, Value * const blockIndex) const {
     325Value * CircularBuffer::getBlockAddress(IDISA::IDISA_Builder * const iBuilder, Value * const self, Value * const blockIndex) const {
    306326    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), modByBufferBlocks(iBuilder, blockIndex));
    307327}
    308328
    309 Value * CircularBuffer::getRawItemPointer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * absolutePosition) const {
    310     Value * ptr = iBuilder->CreateGEP(getBaseAddress(iBuilder, self), {iBuilder->getInt32(0), streamIndex});
     329Value * CircularBuffer::getRawItemPointer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * absolutePosition) const {
     330    Value * ptr = getBaseAddress(iBuilder, self);
    311331    Value * relativePosition = iBuilder->CreateURem(absolutePosition, ConstantInt::get(absolutePosition->getType(), mBufferBlocks * iBuilder->getBitBlockWidth()));
    312332    const auto bw = mBaseType->getArrayElementType()->getScalarSizeInBits();
     
    335355}
    336356
    337 void CircularCopybackBuffer::genCopyBackLogic(IDISA::IDISA_Builder * const b, Value * handle, Value * priorProduced, Value * newProduced, const std::string Name) {
    338     Constant * bufSize = b->getSize(mBufferBlocks * b->getBitBlockWidth());
     357void CircularCopybackBuffer::genCopyBackLogic(IDISA::IDISA_Builder * const b, Value * handle, Value * priorProduced, Value * newProduced, const std::string Name) const {
     358    assert (priorProduced->getType() == newProduced->getType());
     359    Constant * bufSize = ConstantInt::get(priorProduced->getType(), mBufferBlocks * b->getBitBlockWidth());
    339360    Value * priorBufPos = b->CreateURem(priorProduced, bufSize);
    340361    Value * newBufPos = b->CreateURem(newProduced, bufSize);
    341     BasicBlock * copyBack = b->CreateBasicBlock(Name + "_copyBack");
    342     BasicBlock * done = b->CreateBasicBlock(Name + "_copyBackDone");
     362    BasicBlock * copyBack = b->CreateBasicBlock(Name + "_circularCopyBack");
     363    BasicBlock * done = b->CreateBasicBlock(Name + "_circularCopyBackDone");
    343364    Value * wraparound = b->CreateICmpUGT(priorBufPos, newBufPos);
    344365    b->CreateCondBr(wraparound, copyBack, done);
     366
    345367    b->SetInsertPoint(copyBack);
    346     Value * overFlowAreaPtr = b->CreateGEP(handle, b->getSize(mBufferBlocks));
     368    Value * overFlowAreaPtr = b->CreateGEP(handle, b->getInt32(mBufferBlocks));
    347369    createBlockAlignedCopy(b, handle, overFlowAreaPtr, newBufPos);
    348370    b->CreateBr(done);
     371
    349372    b->SetInsertPoint(done);
    350373}
     
    365388    IntegerType * const intAddrTy = iBuilder->getIntPtrTy(DL);
    366389
    367     Constant * blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
     390    Constant * blockSize = ConstantInt::get(itemsToCopy->getType(), iBuilder->getBitBlockWidth());
    368391    Function * f = iBuilder->GetInsertBlock()->getParent();
    369392    BasicBlock * wholeBlockCopy = BasicBlock::Create(iBuilder->getContext(), "wholeBlockCopy", f, 0);
     
    398421}
    399422
    400 Value * SwizzledCopybackBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
     423Value * SwizzledCopybackBuffer::getBlockAddress(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
    401424    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), modByBufferBlocks(iBuilder, blockIndex));
    402425}
     
    408431}
    409432
    410 void SwizzledCopybackBuffer::genCopyBackLogic(IDISA::IDISA_Builder * const b, Value * handle, Value * priorProduced, Value * newProduced, const std::string Name) {
    411     Constant * bufSize = b->getSize(mBufferBlocks * b->getBitBlockWidth());
     433void SwizzledCopybackBuffer::genCopyBackLogic(IDISA::IDISA_Builder * const b, Value * handle, Value * priorProduced, Value * newProduced, const std::string Name) const {
     434    assert (priorProduced->getType() == newProduced->getType());
     435    Constant * bufSize = ConstantInt::get(priorProduced->getType(), mBufferBlocks * b->getBitBlockWidth());
    412436    Value * priorBufPos = b->CreateURem(priorProduced, bufSize);
    413437    Value * newBufPos = b->CreateURem(newProduced, bufSize);
    414     BasicBlock * copyBack = b->CreateBasicBlock(Name + "_copyBack");
    415     BasicBlock * done = b->CreateBasicBlock(Name + "_copyBackDone");
     438    BasicBlock * copyBack = b->CreateBasicBlock(Name + "_swizzledCopyBack");
     439    BasicBlock * done = b->CreateBasicBlock(Name + "_swizzledCopyBackDone");
    416440    Value * wraparound = b->CreateICmpUGT(priorBufPos, newBufPos);
    417441    b->CreateCondBr(wraparound, copyBack, done);
     
    547571
    548572Value * ExpandableBuffer::getStreamBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, const bool readOnly) const {
    549     Value * ptr, * offset;
    550     std::tie(ptr, offset) = getInternalStreamBuffer(iBuilder, self, streamIndex, blockIndex, readOnly);
    551     return iBuilder->CreateGEP(ptr, offset);
     573    report_fatal_error("temporarily not supported");
     574//    Value * ptr, * offset;
     575//    std::tie(ptr, offset) = getInternalStreamBuffer(iBuilder, self, streamIndex, blockIndex, readOnly);
     576//    return iBuilder->CreateGEP(ptr, offset);
    552577}
    553578
    554579Value * ExpandableBuffer::getStreamPackPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool readOnly) const {
    555     Value * ptr, * offset;
    556     std::tie(ptr, offset) = getInternalStreamBuffer(iBuilder, self, streamIndex, blockIndex, readOnly);
    557     return iBuilder->CreateGEP(ptr, {offset, packIndex});
     580    report_fatal_error("temporarily not supported");
     581//    Value * ptr, * offset;
     582//    std::tie(ptr, offset) = getInternalStreamBuffer(iBuilder, self, streamIndex, blockIndex, readOnly);
     583//    return iBuilder->CreateGEP(ptr, {offset, packIndex});
    558584}
    559585
     
    573599}
    574600
    575 Value * ExpandableBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value *, Value *) const {
    576     report_fatal_error("Expandable buffers: getStreamSetBlockPtr is not supported.");
     601Value * ExpandableBuffer::getBlockAddress(IDISA::IDISA_Builder * const iBuilder, Value *, Value *) const {
     602    report_fatal_error("Expandable buffers: getBlockAddress is not supported.");
    577603}
    578604
     
    581607}
    582608
    583 SourceBuffer::SourceBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, unsigned MemoryAddressSpace, unsigned StructAddressSpace)
    584 : StreamSetBuffer(BufferKind::SourceBuffer, type, StructType::get(resolveStreamSetType(b, type)->getPointerTo(MemoryAddressSpace), b->getSizeTy(), b->getSizeTy(), nullptr), 0, StructAddressSpace) {
    585     mUniqueID = "B";
    586     if (MemoryAddressSpace != 0 || StructAddressSpace != 0) {
    587         mUniqueID += "@" + std::to_string(MemoryAddressSpace) + ":" + std::to_string(StructAddressSpace);
    588     }
    589 }
    590 
    591 ExternalBuffer::ExternalBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, llvm::Value * addr, unsigned AddressSpace)
    592 : StreamSetBuffer(BufferKind::ExternalBuffer, type, resolveStreamSetType(b, type), 0, AddressSpace) {
    593     mUniqueID = "E";
    594     if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
    595     mStreamSetBufferPtr = b->CreatePointerBitCastOrAddrSpaceCast(addr, getPointerType());
    596 }
    597 
    598 CircularBuffer::CircularBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
    599 : StreamSetBuffer(BufferKind::CircularBuffer, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
    600     mUniqueID = "C" + std::to_string(bufferBlocks);
    601     if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
    602 }
    603 
    604 CircularBuffer::CircularBuffer(const BufferKind k, const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
    605 : StreamSetBuffer(k, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
    606 
    607 }
    608 
    609 CircularCopybackBuffer::CircularCopybackBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned AddressSpace)
    610 : CircularBuffer(BufferKind::CircularCopybackBuffer, b, type, bufferBlocks, AddressSpace)
    611 , mOverflowBlocks(overflowBlocks) {
    612     if (bufferBlocks < 2 * overflowBlocks) {
    613         report_fatal_error("CircularCopybackBuffer: bufferBlocks < 2 * overflowBlocks");
    614     }
    615     mUniqueID = "CC" + std::to_string(bufferBlocks);
    616     if (mOverflowBlocks != 1) mUniqueID += "_" + std::to_string(mOverflowBlocks);
    617     if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
    618 }
    619 
    620 ExpandableBuffer::ExpandableBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
    621 : StreamSetBuffer(BufferKind::ExpandableBuffer, type, resolveExpandableStreamSetType(b, type), bufferBlocks, AddressSpace)
    622 , mInitialCapacity(type->getArrayNumElements()) {
    623     mUniqueID = "XP" + std::to_string(bufferBlocks);
    624     if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
    625 }
    626 
    627 SwizzledCopybackBuffer::SwizzledCopybackBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned fieldwidth, unsigned AddressSpace)
    628 : StreamSetBuffer(BufferKind::SwizzledCopybackBuffer, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks), mFieldWidth(fieldwidth) {
    629     mUniqueID = "SW" + std::to_string(fieldwidth) + ":" + std::to_string(bufferBlocks);
    630     if (bufferBlocks < 2 * overflowBlocks) {
    631         report_fatal_error("SwizzledCopybackBuffer: bufferBlocks < 2 * overflowBlocks");
    632     }
    633     if (mOverflowBlocks != 1) {
    634         mUniqueID += "_" + std::to_string(mOverflowBlocks);
    635     }
    636     if (AddressSpace > 0) {
    637         mUniqueID += "@" + std::to_string(AddressSpace);
    638     }
    639 }
    640609
    641610Value * DynamicBuffer::getBaseAddress(IDISA::IDISA_Builder * const b, Value * const handle) const {
     
    647616}
    648617
    649 Value * DynamicBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const b, Value * handle, Value * blockIndex) const {
     618Value * DynamicBuffer::getBlockAddress(IDISA::IDISA_Builder * const b, Value * handle, Value * blockIndex) const {
    650619    Value * const wkgBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::WorkingBlocks))}));
     620    assert (blockIndex->getType() == wkgBlocks->getType());
    651621    return b->CreateGEP(getBaseAddress(b, handle), b->CreateURem(blockIndex, wkgBlocks));
    652622}
    653623
    654 Value * DynamicBuffer::getRawItemPointer(IDISA::IDISA_Builder * const b, Value * handle, Value * streamIndex, Value * absolutePosition) const {
    655     Value * absBlock = b->CreateUDiv(absolutePosition, b->getSize(b->getBitBlockWidth()));
    656     Value * blockPos = b->CreateURem(absolutePosition, b->getSize(b->getBitBlockWidth()));
    657     Value * blockPtr = b->CreateGEP(getStreamSetBlockPtr(b, handle, absBlock), {b->getInt32(0), streamIndex});
     624Value * DynamicBuffer::getRawItemPointer(IDISA::IDISA_Builder * const b, Value * handle, Value * absolutePosition) const {
     625    Constant * blockSize = ConstantInt::get(absolutePosition->getType(), b->getBitBlockWidth());
     626    Value * absBlock = b->CreateUDiv(absolutePosition, blockSize);
     627    Value * blockPos = b->CreateURem(absolutePosition, blockSize);
     628    Value * blockPtr = getBlockAddress(b, handle, absBlock);
    658629    const auto bw = mBaseType->getArrayElementType()->getScalarSizeInBits();
    659630    if (bw < 8) {
     
    669640
    670641Value * DynamicBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const b, Value * handle, Value * fromPosition, Value * availItems, bool reverse) const {
    671     Constant * blockSize = b->getSize(b->getBitBlockWidth());
    672642    Value * const bufBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::WorkingBlocks))}));
     643    Constant * blockSize = ConstantInt::get(bufBlocks->getType(), b->getBitBlockWidth());
    673644    Value * bufSize = b->CreateMul(bufBlocks, blockSize);
     645    assert (bufSize->getType() == fromPosition->getType());
    674646    Value * itemsFromBase = b->CreateURem(fromPosition, bufSize);
    675647    if (reverse) {
    676648        Value * bufAvail = b->CreateSelect(b->CreateICmpEQ(itemsFromBase, b->getSize(0)), bufSize, itemsFromBase);
    677649        return b->CreateSelect(b->CreateICmpULT(bufAvail, availItems), bufAvail, availItems);
    678     }
    679     else {
     650    } else {
    680651        Value * linearSpace = b->CreateSub(bufSize, itemsFromBase, "linearSpace");
    681652        return b->CreateSelect(b->CreateICmpULT(availItems, linearSpace), availItems, linearSpace);
     
    683654}
    684655
    685 Value * DynamicBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const b, Value * handle, Value * fromPosition, bool reverse) const {
    686     Constant * blockSize = b->getSize(b->getBitBlockWidth());
     656Value * DynamicBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const b, Value * handle, Value * fromPosition, bool reverse) const {   
    687657    Value * bufBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(Field::WorkingBlocks))}));
     658    Constant * blockSize = ConstantInt::get(bufBlocks->getType(), b->getBitBlockWidth());
    688659    Value * bufSize = b->CreateMul(bufBlocks, blockSize);
     660    assert (bufSize->getType() == fromPosition->getType());
    689661    Value * bufRem = b->CreateURem(fromPosition, bufSize);
    690662    if (reverse) {
    691663        return b->CreateSelect(b->CreateICmpEQ(bufRem, b->getSize(0)), bufSize, bufRem);
    692664    }
    693     bufSize = b->CreateMul(b->CreateAdd(bufBlocks, b->getSize(mOverflowBlocks)), blockSize);
     665    Constant * overflow = ConstantInt::get(bufBlocks->getType(), mOverflowBlocks);
     666    bufSize = b->CreateMul(b->CreateAdd(bufBlocks, overflow), blockSize);
    694667    return b->CreateSub(bufSize, bufRem, "linearWritable");
    695668}
     
    700673}
    701674
    702 void DynamicBuffer::genCopyBackLogic(IDISA::IDISA_Builder * const b, Value * handle, Value * priorProducedCount, Value * newProducedCount, const std::string Name) {
     675void DynamicBuffer::genCopyBackLogic(IDISA::IDISA_Builder * const b, Value * handle, Value * priorProducedCount, Value * newProducedCount, const std::string Name) const {
     676    assert (priorProducedCount->getType() == newProducedCount->getType());   
    703677    Value * workingBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::WorkingBlocks))}));
    704     Value * bufSize = b->CreateMul(workingBlocks, b->getSize(b->getBitBlockWidth()));
     678    assert (workingBlocks->getType() == newProducedCount->getType());
     679    Value * bufSize = b->CreateMul(workingBlocks, ConstantInt::get(workingBlocks->getType(), b->getBitBlockWidth()));
    705680    Value * priorBufPos = b->CreateURem(priorProducedCount, bufSize);
    706681    Value * newBufPos = b->CreateURem(newProducedCount, bufSize);
    707     BasicBlock * copyBack = b->CreateBasicBlock(Name + "_copyBack");
    708     BasicBlock * done = b->CreateBasicBlock(Name + "_copyBackDone");
     682    BasicBlock * copyBack = b->CreateBasicBlock(Name + "_dynamicCopyBack");
     683    BasicBlock * done = b->CreateBasicBlock(Name + "_dynamicCopyBackDone");
     684
    709685    Value * wraparound = b->CreateICmpUGT(priorBufPos, newBufPos);
    710686    b->CreateCondBr(wraparound, copyBack, done);
     687
    711688    b->SetInsertPoint(copyBack);
    712689    Value * bufBasePtrField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::BaseAddress))});
     
    715692    createBlockAlignedCopy(b, bufBasePtr, overFlowAreaPtr, newBufPos);
    716693    b->CreateBr(done);
     694
    717695    b->SetInsertPoint(done);
    718696}
     
    782760    Value * workingBlocksField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::WorkingBlocks))});
    783761    Value * capacityField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::AllocatedCapacity))});
    784    
     762
    785763    Value * oldBufPtr = b->CreateLoad(bufBasePtrField);
    786764    Value * currentWorkingBlocks = b->CreateLoad(workingBlocksField);
     
    833811}
    834812
     813SourceBuffer::SourceBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, unsigned MemoryAddressSpace, unsigned StructAddressSpace)
     814: StreamSetBuffer(BufferKind::SourceBuffer, type, StructType::get(resolveStreamSetType(b, type)->getPointerTo(MemoryAddressSpace), b->getSizeTy(), b->getSizeTy(), nullptr), 0, StructAddressSpace) {
     815    mUniqueID = "B";
     816    if (MemoryAddressSpace != 0 || StructAddressSpace != 0) {
     817        mUniqueID += "@" + std::to_string(MemoryAddressSpace) + ":" + std::to_string(StructAddressSpace);
     818    }
     819}
     820
     821ExternalBuffer::ExternalBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, llvm::Value * addr, unsigned AddressSpace)
     822: StreamSetBuffer(BufferKind::ExternalBuffer, type, resolveStreamSetType(b, type), 0, AddressSpace) {
     823    mUniqueID = "E";
     824    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
     825    mStreamSetBufferPtr = b->CreatePointerBitCastOrAddrSpaceCast(addr, getPointerType());
     826}
     827
     828CircularBuffer::CircularBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
     829: StreamSetBuffer(BufferKind::CircularBuffer, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
     830    mUniqueID = "C" + std::to_string(bufferBlocks);
     831    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
     832}
     833
     834CircularBuffer::CircularBuffer(const BufferKind k, const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
     835: StreamSetBuffer(k, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
     836
     837}
     838
     839CircularCopybackBuffer::CircularCopybackBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned AddressSpace)
     840: CircularBuffer(BufferKind::CircularCopybackBuffer, b, type, bufferBlocks, AddressSpace)
     841, mOverflowBlocks(overflowBlocks) {
     842    if (bufferBlocks < 2 * overflowBlocks) {
     843        report_fatal_error("CircularCopybackBuffer: bufferBlocks < 2 * overflowBlocks");
     844    }
     845    mUniqueID = "CC" + std::to_string(bufferBlocks);
     846    if (mOverflowBlocks != 1) mUniqueID += "_" + std::to_string(mOverflowBlocks);
     847    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
     848}
     849
     850ExpandableBuffer::ExpandableBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
     851: StreamSetBuffer(BufferKind::ExpandableBuffer, type, resolveExpandableStreamSetType(b, type), bufferBlocks, AddressSpace)
     852, mInitialCapacity(type->getArrayNumElements()) {
     853    mUniqueID = "XP" + std::to_string(bufferBlocks);
     854    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
     855}
     856
     857SwizzledCopybackBuffer::SwizzledCopybackBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned fieldwidth, unsigned AddressSpace)
     858: StreamSetBuffer(BufferKind::SwizzledCopybackBuffer, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks), mFieldWidth(fieldwidth) {
     859    mUniqueID = "SW" + std::to_string(fieldwidth) + ":" + std::to_string(bufferBlocks);
     860    if (bufferBlocks < 2 * overflowBlocks) {
     861        report_fatal_error("SwizzledCopybackBuffer: bufferBlocks < 2 * overflowBlocks");
     862    }
     863    if (mOverflowBlocks != 1) {
     864        mUniqueID += "_" + std::to_string(mOverflowBlocks);
     865    }
     866    if (AddressSpace > 0) {
     867        mUniqueID += "@" + std::to_string(AddressSpace);
     868    }
     869}
     870
    835871inline StructType * getDynamicBufferStructType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * baseType, const unsigned addrSpace) {
    836872    IntegerType * sizeTy = b->getSizeTy();
     
    869905, mBaseType(baseType)
    870906, mProducer(nullptr) {
    871 
     907    assert(k == BufferKind::SourceBuffer || k == BufferKind::ExternalBuffer || BufferBlocks);
    872908}
    873909
Note: See TracChangeset for help on using the changeset viewer.