Changeset 5316


Ignore:
Timestamp:
Feb 13, 2017, 3:10:39 PM (2 years ago)
Author:
nmedfort
Message:

First version of expandable buffers + minor change to array-test to use them.

Location:
icGREP/icgrep-devel/icgrep
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/array-test.cpp

    r5310 r5316  
    116116    ExternalFileBuffer ByteStream(iBuilder, byteStreamTy);
    117117    SingleBlockBuffer BasisBits(iBuilder, iBuilder->getStreamSetTy(8, 1));
    118     SingleBlockBuffer matches(iBuilder, iBuilder->getStreamSetTy(count, 1));
     118    ExpandableBuffer matches(iBuilder, iBuilder->getStreamSetTy(count, 1), 2);
    119119
    120120    MMapSourceKernel mmapK(iBuilder);
     
    152152    IDISA::IDISA_Builder * idb = IDISA::GetIDISA_Builder(M);
    153153
    154     llvm::Function * f = pipeline(idb, 10);
     154    llvm::Function * f = pipeline(idb, 2);
    155155
    156156    verifyModule(*M, &dbgs());
  • icGREP/icgrep-devel/icgrep/kernels/streamset.cpp

    r5311 r5316  
    2121using namespace IDISA;
    2222
     23ArrayType * resolveStreamSetType(IDISA_Builder * const b, Type * const type);
     24
     25StructType * resolveExpandableStreamSetType(IDISA_Builder * const b, Type * const type);
     26
    2327void StreamSetBuffer::allocateBuffer() {
    2428    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType(), iBuilder->getSize(mBufferBlocks));
     
    4246Value * StreamSetBuffer::getRawItemPointer(Value * self, Value * streamIndex, Value * absolutePosition) const {
    4347    Value * ptr = self;
    44     if (isa<ConstantInt>(streamIndex) && cast<ConstantInt>(streamIndex)->isZero()) {
    45         ptr = iBuilder->CreateGEP(ptr, {iBuilder->getInt32(0), streamIndex});
     48    if (!isa<ConstantInt>(streamIndex) || !cast<ConstantInt>(streamIndex)->isZero()) {
     49        ptr = iBuilder->CreateGEP(ptr, {ConstantInt::getNullValue(streamIndex->getType()), streamIndex});
    4650    }
    4751    IntegerType * const ty = cast<IntegerType>(mBaseType->getArrayElementType()->getVectorElementType());
     
    106110    if (mBufferBlocks == 1) {
    107111        offset = ConstantInt::getNullValue(iBuilder->getSizeTy());
    108     } else if ((mBufferBlocks & (mBufferBlocks - 1)) == 0) { // is power of 2
     112    } else if (LLVM_LIKELY((mBufferBlocks & (mBufferBlocks - 1)) == 0)) { // is power of 2
    109113        offset = iBuilder->CreateAnd(blockIndex, ConstantInt::get(blockIndex->getType(), mBufferBlocks - 1));
    110114    } else {
     
    161165    if (mBufferBlocks == 1) {
    162166        offset = ConstantInt::getNullValue(iBuilder->getSizeTy());
    163     } else if ((mBufferBlocks & (mBufferBlocks - 1)) == 0) { // is power of 2
     167    } else if (LLVM_LIKELY((mBufferBlocks & (mBufferBlocks - 1)) == 0)) { // is power of 2
    164168        offset = iBuilder->CreateAnd(blockIndex, ConstantInt::get(blockIndex->getType(), mBufferBlocks - 1));
    165169    } else {
     
    173177// Expandable Buffer
    174178
    175 void ExpandableBuffer::ensureStreamCapacity(llvm::Value * self, llvm::Value * streamIndex) const {
    176 
     179void ExpandableBuffer::allocateBuffer() {
     180    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType());
     181    Value * const capacityPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
     182    iBuilder->CreateStore(iBuilder->getSize(mInitialCapacity), capacityPtr);
     183    Type * const bufferType = getType()->getStructElementType(1)->getPointerElementType();
     184    ConstantInt * const size = iBuilder->getSize(mBufferBlocks * mInitialCapacity);
     185    Value * const ptr = iBuilder->CreateAlignedMalloc(bufferType, size, iBuilder->getCacheAlignment());
     186    const auto alignment = bufferType->getPrimitiveSizeInBits() / 8;
     187    iBuilder->CreateMemZero(ptr, size, alignment);
     188    Value * const streamSetPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
     189    iBuilder->CreateStore(ptr, streamSetPtr);
     190}
     191
     192std::pair<Value *, Value *> ExpandableBuffer::getExpandedStreamOffset(llvm::Value * self, llvm::Value * streamIndex, Value * blockIndex) const {
     193
     194    // MDNode *Weights = MDBuilder(Ctx).createBranchWeights(42, 13);
     195
     196    // ENTRY
     197    Value * const capacityPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
     198    Value * const capacity = iBuilder->CreateLoad(capacityPtr);
     199    Value * const streamSetPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
     200    Value * const streamSet = iBuilder->CreateLoad(streamSetPtr);
     201
     202    // Are we guaranteed that we can access this stream?
     203    if (LLVM_UNLIKELY(isa<ConstantInt>(streamIndex))) {
     204        if (LLVM_LIKELY(cast<ConstantInt>(streamIndex)->getLimitedValue() < mInitialCapacity)) {
     205            return {streamSet, capacity};
     206        }
     207    }
     208
     209    BasicBlock * const entry = iBuilder->GetInsertBlock();
     210    BasicBlock * const expand = BasicBlock::Create(iBuilder->getContext(), "expand", entry->getParent());
     211    BasicBlock * const resume = BasicBlock::Create(iBuilder->getContext(), "resume", entry->getParent());
     212
     213    assert (streamIndex->getType() == capacity->getType());
     214    Value * cond = iBuilder->CreateICmpULT(streamIndex, capacity);
     215    iBuilder->CreateCondBr(cond, resume, expand);
     216    // EXPAND
     217    iBuilder->SetInsertPoint(expand);
     218    /// TODO: this should call a function rather than be inlined into the block. REVISIT once tested.
     219    Value * newCapacity = iBuilder->CreateMul(streamIndex, iBuilder->getSize(2));
     220    iBuilder->CreateStore(newCapacity, capacityPtr);
     221    Type * bufferType = getType()->getStructElementType(1)->getPointerElementType();
     222    Value * size = iBuilder->CreateMul(newCapacity, iBuilder->getSize(mBufferBlocks));
     223    Value * newStreamSet = iBuilder->CreateAlignedMalloc(bufferType, size, iBuilder->getCacheAlignment());
     224    iBuilder->CreateStore(newStreamSet, streamSetPtr);
     225    Value * const diffCapacity = iBuilder->CreateSub(newCapacity, capacity);
     226    const auto alignment = bufferType->getPrimitiveSizeInBits() / 8;
     227    for (unsigned i = 0; i < mBufferBlocks; ++i) {
     228        ConstantInt * const offset = iBuilder->getSize(i);
     229        Value * srcOffset = iBuilder->CreateMul(capacity, offset);
     230        Value * srcPtr = iBuilder->CreateGEP(streamSet, srcOffset);
     231        Value * destOffset = iBuilder->CreateMul(newCapacity, offset);
     232        Value * destPtr = iBuilder->CreateGEP(newStreamSet, destOffset);
     233        iBuilder->CreateMemCpy(destPtr, srcPtr, capacity, alignment);
     234        Value * destZeroOffset = iBuilder->CreateAdd(destOffset, capacity);
     235        Value * destZeroPtr = iBuilder->CreateGEP(newStreamSet, destZeroOffset);
     236        iBuilder->CreateMemZero(destZeroPtr, diffCapacity, alignment);
     237    }
     238
     239    iBuilder->CreateAlignedFree(streamSet);
     240    iBuilder->CreateBr(resume);
     241    // RESUME
     242    iBuilder->SetInsertPoint(resume);
     243
     244    PHINode * phiStreamSet = iBuilder->CreatePHI(streamSet->getType(), 2);
     245    phiStreamSet->addIncoming(streamSet, entry);
     246    phiStreamSet->addIncoming(newStreamSet, expand);
     247
     248    PHINode * phiCapacity = iBuilder->CreatePHI(capacity->getType(), 2);
     249    phiCapacity->addIncoming(capacity, entry);
     250    phiCapacity->addIncoming(newCapacity, expand);
     251
     252    Value * offset = iBuilder->CreateAdd(iBuilder->CreateMul(blockIndex, phiCapacity), streamIndex);
     253
     254    return {phiStreamSet, offset};
    177255}
    178256
    179257llvm::Value * ExpandableBuffer::getStream(llvm::Value * self, Value * streamIndex, Value * blockIndex) const {
    180     ensureStreamCapacity(self, streamIndex);
    181 
    182 
    183 
    184     return nullptr;
    185 }
    186 
    187 llvm::Value * ExpandableBuffer::getStream(llvm::Value * self, llvm::Value * streamIndex, Value *blockIndex, Value *packIndex) const {
    188     ensureStreamCapacity(self, streamIndex);
    189 
    190 
    191     return nullptr;
     258    Value * ptr, * offset;
     259    std::tie(ptr, offset) = getExpandedStreamOffset(self, streamIndex, blockIndex);
     260    return iBuilder->CreateGEP(ptr, offset);
     261}
     262
     263llvm::Value * ExpandableBuffer::getStream(llvm::Value * self, llvm::Value * streamIndex, Value * blockIndex, Value * packIndex) const {
     264    Value * ptr, * offset;
     265    std::tie(ptr, offset) = getExpandedStreamOffset(self, streamIndex, blockIndex);
     266    return iBuilder->CreateGEP(ptr, {offset, packIndex});
    192267}
    193268
     
    201276
    202277// Constructors
    203 
    204278SingleBlockBuffer::SingleBlockBuffer(IDISA::IDISA_Builder * b, llvm::Type * type)
    205 : StreamSetBuffer(BufferKind::BlockBuffer, b, type, 1, 0) {
     279: StreamSetBuffer(BufferKind::BlockBuffer, b, type, resolveStreamSetType(b, type), 1, 0) {
    206280
    207281}
    208282
    209283ExternalFileBuffer::ExternalFileBuffer(IDISA::IDISA_Builder * b, llvm::Type * type, unsigned AddressSpace)
    210 : StreamSetBuffer(BufferKind::ExternalFileBuffer, b, type, 0, AddressSpace) {
     284: StreamSetBuffer(BufferKind::ExternalFileBuffer, b, type, resolveStreamSetType(b, type), 0, AddressSpace) {
    211285
    212286}
    213287
    214288CircularBuffer::CircularBuffer(IDISA::IDISA_Builder * b, llvm::Type * type, size_t bufferBlocks, unsigned AddressSpace)
    215 : StreamSetBuffer(BufferKind::CircularBuffer, b, type, bufferBlocks, AddressSpace) {
     289: StreamSetBuffer(BufferKind::CircularBuffer, b, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
    216290
    217291}
    218292
    219293CircularCopybackBuffer::CircularCopybackBuffer(IDISA::IDISA_Builder * b, llvm::Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned AddressSpace)
    220 : StreamSetBuffer(BufferKind::CircularCopybackBuffer, b, type, bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks) {
     294: StreamSetBuffer(BufferKind::CircularCopybackBuffer, b, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks) {
    221295
    222296}
    223297
    224298ExpandableBuffer::ExpandableBuffer(IDISA::IDISA_Builder * b, llvm::Type * type, size_t bufferBlocks, unsigned AddressSpace)
    225 : StreamSetBuffer(BufferKind::ExpandableBuffer, b, type, bufferBlocks, AddressSpace) {
    226 
    227 }
    228 
    229 inline Type * resolveStreamSetType(IDISA_Builder * const b, Type * const type) {
     299: StreamSetBuffer(BufferKind::ExpandableBuffer, b, type, resolveExpandableStreamSetType(b, type), bufferBlocks, AddressSpace)
     300, mInitialCapacity(type->getArrayNumElements()) {
     301
     302}
     303
     304inline StreamSetBuffer::StreamSetBuffer(BufferKind k, IDISA::IDISA_Builder * b, Type * baseType, Type * resolvedType, unsigned blocks, unsigned AddressSpace)
     305: mBufferKind(k)
     306, iBuilder(b)
     307, mType(resolvedType)
     308, mBufferBlocks(blocks)
     309, mAddressSpace(AddressSpace)
     310, mStreamSetBufferPtr(nullptr)
     311, mBaseType(baseType) {
     312
     313}
     314
     315// Helper routines
     316ArrayType * resolveStreamSetType(IDISA_Builder * const b, Type * const type) {
    230317    if (type->isArrayTy()) {
    231318        Type * ty = type->getArrayElementType();
     
    236323                ty = b->getBitBlockType();
    237324                if (fieldWidth != 1) {
    238                     ty = llvm::ArrayType::get(ty, fieldWidth);
     325                    ty = ArrayType::get(ty, fieldWidth);
    239326                }
    240327                return ArrayType::get(ty, type->getArrayNumElements());
     
    249336}
    250337
    251 StreamSetBuffer::StreamSetBuffer(BufferKind k, IDISA::IDISA_Builder * b, Type * type, unsigned blocks, unsigned AddressSpace)
    252 : mBufferKind(k)
    253 , iBuilder(b)
    254 , mType(resolveStreamSetType(b, type))
    255 , mBufferBlocks(blocks)
    256 , mAddressSpace(AddressSpace)
    257 , mStreamSetBufferPtr(nullptr)
    258 , mBaseType(type) {
    259 
    260 }
     338StructType * resolveExpandableStreamSetType(IDISA_Builder * const b, Type * const type) {
     339    if (type->isArrayTy()) {
     340        Type * ty = type->getArrayElementType();
     341        if (LLVM_LIKELY(ty->isVectorTy() && ty->getVectorNumElements() == 0)) {
     342            ty = ty->getVectorElementType();
     343            if (LLVM_LIKELY(ty->isIntegerTy())) {
     344                const auto fieldWidth = cast<IntegerType>(ty)->getBitWidth();
     345                ty = b->getBitBlockType();
     346                if (fieldWidth != 1) {
     347                    ty = ArrayType::get(ty, fieldWidth);
     348                }
     349                return StructType::get(b->getSizeTy(), ty->getPointerTo(), nullptr);
     350            }
     351        }
     352    }
     353    std::string tmp;
     354    raw_string_ostream out(tmp);
     355    type->print(out);
     356    out << " is an unvalid stream set buffer type.";
     357    report_fatal_error(out.str());
     358}
  • icGREP/icgrep-devel/icgrep/kernels/streamset.h

    r5311 r5316  
    5858protected:
    5959
    60     StreamSetBuffer(BufferKind k, IDISA::IDISA_Builder * b, llvm::Type * type, unsigned blocks, unsigned AddressSpace);
     60    StreamSetBuffer(BufferKind k, IDISA::IDISA_Builder * b, llvm::Type * baseType, llvm::Type * resolvedType, unsigned blocks, unsigned AddressSpace);
    6161
    6262    // Get the buffer pointer for a given block of the stream.
     
    160160    llvm::Value * getLinearlyAccessibleItems(llvm::Value * fromPosition) const override;
    161161   
     162    void allocateBuffer() override;
     163
    162164protected:
    163165
    164     void ensureStreamCapacity(llvm::Value * self, llvm::Value * streamIndex) const;
     166    llvm::Value * getStreamSetPtr(llvm::Value * self, llvm::Value * blockIndex) const override;
    165167
    166     llvm::Value * getStreamSetPtr(llvm::Value * self, llvm::Value * blockIndex) const override;
     168private:
     169
     170    std::pair<llvm::Value *, llvm::Value *> getExpandedStreamOffset(llvm::Value * self, llvm::Value * streamIndex, llvm::Value * blockIndex) const;
     171
     172private:
     173
     174    const uint64_t  mInitialCapacity;
     175
    167176};
    168177
  • icGREP/icgrep-devel/icgrep/pablo/pablo_compiler.cpp

    r5311 r5316  
    529529        if (DebugOptionIsSet(DumpTrace)) {
    530530            const String & name = isa<Var>(expr) ? cast<Var>(expr)->getName() : cast<Statement>(expr)->getName();
    531             if (value->getType()->isVectorTy()) {
     531            Type * type = value->getType();
     532            if (type->isPointerTy()) {
     533                type = type->getPointerElementType();
     534                if (!type->isIntegerTy() && !type->isVectorTy()) {
     535                    return;
     536                }
     537                value = iBuilder->CreateLoad(value);
     538            }
     539            if (type->isVectorTy()) {
    532540                iBuilder->CallPrintRegister(name.str(), value);
    533             } else if (value->getType()->isIntegerTy()) {
     541            } else if (type->isIntegerTy()) {
    534542                iBuilder->CallPrintInt(name.str(), value);
    535543            }
Note: See TracChangeset for help on using the changeset viewer.