Ignore:
Timestamp:
May 1, 2018, 6:18:30 PM (16 months ago)
Author:
nmedfort
Message:

Removed temporary buffers from pipeline and placed them in the source kernels.

Location:
icGREP/icgrep-devel/icgrep/kernels
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/kernels/kernel_builder.cpp

    r5998 r6008  
    298298}
    299299
    300 /** ------------------------------------------------------------------------------------------------------------- *
    301  * @brief AcquireTemporaryBuffer
    302  ** ------------------------------------------------------------------------------------------------------------- */
    303 std::pair<Value *, Value *> KernelBuilder::AcquireTemporaryBuffer(const std::string & name, Value * offset, Value * itemsToCopy) {
    304     const StreamSetBuffer * const buf = mKernel->getAnyStreamSetBuffer(name);
    305     const auto itemWidth = getItemWidth(buf->getBaseType());
    306     const Binding & binding = mKernel->getBinding(name);
    307     if (LLVM_UNLIKELY(!binding.getRate().isFixed())) {
    308         Constant * const BIT_BLOCK_WIDTH = ConstantInt::get(offset->getType(), getBitBlockWidth());
    309         Value * const alignedOffset = CreateAnd(offset, CreateNeg(BIT_BLOCK_WIDTH));
    310         itemsToCopy = CreateAdd(itemsToCopy, CreateSub(offset, alignedOffset));
    311         offset = alignedOffset;
    312     }
    313     Value * bytesToCopy = itemsToCopy;
    314     if (itemWidth < 8) {
    315         bytesToCopy = CreateCeilUDiv(itemsToCopy, getSize(8 / itemWidth));
    316     } else if (itemWidth > 8) {
    317         bytesToCopy = CreateMul(itemsToCopy, getSize(itemWidth / 8));
    318     }
    319     Constant * const baseSize = ConstantExpr::getTrunc(ConstantExpr::getSizeOf(buf->getStreamSetBlockType()), getSizeTy());
    320     Constant * const itemsConsumedPerIteration = getSize(std::max(ceiling(mKernel->getUpperBound(binding.getRate())), 1U));
    321     Constant * const paddedSize =  ConstantExpr::getMul(baseSize, itemsConsumedPerIteration);
    322 
    323     // one is added to bytes to copy to ensure that the stream is "zero-extended" by one block to properly handle any
    324     // final block processing.o
    325     Value * const size = CreateRoundUp(CreateAdd(bytesToCopy, getSize(1)), paddedSize);
    326     Value * const handle = getStreamHandle(name);
    327     Value * const base = buf->getBaseAddress(this, handle);
    328     Value * const buffer = CreateAlignedMalloc(size, getCacheAlignment());
    329     // TODO: handle split copy? currently no SourceBuffers could support it and I'm not sure how useful it'd be to do so.
    330     Value * const from = buf->getRawItemPointer(this, handle, offset);
    331     CreateMemCpy(buffer, from, bytesToCopy, 1);
    332     CreateMemZero(CreateGEP(buffer, bytesToCopy), CreateSub(size, bytesToCopy), 1);
    333     // get the difference between our base and from position then compute an offsetted temporary buffer address
    334     Value * const diff = CreatePtrDiff(CreatePointerCast(base, from->getType()), from);
    335     Value * const offsettedBuffer = CreatePointerCast(CreateGEP(buffer, diff), base->getType());
    336     buf->setBaseAddress(this, handle, offsettedBuffer);
    337     Value * const tempBuffer = CreatePointerCast(buffer, base->getType());
    338     return std::make_pair(base, tempBuffer);
    339 }
    340 
    341300Value * KernelBuilder::getConsumerLock(const std::string & name) {
    342301    return getScalarField(name + CONSUMER_SUFFIX);
  • icGREP/icgrep-devel/icgrep/kernels/kernel_builder.h

    r5998 r6008  
    153153    void CreateCopyToOverflow(const std::string & name);
    154154
    155     std::pair<llvm::Value *, llvm::Value *> AcquireTemporaryBuffer(const std::string & name, llvm::Value * const offset, llvm::Value * const itemsToCopy);
    156 
    157155    void setBaseAddress(const std::string & name, llvm::Value * addr);
    158156
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

    r5985 r6008  
    7070}
    7171
     72
    7273void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
    7374
     
    7778    BasicBlock * const exit = b->CreateBasicBlock("mmapSourceExit");
    7879
    79     Value * const fileSize = b->getScalarField("fileSize");
    8080    Constant * const PAGE_SIZE = b->getSize(getpagesize());
    81 
     81    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
     82    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
     83
     84    Value * const fileItems = b->getScalarField("fileSize");
    8285    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
    83     Value * const consumedBytes = b->CreateMul(consumedItems, b->getSize(codeUnitWidth / 8));
     86    Value * const consumedBytes = b->CreateMul(consumedItems, CODE_UNIT_BYTES);
    8487    Value * const consumedPageOffset = b->CreateAnd(consumedBytes, ConstantExpr::getNeg(PAGE_SIZE));
    8588    Value * const consumedBuffer = b->getRawOutputPointer("sourceBuffer", consumedPageOffset);
     
    99102    b->SetInsertPoint(checkRemaining);
    100103    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
    101     Value * const pageItems = b->CreateUDiv(PAGE_SIZE, b->getSize(codeUnitWidth / 8));
    102     Value * const nextProducedItems = b->CreateAdd(producedItems, pageItems);
    103     Value * const lastPage = b->CreateICmpULE(fileSize, nextProducedItems);
     104    Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE);
     105    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
    104106    b->CreateUnlikelyCondBr(lastPage, setTermination, exit);
    105107
     108    // If this is the last page, create a temporary buffer of up to two pages size, copy the unconsumed data
     109    // and zero any bytes that are not used.
    106110    b->SetInsertPoint(setTermination);
     111    Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
     112    Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
     113    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
     114    Value * const unconsumedBytes = b->CreatePtrDiff(readEnd, readStart);
     115    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
     116    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
     117    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
     118    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
     119    // get the difference between our base and from position then compute an offsetted temporary buffer address
     120    Value * const base = b->getBaseAddress("sourceBuffer");
     121    Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
     122    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
     123    b->CreateConsumerWait();
     124    // Unmap the file buffer and set the temporary buffer as the new source buffer
     125    Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES);
     126    b->CreateMUnmap(base, fileSize);
     127    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
     128    b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
     129    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
    107130    b->setTerminationSignal();
     131    BasicBlock * const terminationExit = b->GetInsertBlock();
    108132    b->CreateBr(exit);
    109133
     
    112136    PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
    113137    newProducedItems->addIncoming(nextProducedItems, checkRemaining);
    114     newProducedItems->addIncoming(fileSize, setTermination);
     138    newProducedItems->addIncoming(fileItems, terminationExit);
    115139    b->setProducedItemCount("sourceBuffer", newProducedItems);
    116140}
    117141
    118 void MMapSourceKernel::unmapSourceBuffer(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
    119     Value * fileSize = b->getBufferedSize("sourceBuffer");
    120     if (LLVM_UNLIKELY(codeUnitWidth > 8)) {
    121         fileSize = b->CreateMul(fileSize, b->getSize(codeUnitWidth / 8));
    122     }
    123     b->CreateMUnmap(b->getBaseAddress("sourceBuffer"), fileSize);
     142void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
     143    b->CreateFree(b->getScalarField("buffer"));
    124144}
    125145
     
    257277    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), finalizeMMap, finalizeRead);
    258278    b->SetInsertPoint(finalizeMMap);
    259     MMapSourceKernel::unmapSourceBuffer(mCodeUnitWidth, b);
     279    MMapSourceKernel::freeBuffer(b);
    260280    b->CreateBr(finalizeDone);
    261281    b->SetInsertPoint(finalizeRead);
     
    316336    b->setBaseAddress("sourceBuffer", fileSource);
    317337    Value * const fileSize = b->getScalarField("fileSize");
    318     Value * const fileItems = b->CreateUDiv(fileSize, b->getSize(mCodeUnitWidth / 8));
    319     b->setBufferedSize("sourceBuffer", fileItems);
    320     b->setCapacity("sourceBuffer", fileItems);
    321     b->setProducedItemCount("sourceBuffer", fileItems);
    322     b->setTerminationSignal();
     338    b->setBufferedSize("sourceBuffer", fileSize);
     339    b->setCapacity("sourceBuffer", fileSize);
     340    if (mStreamSetCount > 1) {
     341        b->setProducedItemCount("sourceBuffer", fileSize);
     342        b->setTerminationSignal();
     343    }
    323344}
    324345
    325346void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
    326 
     347    if (mStreamSetCount == 1) {
     348        Constant * const PAGE_SIZE = b->getSize(getStride());
     349        Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
     350
     351        BasicBlock * const entry = b->GetInsertBlock();
     352        BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary");
     353        BasicBlock * const exit = b->CreateBasicBlock("exit");
     354
     355        Value * const fileItems = b->getScalarField("fileSize");
     356        Value * const producedItems = b->getProducedItemCount("sourceBuffer");
     357        Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE);
     358        Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
     359        b->CreateUnlikelyCondBr(lastPage, createTemporary, exit);
     360
     361        b->SetInsertPoint(createTemporary);
     362        Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
     363        Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
     364        Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
     365        Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
     366        Value * const unconsumedBytes = b->CreatePtrDiff(readEnd, readStart);
     367        Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
     368        Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
     369        b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
     370        b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
     371
     372        // get the difference between our base and from position then compute an offsetted temporary buffer address
     373        Value * const base = b->getBaseAddress("sourceBuffer");
     374        Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
     375        Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
     376        b->CreateConsumerWait();
     377
     378        // set the temporary buffer as the new source buffer
     379        PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
     380        b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
     381        b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
     382        b->setTerminationSignal();
     383        BasicBlock * const terminationExit = b->GetInsertBlock();
     384        b->CreateBr(exit);
     385
     386        b->SetInsertPoint(exit);
     387        PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
     388        newProducedItems->addIncoming(nextProducedItems, entry);
     389        newProducedItems->addIncoming(fileItems, terminationExit);
     390        b->setProducedItemCount("sourceBuffer", newProducedItems);
     391    }
     392}
     393
     394void MemorySourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
     395    if (mStreamSetCount == 1) {
     396        b->CreateFree(b->getScalarField("buffer"));
     397    }
    327398}
    328399
     
    337408, mFileSizeFunction(nullptr) {
    338409    addAttribute(MustExplicitlyTerminate());
    339     setStride((8 * getpagesize()) / codeUnitWidth);
     410    setStride(getpagesize());
    340411}
    341412
     
    349420, mCodeUnitWidth(codeUnitWidth) {
    350421    addAttribute(MustExplicitlyTerminate());
    351     setStride((8 * getpagesize()) / codeUnitWidth);
     422    setStride(getpagesize());
    352423}
    353424
     
    356427: SegmentOrientedKernel("FD_source@" + std::to_string(codeUnitWidth)
    357428, {}
    358 // output
     429// output stream
    359430, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
    360431// input scalar
    361432, {Binding{b->getInt8Ty(), "useMMap"}, Binding{b->getInt32Ty(), "fileDescriptor"}}
    362433, {}
     434// internal scalar
    363435, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}, Binding{b->getSizeTy(), "fileSize"}})
    364436, mCodeUnitWidth(codeUnitWidth)
    365437, mFileSizeFunction(nullptr) {
    366438    addAttribute(MustExplicitlyTerminate());
    367     setStride((8 * getpagesize()) / codeUnitWidth);
    368 }
    369 
    370 MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Type * const type, const unsigned codeUnitWidth)
    371 : SegmentOrientedKernel("memory_source",
    372     {},
    373     {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}},
    374     {Binding{cast<PointerType>(type), "fileSource"}, Binding{b->getSizeTy(), "fileSize"}}, {}, {})
     439    setStride(getpagesize());
     440}
     441
     442MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned streamSetCount, const unsigned codeUnitWidth)
     443: SegmentOrientedKernel("memory_source@" + std::to_string(streamSetCount) + ":" + std::to_string(codeUnitWidth),
     444{},
     445// output stream
     446{Binding{b->getStreamSetTy(streamSetCount, codeUnitWidth), "sourceBuffer"}},
     447// input scalar
     448{Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "fileSource"}, Binding{b->getSizeTy(), "fileSize"}},
     449{},
     450// internal scalar
     451{Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
     452, mStreamSetCount(streamSetCount)
    375453, mCodeUnitWidth(codeUnitWidth) {
    376454    addAttribute(MustExplicitlyTerminate());
    377 }
    378 
    379 }
     455    setStride(getpagesize());
     456}
     457
     458}
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.h

    r5985 r6008  
    2727    }
    2828    void generateFinalizeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override {
    29         unmapSourceBuffer(mCodeUnitWidth, iBuilder);
     29        freeBuffer(iBuilder);
    3030    }
    3131protected:
     
    3333    static void generateInitializeMethod(llvm::Function * fileSize, const unsigned codeUnitWidth, const std::unique_ptr<kernel::KernelBuilder> & b);
    3434    static void generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<kernel::KernelBuilder> & b);
    35     static void unmapSourceBuffer(const unsigned codeUnitWidth, const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     35    static void freeBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3636protected:
    3737    const unsigned mCodeUnitWidth;
     
    7878class MemorySourceKernel final : public SegmentOrientedKernel {
    7979public:
    80     MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, llvm::Type * const type, const unsigned codeUnitWidth = 8);
     80    MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const unsigned streamSetCount = 1, const unsigned codeUnitWidth = 8);
    8181    bool hasSignature() const override { return false; }
    8282protected:
    8383    void generateInitializeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
    8484    void generateDoSegmentMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
     85    void generateFinalizeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
    8586private:
     87    const unsigned mStreamSetCount;
    8688    const unsigned mCodeUnitWidth;
    8789};
Note: See TracChangeset for help on using the changeset viewer.