Changeset 6008


Ignore:
Timestamp:
May 1, 2018, 6:18:30 PM (6 months ago)
Author:
nmedfort
Message:

Removed temporary buffers from pipeline and placed them in the source kernels.

Location:
icGREP/icgrep-devel/icgrep
Files:
13 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/character_deletion.cpp

    r5985 r6008  
    7676    char *fileBuffer = const_cast<char *>(mappedFile.data());
    7777
    78     if (codegen::SegmentSize < 2) {
    79         codegen::SegmentSize = 2;
    80     }
    81 
    8278    const int inputBufferBlocks = codegen::BufferSegments * codegen::ThreadNum * 16;
    8379
     
    105101    StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), inputBufferBlocks);
    106102
    107     kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder, iBuilder->getInt8PtrTy());
     103    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
    108104    sourceK->setInitialArguments({inputStream, fileSize});
    109105    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
  • icGREP/icgrep-devel/icgrep/character_deposit.cpp

    r5986 r6008  
    7979    char *fileBuffer = const_cast<char *>(mappedFile.data());
    8080
    81     if (codegen::SegmentSize < 2) {
    82         codegen::SegmentSize = 2;
    83     }
    84 
    8581    const auto bufferSize = codegen::ThreadNum * codegen::SegmentSize;
    8682
     
    109105    StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), bufferSize);
    110106
    111     kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder, iBuilder->getInt8PtrTy());
     107    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
    112108    sourceK->setInitialArguments({inputStream, fileSize});
    113109    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
  • icGREP/icgrep-devel/icgrep/editd/editd.cpp

    r5935 r6008  
    241241    Type * const sizeTy = idb->getSizeTy();
    242242    Type * const voidTy = idb->getVoidTy();
    243     Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(idb->getBitBlockType(), 8), 1), 0);
     243    Type * const inputType = idb->getIntNTy(1)->getPointerTo();
    244244
    245245    idb->LinkFunction("wrapped_report_pos", &wrapped_report_pos);
     
    258258
    259259    auto ChStream = pxDriver.addBuffer<SourceBuffer>(idb, idb->getStreamSetTy(4));
    260     auto mmapK = pxDriver.addKernelInstance<MemorySourceKernel>(idb, inputType);
     260    auto mmapK = pxDriver.addKernelInstance<MemorySourceKernel>(idb, 4, 1);
    261261    mmapK->setInitialArguments({inputStream, fileSize});
    262262    pxDriver.makeKernelCall(mmapK, {}, {ChStream});
     
    417417    Type * const sizeTy = idb->getSizeTy();
    418418    Type * const voidTy = idb->getVoidTy();
    419     Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(idb->getBitBlockType(), 8), 1), 0);
     419    Type * const inputType = idb->getIntNTy(1)->getPointerTo();
    420420    Type * const patternPtrTy = PointerType::get(idb->getInt8Ty(), 0);
    421421
     
    437437
    438438    auto ChStream = pxDriver.addBuffer<SourceBuffer>(idb, idb->getStreamSetTy(4));
    439     auto mmapK = pxDriver.addKernelInstance<MemorySourceKernel>(idb, inputType);
     439    auto mmapK = pxDriver.addKernelInstance<MemorySourceKernel>(idb, 4, 1);
    440440    mmapK->setInitialArguments({inputStream, fileSize});
    441441    pxDriver.makeKernelCall(mmapK, {}, {ChStream});
     
    680680    fileSize->setName("fileSize");
    681681
    682     StreamSetBuffer * MatchResults = pxDriver.addBuffer<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(editDistance+1));
    683     kernel::Kernel * sourceK = pxDriver.addKernelInstance<kernel::MemorySourceKernel>(iBuilder, inputType);
     682    StreamSetBuffer * MatchResults = pxDriver.addBuffer<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(editDistance + 1));
     683    kernel::Kernel * sourceK = pxDriver.addKernelInstance<kernel::MemorySourceKernel>(iBuilder, editDistance + 1, 8);
    684684    sourceK->setInitialArguments({inputStream, fileSize});
    685685    pxDriver.makeKernelCall(sourceK, {}, {MatchResults});
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.cpp

    r5999 r6008  
    817817    idb->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", mainFunc, 0));
    818818    StreamSetBuffer * ByteStream = mGrepDriver->addBuffer<SourceBuffer>(idb, idb->getStreamSetTy(1, 8));
    819     kernel::Kernel * sourceK = mGrepDriver->addKernelInstance<kernel::MemorySourceKernel>(idb, idb->getInt8PtrTy());
     819    kernel::Kernel * sourceK = mGrepDriver->addKernelInstance<kernel::MemorySourceKernel>(idb);
    820820    sourceK->setInitialArguments({buffer, length});
    821821    mGrepDriver->makeKernelCall(sourceK, {}, {ByteStream});
  • icGREP/icgrep-devel/icgrep/kernels/kernel_builder.cpp

    r5998 r6008  
    298298}
    299299
    300 /** ------------------------------------------------------------------------------------------------------------- *
    301  * @brief AcquireTemporaryBuffer
    302  ** ------------------------------------------------------------------------------------------------------------- */
    303 std::pair<Value *, Value *> KernelBuilder::AcquireTemporaryBuffer(const std::string & name, Value * offset, Value * itemsToCopy) {
    304     const StreamSetBuffer * const buf = mKernel->getAnyStreamSetBuffer(name);
    305     const auto itemWidth = getItemWidth(buf->getBaseType());
    306     const Binding & binding = mKernel->getBinding(name);
    307     if (LLVM_UNLIKELY(!binding.getRate().isFixed())) {
    308         Constant * const BIT_BLOCK_WIDTH = ConstantInt::get(offset->getType(), getBitBlockWidth());
    309         Value * const alignedOffset = CreateAnd(offset, CreateNeg(BIT_BLOCK_WIDTH));
    310         itemsToCopy = CreateAdd(itemsToCopy, CreateSub(offset, alignedOffset));
    311         offset = alignedOffset;
    312     }
    313     Value * bytesToCopy = itemsToCopy;
    314     if (itemWidth < 8) {
    315         bytesToCopy = CreateCeilUDiv(itemsToCopy, getSize(8 / itemWidth));
    316     } else if (itemWidth > 8) {
    317         bytesToCopy = CreateMul(itemsToCopy, getSize(itemWidth / 8));
    318     }
    319     Constant * const baseSize = ConstantExpr::getTrunc(ConstantExpr::getSizeOf(buf->getStreamSetBlockType()), getSizeTy());
    320     Constant * const itemsConsumedPerIteration = getSize(std::max(ceiling(mKernel->getUpperBound(binding.getRate())), 1U));
    321     Constant * const paddedSize =  ConstantExpr::getMul(baseSize, itemsConsumedPerIteration);
    322 
    323     // one is added to bytes to copy to ensure that the stream is "zero-extended" by one block to properly handle any
    324     // final block processing.o
    325     Value * const size = CreateRoundUp(CreateAdd(bytesToCopy, getSize(1)), paddedSize);
    326     Value * const handle = getStreamHandle(name);
    327     Value * const base = buf->getBaseAddress(this, handle);
    328     Value * const buffer = CreateAlignedMalloc(size, getCacheAlignment());
    329     // TODO: handle split copy? currently no SourceBuffers could support it and I'm not sure how useful it'd be to do so.
    330     Value * const from = buf->getRawItemPointer(this, handle, offset);
    331     CreateMemCpy(buffer, from, bytesToCopy, 1);
    332     CreateMemZero(CreateGEP(buffer, bytesToCopy), CreateSub(size, bytesToCopy), 1);
    333     // get the difference between our base and from position then compute an offsetted temporary buffer address
    334     Value * const diff = CreatePtrDiff(CreatePointerCast(base, from->getType()), from);
    335     Value * const offsettedBuffer = CreatePointerCast(CreateGEP(buffer, diff), base->getType());
    336     buf->setBaseAddress(this, handle, offsettedBuffer);
    337     Value * const tempBuffer = CreatePointerCast(buffer, base->getType());
    338     return std::make_pair(base, tempBuffer);
    339 }
    340 
    341300Value * KernelBuilder::getConsumerLock(const std::string & name) {
    342301    return getScalarField(name + CONSUMER_SUFFIX);
  • icGREP/icgrep-devel/icgrep/kernels/kernel_builder.h

    r5998 r6008  
    153153    void CreateCopyToOverflow(const std::string & name);
    154154
    155     std::pair<llvm::Value *, llvm::Value *> AcquireTemporaryBuffer(const std::string & name, llvm::Value * const offset, llvm::Value * const itemsToCopy);
    156 
    157155    void setBaseAddress(const std::string & name, llvm::Value * addr);
    158156
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

    r5985 r6008  
    7070}
    7171
     72
    7273void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
    7374
     
    7778    BasicBlock * const exit = b->CreateBasicBlock("mmapSourceExit");
    7879
    79     Value * const fileSize = b->getScalarField("fileSize");
    8080    Constant * const PAGE_SIZE = b->getSize(getpagesize());
    81 
     81    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
     82    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
     83
     84    Value * const fileItems = b->getScalarField("fileSize");
    8285    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
    83     Value * const consumedBytes = b->CreateMul(consumedItems, b->getSize(codeUnitWidth / 8));
     86    Value * const consumedBytes = b->CreateMul(consumedItems, CODE_UNIT_BYTES);
    8487    Value * const consumedPageOffset = b->CreateAnd(consumedBytes, ConstantExpr::getNeg(PAGE_SIZE));
    8588    Value * const consumedBuffer = b->getRawOutputPointer("sourceBuffer", consumedPageOffset);
     
    99102    b->SetInsertPoint(checkRemaining);
    100103    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
    101     Value * const pageItems = b->CreateUDiv(PAGE_SIZE, b->getSize(codeUnitWidth / 8));
    102     Value * const nextProducedItems = b->CreateAdd(producedItems, pageItems);
    103     Value * const lastPage = b->CreateICmpULE(fileSize, nextProducedItems);
     104    Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE);
     105    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
    104106    b->CreateUnlikelyCondBr(lastPage, setTermination, exit);
    105107
     108    // If this is the last page, create a temporary buffer of up to two pages size, copy the unconsumed data
     109    // and zero any bytes that are not used.
    106110    b->SetInsertPoint(setTermination);
     111    Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
     112    Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
     113    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
     114    Value * const unconsumedBytes = b->CreatePtrDiff(readEnd, readStart);
     115    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
     116    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
     117    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
     118    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
     119    // get the difference between our base and from position then compute an offsetted temporary buffer address
     120    Value * const base = b->getBaseAddress("sourceBuffer");
     121    Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
     122    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
     123    b->CreateConsumerWait();
     124    // Unmap the file buffer and set the temporary buffer as the new source buffer
     125    Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES);
     126    b->CreateMUnmap(base, fileSize);
     127    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
     128    b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
     129    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
    107130    b->setTerminationSignal();
     131    BasicBlock * const terminationExit = b->GetInsertBlock();
    108132    b->CreateBr(exit);
    109133
     
    112136    PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
    113137    newProducedItems->addIncoming(nextProducedItems, checkRemaining);
    114     newProducedItems->addIncoming(fileSize, setTermination);
     138    newProducedItems->addIncoming(fileItems, terminationExit);
    115139    b->setProducedItemCount("sourceBuffer", newProducedItems);
    116140}
    117141
    118 void MMapSourceKernel::unmapSourceBuffer(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
    119     Value * fileSize = b->getBufferedSize("sourceBuffer");
    120     if (LLVM_UNLIKELY(codeUnitWidth > 8)) {
    121         fileSize = b->CreateMul(fileSize, b->getSize(codeUnitWidth / 8));
    122     }
    123     b->CreateMUnmap(b->getBaseAddress("sourceBuffer"), fileSize);
     142void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
     143    b->CreateFree(b->getScalarField("buffer"));
    124144}
    125145
     
    257277    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), finalizeMMap, finalizeRead);
    258278    b->SetInsertPoint(finalizeMMap);
    259     MMapSourceKernel::unmapSourceBuffer(mCodeUnitWidth, b);
     279    MMapSourceKernel::freeBuffer(b);
    260280    b->CreateBr(finalizeDone);
    261281    b->SetInsertPoint(finalizeRead);
     
    316336    b->setBaseAddress("sourceBuffer", fileSource);
    317337    Value * const fileSize = b->getScalarField("fileSize");
    318     Value * const fileItems = b->CreateUDiv(fileSize, b->getSize(mCodeUnitWidth / 8));
    319     b->setBufferedSize("sourceBuffer", fileItems);
    320     b->setCapacity("sourceBuffer", fileItems);
    321     b->setProducedItemCount("sourceBuffer", fileItems);
    322     b->setTerminationSignal();
     338    b->setBufferedSize("sourceBuffer", fileSize);
     339    b->setCapacity("sourceBuffer", fileSize);
     340    if (mStreamSetCount > 1) {
     341        b->setProducedItemCount("sourceBuffer", fileSize);
     342        b->setTerminationSignal();
     343    }
    323344}
    324345
    325346void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
    326 
     347    if (mStreamSetCount == 1) {
     348        Constant * const PAGE_SIZE = b->getSize(getStride());
     349        Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
     350
     351        BasicBlock * const entry = b->GetInsertBlock();
     352        BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary");
     353        BasicBlock * const exit = b->CreateBasicBlock("exit");
     354
     355        Value * const fileItems = b->getScalarField("fileSize");
     356        Value * const producedItems = b->getProducedItemCount("sourceBuffer");
     357        Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE);
     358        Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
     359        b->CreateUnlikelyCondBr(lastPage, createTemporary, exit);
     360
     361        b->SetInsertPoint(createTemporary);
     362        Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
     363        Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
     364        Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
     365        Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
     366        Value * const unconsumedBytes = b->CreatePtrDiff(readEnd, readStart);
     367        Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
     368        Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
     369        b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
     370        b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
     371
     372        // get the difference between our base and from position then compute an offsetted temporary buffer address
     373        Value * const base = b->getBaseAddress("sourceBuffer");
     374        Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
     375        Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
     376        b->CreateConsumerWait();
     377
     378        // set the temporary buffer as the new source buffer
     379        PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
     380        b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
     381        b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
     382        b->setTerminationSignal();
     383        BasicBlock * const terminationExit = b->GetInsertBlock();
     384        b->CreateBr(exit);
     385
     386        b->SetInsertPoint(exit);
     387        PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
     388        newProducedItems->addIncoming(nextProducedItems, entry);
     389        newProducedItems->addIncoming(fileItems, terminationExit);
     390        b->setProducedItemCount("sourceBuffer", newProducedItems);
     391    }
     392}
     393
     394void MemorySourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
     395    if (mStreamSetCount == 1) {
     396        b->CreateFree(b->getScalarField("buffer"));
     397    }
    327398}
    328399
     
    337408, mFileSizeFunction(nullptr) {
    338409    addAttribute(MustExplicitlyTerminate());
    339     setStride((8 * getpagesize()) / codeUnitWidth);
     410    setStride(getpagesize());
    340411}
    341412
     
    349420, mCodeUnitWidth(codeUnitWidth) {
    350421    addAttribute(MustExplicitlyTerminate());
    351     setStride((8 * getpagesize()) / codeUnitWidth);
     422    setStride(getpagesize());
    352423}
    353424
     
    356427: SegmentOrientedKernel("FD_source@" + std::to_string(codeUnitWidth)
    357428, {}
    358 // output
     429// output stream
    359430, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
    360431// input scalar
    361432, {Binding{b->getInt8Ty(), "useMMap"}, Binding{b->getInt32Ty(), "fileDescriptor"}}
    362433, {}
     434// internal scalar
    363435, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}, Binding{b->getSizeTy(), "fileSize"}})
    364436, mCodeUnitWidth(codeUnitWidth)
    365437, mFileSizeFunction(nullptr) {
    366438    addAttribute(MustExplicitlyTerminate());
    367     setStride((8 * getpagesize()) / codeUnitWidth);
    368 }
    369 
    370 MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Type * const type, const unsigned codeUnitWidth)
    371 : SegmentOrientedKernel("memory_source",
    372     {},
    373     {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}},
    374     {Binding{cast<PointerType>(type), "fileSource"}, Binding{b->getSizeTy(), "fileSize"}}, {}, {})
     439    setStride(getpagesize());
     440}
     441
     442MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned streamSetCount, const unsigned codeUnitWidth)
     443: SegmentOrientedKernel("memory_source@" + std::to_string(streamSetCount) + ":" + std::to_string(codeUnitWidth),
     444{},
     445// output stream
     446{Binding{b->getStreamSetTy(streamSetCount, codeUnitWidth), "sourceBuffer"}},
     447// input scalar
     448{Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "fileSource"}, Binding{b->getSizeTy(), "fileSize"}},
     449{},
     450// internal scalar
     451{Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
     452, mStreamSetCount(streamSetCount)
    375453, mCodeUnitWidth(codeUnitWidth) {
    376454    addAttribute(MustExplicitlyTerminate());
    377 }
    378 
    379 }
     455    setStride(getpagesize());
     456}
     457
     458}
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.h

    r5985 r6008  
    2727    }
    2828    void generateFinalizeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override {
    29         unmapSourceBuffer(mCodeUnitWidth, iBuilder);
     29        freeBuffer(iBuilder);
    3030    }
    3131protected:
     
    3333    static void generateInitializeMethod(llvm::Function * fileSize, const unsigned codeUnitWidth, const std::unique_ptr<kernel::KernelBuilder> & b);
    3434    static void generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<kernel::KernelBuilder> & b);
    35     static void unmapSourceBuffer(const unsigned codeUnitWidth, const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     35    static void freeBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
    3636protected:
    3737    const unsigned mCodeUnitWidth;
     
    7878class MemorySourceKernel final : public SegmentOrientedKernel {
    7979public:
    80     MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, llvm::Type * const type, const unsigned codeUnitWidth = 8);
     80    MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const unsigned streamSetCount = 1, const unsigned codeUnitWidth = 8);
    8181    bool hasSignature() const override { return false; }
    8282protected:
    8383    void generateInitializeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
    8484    void generateDoSegmentMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
     85    void generateFinalizeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
    8586private:
     87    const unsigned mStreamSetCount;
    8688    const unsigned mCodeUnitWidth;
    8789};
  • icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.cpp

    r5985 r6008  
    203203    BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks());
    204204
    205     kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder, iBuilder->getInt8PtrTy());
     205    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
    206206    sourceK->setInitialArguments({inputStream, fileSize});
    207207    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
  • icGREP/icgrep-devel/icgrep/lz4_grep.cpp

    r5957 r6008  
    8989    //char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
    9090    char *fileBuffer = const_cast<char *>(mappedFile.data());
    91 
    92     if (codegen::SegmentSize < 2) {
    93         codegen::SegmentSize = 2;
    94     }
    95 
    96 
    9791    re::RE * re_ast = re::RE_Parser::parse(regexString, re::MULTILINE_MODE_FLAG);
    98 
    99 
    100 
    10192    LZ4GrepGenerator g;
    102 
    10393    if (countOnly) {
    10494        g.generateCountOnlyGrepPipeline(re_ast);
  • icGREP/icgrep-devel/icgrep/lz4d.cpp

    r5864 r6008  
    8282
    8383   
    84     kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder, iBuilder->getInt8PtrTy());
     84    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
    8585    sourceK->setInitialArguments({inputStream, fileSize});
    8686    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
     
    141141    mappedFile.open(fileName, lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
    142142    char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
    143 
    144     if (codegen::SegmentSize < 2) {
    145         codegen::SegmentSize = 2;
    146     }
    147 
    148143    ParabixDriver pxDriver("lz4d");
    149144    generatePipeline(pxDriver);
  • icGREP/icgrep-devel/icgrep/lz4d_ext_dep.cpp

    r5985 r6008  
    7474    //char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
    7575    char *fileBuffer = const_cast<char *>(mappedFile.data());
    76 
    77     if (codegen::SegmentSize < 2) {
    78         codegen::SegmentSize = 2;
    79     }
    80 
    8176    LZ4Generator g;
    8277    if (extractOnly) {
  • icGREP/icgrep-devel/icgrep/toolchain/pipeline.cpp

    r6005 r6008  
    4545    , terminated(nullptr)
    4646    , noMore(nullptr)
    47     , requiresTemporaryBuffers(nullptr)
    4847    , deadLockCounter(nullptr)
    4948    , anyProgress(nullptr)
     
    8584    ChannelGraph pruneGraph(ChannelGraph && G, VertexList && V) const;
    8685
    87     bool isPotentiallyUnsafeInputBuffer(const StreamSetBuffer * const buffer);
    88 
    89     void allocateTemporaryBufferPointerArray(const std::unique_ptr<KernelBuilder> & b, const Kernel * const kernel);
    90 
    9186    void checkIfAllInputKernelsHaveFinished(const std::unique_ptr<KernelBuilder> & b, const unsigned index);
    9287
     
    10297
    10398    void runKernel(const std::unique_ptr<KernelBuilder> & b, const Kernel * const kernel);
    104 
    105     void allocateTemporaryBuffers(const std::unique_ptr<KernelBuilder> & b, const Kernel * kernel);
    106 
    107     void freeTemporaryBuffers(const std::unique_ptr<KernelBuilder> & b, const Kernel * kernel);
    10899
    109100    Value * getFullyProcessedItemCount(const std::unique_ptr<KernelBuilder> & b, const Binding & binding, Value * const final) const;
     
    119110
    120111    Value *                             noMore;
    121     Value *                             requiresTemporaryBuffers;
    122112
    123113    DependencyGraph                     dependencyGraph;
    124114    ChannelGraph                        inputGraph;
    125115    ChannelGraph                        outputGraph;
    126 
    127     std::vector<Value *>                temporaryBufferPtrs;
    128116
    129117    BasicBlock *                        kernelFinished;
     
    727715    // TODO: add in some checks to verify the kernel actually adheres to its rate
    728716
     717
    729718    BasicBlock * const kernelCodeExit = b->GetInsertBlock();
    730719    terminated->addIncoming(finalStride, kernelCodeExit);
     
    767756            producedItemCount.emplace(buffer, produced);
    768757        }
    769 
    770     }
    771 
     758    }
    772759
    773760    // If this kernel is the last consumer of a input buffer, update the consumed count for that buffer.
     
    821808    const Kernel * const kernel = kernels[index];
    822809    b->setKernel(kernel);
    823     requiresTemporaryBuffers = nullptr;
    824810    for (auto e : make_iterator_range(in_edges(index, inputGraph))) {
    825811        const Channel & c = inputGraph[e];
    826812        const Binding & input = kernel->getStreamInput(c.operand);
    827 
    828813        Value * requiredInput = getStrideLength(b, kernel, input);
    829814        if (LLVM_UNLIKELY(input.hasLookahead())) {
     
    846831        b->CreateLikelyCondBr(check, hasSufficientInput, kernelFinished);
    847832        b->SetInsertPoint(hasSufficientInput);
    848         if (isPotentiallyUnsafeInputBuffer(c.buffer)) {
    849             Value * const notEnough = b->CreateNot(hasEnough);
    850             if (requiresTemporaryBuffers) {
    851                 requiresTemporaryBuffers = b->CreateOr(requiresTemporaryBuffers, notEnough);
    852             } else {
    853                 requiresTemporaryBuffers = notEnough;
    854             }
    855         }
    856833    }
    857834}
     
    867844        assert (c.buffer->getProducer() == kernel);
    868845        const Binding & output = kernel->getStreamOutput(c.buffer);
    869 
    870846        Value * requiredSpace = getStrideLength(b, kernel, output);
    871847        const auto & name = output.getName();
     
    944920    }
    945921    #endif
    946 
    947     allocateTemporaryBufferPointerArray(b, kernel);
    948922
    949923    runKernel(b, kernel);
     
    1014988    arguments[1] = isFinal;
    1015989
    1016     if (requiresTemporaryBuffers) {
    1017         allocateTemporaryBuffers(b, kernel);
    1018     }
    1019 
    1020990    b->createDoSegmentCall(arguments);
    1021 
    1022     if (requiresTemporaryBuffers) {
    1023         freeTemporaryBuffers(b, kernel);
    1024     }
    1025991}
    1026992
     
    10621028        }
    10631029    }
    1064 }
    1065 
    1066 /** ------------------------------------------------------------------------------------------------------------- *
    1067  * @brief allocateTemporaryBufferPointerArray
    1068  ** ------------------------------------------------------------------------------------------------------------- */
    1069 void PipelineGenerator::allocateTemporaryBufferPointerArray(const std::unique_ptr<KernelBuilder> & b, const Kernel * const kernel) {
    1070     // TODO: whenever two kernels are using the same "unsafe" buffer, they'll both create and destroy their own
    1071     // temporary copies of it. This could be optimized to have it done at production and deleted after the last
    1072     // consuming kernel utilizes it.
    1073     if (requiresTemporaryBuffers) {
    1074         const auto & inputs = kernel->getStreamInputs();
    1075         temporaryBufferPtrs.resize(inputs.size());
    1076         std::fill(temporaryBufferPtrs.begin(), temporaryBufferPtrs.end(), nullptr);
    1077         for (unsigned i = 0; i < inputs.size(); ++i) {
    1078             const StreamSetBuffer * const buffer = kernel->getStreamSetInputBuffer(i);
    1079             if (LLVM_UNLIKELY(isPotentiallyUnsafeInputBuffer(buffer))) {
    1080                 assert (temporaryBufferPtrs[i] == nullptr);
    1081                 PointerType * const ptrTy = buffer->getStreamSetPointerType();
    1082                 StructType * const structTy = StructType::create(b->getContext(), {ptrTy, ptrTy});
    1083                 AllocaInst * const tempBuffer = b->CreateAlloca(structTy);
    1084                 b->CreateStore(Constant::getNullValue(structTy), tempBuffer);
    1085                 temporaryBufferPtrs[i] = tempBuffer;
    1086             }
    1087         }
    1088     }
    1089 }
    1090 
    1091 /** ------------------------------------------------------------------------------------------------------------- *
    1092  * @brief isPotentiallyUnsafeInputBuffer
    1093  *
    1094  * We cannot trust that the final block of any single stream source or external buffer can be safely read past its
    1095  * final item since kernels may attempt to load aligned blocks of data, leading to potentially-intermittent
    1096  * segmentation faults, depending on whether the access crosses a page boundary.
    1097  ** ------------------------------------------------------------------------------------------------------------- */
    1098 inline bool PipelineGenerator::isPotentiallyUnsafeInputBuffer(const StreamSetBuffer * const buffer) {
    1099     return isa<SourceBuffer>(buffer) && buffer->getNumOfStreams() == 1;
    1100 }
    1101 
    1102 /** ------------------------------------------------------------------------------------------------------------- *
    1103  * @brief allocateTemporaryBuffers
    1104  ** ------------------------------------------------------------------------------------------------------------- */
    1105 void PipelineGenerator::allocateTemporaryBuffers(const std::unique_ptr<KernelBuilder> & b, const Kernel * kernel) {
    1106     ConstantInt * const ZERO = b->getInt32(0);
    1107     ConstantInt * const ONE = b->getInt32(1);
    1108     BasicBlock * const allocateBuffers = b->CreateBasicBlock();
    1109     BasicBlock * const runKernel = b->CreateBasicBlock();
    1110     b->CreateUnlikelyCondBr(requiresTemporaryBuffers, allocateBuffers, runKernel);
    1111 
    1112     b->SetInsertPoint(allocateBuffers);
    1113     for (unsigned i = 0; i < temporaryBufferPtrs.size(); ++i) {
    1114         if (temporaryBufferPtrs[i]) {
    1115             const Binding & input = kernel->getStreamInput(i);
    1116             const auto p = producedItemCount.find(kernel->getStreamSetInputBuffer(i));
    1117             assert (p != producedItemCount.end());
    1118             Value * const produced = p->second;
    1119             Value * const processed = b->getProcessedItemCount(input.getName());
    1120             Value * const unprocessed = b->CreateSub(produced, processed);
    1121             const auto temp = b->AcquireTemporaryBuffer(input.getName(), processed, unprocessed);
    1122             b->CreateStore(temp.first, b->CreateGEP(temporaryBufferPtrs[i], { ZERO, ZERO }));
    1123             b->CreateStore(temp.second, b->CreateGEP(temporaryBufferPtrs[i], { ZERO, ONE }));
    1124         }
    1125     }
    1126     b->CreateBr(runKernel);
    1127 
    1128     b->SetInsertPoint(runKernel);
    1129 }
    1130 
    1131 /** ------------------------------------------------------------------------------------------------------------- *
    1132  * @brief freeTemporaryBuffers
    1133  ** ------------------------------------------------------------------------------------------------------------- */
    1134 void PipelineGenerator::freeTemporaryBuffers(const std::unique_ptr<KernelBuilder> & b, const Kernel * kernel) {
    1135     ConstantInt * const ZERO = b->getInt32(0);
    1136     ConstantInt * const ONE = b->getInt32(1);
    1137 
    1138     BasicBlock * const freeBuffers = b->CreateBasicBlock();
    1139     BasicBlock * const finishedKernel = b->CreateBasicBlock();
    1140     b->CreateUnlikelyCondBr(requiresTemporaryBuffers, freeBuffers, finishedKernel);
    1141     b->SetInsertPoint(freeBuffers);
    1142     for (unsigned i = 0; i < temporaryBufferPtrs.size(); ++i) {
    1143         if (temporaryBufferPtrs[i]) {
    1144             Value * const originalBuffer = b->CreateLoad(b->CreateGEP(temporaryBufferPtrs[i], { ZERO, ZERO }));
    1145             const Binding & input = kernel->getStreamInput(i);
    1146             b->setBaseAddress(input.getName(), originalBuffer);
    1147             Value * const temporaryBuffer = b->CreateLoad(b->CreateGEP(temporaryBufferPtrs[i], { ZERO, ONE }));
    1148             b->CreateFree(temporaryBuffer);
    1149         }
    1150     }
    1151     b->CreateBr(finishedKernel);
    1152 
    1153     b->SetInsertPoint(finishedKernel);
    11541030}
    11551031
Note: See TracChangeset for help on using the changeset viewer.