Ignore:
Timestamp:
Feb 2, 2018, 2:49:08 PM (15 months ago)
Author:
nmedfort
Message:

Revised pipeline structure to better control I/O rates

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

    r5771 r5856  
    99#include <sys/stat.h>
    1010#include <fcntl.h>
     11#include <toolchain/toolchain.h>
    1112
    1213using namespace llvm;
     
    2829}
    2930
    30 void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const unsigned segmentSize, const std::unique_ptr<KernelBuilder> & kb) {
    31     BasicBlock * const emptyFile = kb->CreateBasicBlock("EmptyFile");
     31void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & kb) {
     32
     33    BasicBlock * const emptyFile = kb->CreateBasicBlock("emptyFile");
    3234    BasicBlock * const nonEmptyFile = kb->CreateBasicBlock("NonEmptyFile");
    3335    BasicBlock * const exit = kb->CreateBasicBlock("Exit");
     
    3537    Value * const fd = kb->getScalarField("fileDescriptor");
    3638    assert (fileSizeMethod);
    37     Value * fileSize = kb->CreateCall(fileSizeMethod, fd);
    38     fileSize = kb->CreateZExtOrTrunc(fileSize, sizeTy);
     39    Value * fileSize = kb->CreateZExtOrTrunc(kb->CreateCall(fileSizeMethod, fd), sizeTy);
     40    kb->CreateLikelyCondBr(kb->CreateIsNotNull(fileSize), nonEmptyFile, emptyFile);
     41
     42    kb->SetInsertPoint(nonEmptyFile);
     43    PointerType * const codeUnitPtrTy = kb->getIntNTy(codeUnitWidth)->getPointerTo();
     44    Value * const fileBuffer = kb->CreatePointerCast(kb->CreateFileSourceMMap(fd, fileSize), codeUnitPtrTy);
     45    kb->setScalarField("buffer", fileBuffer);   
     46    kb->setBaseAddress("sourceBuffer", fileBuffer);   
     47    kb->CreateMAdvise(fileBuffer, fileSize, CBuilder::ADVICE_WILLNEED);
    3948    if (codeUnitWidth > 8) {
    4049        fileSize = kb->CreateUDiv(fileSize, kb->getSize(codeUnitWidth / 8));
    4150    }
    42     Value * const isEmpty = kb->CreateICmpEQ(fileSize, ConstantInt::getNullValue(fileSize->getType()));
    43     kb->CreateUnlikelyCondBr(isEmpty, emptyFile, nonEmptyFile);
    44     // we cannot mmap a 0 length file; just create a 1-page sized fake file buffer for simplicity
     51    kb->setBufferedSize("sourceBuffer", fileSize);
     52    kb->setScalarField("fileSize", fileSize);
     53    kb->setProducedItemCount("sourceBuffer", fileSize);
     54    kb->setCapacity("sourceBuffer", fileSize);
     55    kb->CreateBr(exit);
     56
    4557    kb->SetInsertPoint(emptyFile);
    46     Constant * const readSize = kb->getSize(segmentSize);
    47     Value * const fakeFileBuffer = kb->CreateAnonymousMMap(readSize);
     58    kb->setTerminationSignal();
    4859    kb->CreateBr(exit);
    4960
    50     kb->SetInsertPoint(nonEmptyFile);
    51     Value * fileBackedBuffer = kb->CreateFileSourceMMap(fd, fileSize);
    52     kb->CreateBr(exit);
    53 
    5461    kb->SetInsertPoint(exit);
    55     PHINode * const buffer = kb->CreatePHI(fileBackedBuffer->getType(), 2);
    56     buffer->addIncoming(fakeFileBuffer, emptyFile);
    57     buffer->addIncoming(fileBackedBuffer, nonEmptyFile);
    58     PHINode * const bufferSize = kb->CreatePHI(sizeTy, 2);
    59     bufferSize->addIncoming(readSize, emptyFile);
    60     bufferSize->addIncoming(fileSize, nonEmptyFile);
    61 
    62     PointerType * const codeUnitPtrTy = kb->getIntNTy(codeUnitWidth)->getPointerTo();
    63     Value * bufferPtr = kb->CreatePointerCast(buffer, codeUnitPtrTy);
    64     kb->setScalarField("buffer", bufferPtr);
    65     kb->setScalarField("fileSize", fileSize);
    66 
    67     kb->setBaseAddress("sourceBuffer", bufferPtr);
    68     kb->setBufferedSize("sourceBuffer", bufferSize);
    69     kb->setCapacity("sourceBuffer", fileSize);
    70     kb->CreateMAdvise(buffer, fileSize, CBuilder::ADVICE_WILLNEED);
    71 
    72 }
    73 
    74 void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned segmentSize, const std::unique_ptr<KernelBuilder> & kb) {
     62}
     63
     64void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & kb) {
    7565
    7666    BasicBlock * dropPages = kb->CreateBasicBlock("dropPages");
    77     BasicBlock * processSegment = kb->CreateBasicBlock("produceData");
     67    BasicBlock * checkRemaining = kb->CreateBasicBlock("checkRemaining");
    7868    BasicBlock * setTermination = kb->CreateBasicBlock("setTermination");
    79     BasicBlock * mmapSourceExit = kb->CreateBasicBlock("mmapSourceExit");
    80 
    81     Constant * const readSize = kb->getSize(segmentSize);
     69    BasicBlock * exit = kb->CreateBasicBlock("mmapSourceExit");
     70
     71    Value * const fileSize = kb->getScalarField("fileSize");
    8272    Constant * const pageSize = kb->getSize(getpagesize());
    8373
     
    8575    consumed = kb->CreateMul(consumed, kb->getSize(codeUnitWidth / 8));
    8676    consumed = kb->CreateAnd(consumed, ConstantExpr::getNeg(pageSize));
    87 
    8877    Value * const consumedBuffer = kb->getRawOutputPointer("sourceBuffer", consumed);
    8978    Value * const readableBuffer = kb->getScalarField("buffer");
     
    9180
    9281    // avoid calling madvise unless an actual page table change could occur
    93     kb->CreateLikelyCondBr(kb->CreateIsNotNull(unnecessaryBytes), processSegment, dropPages);
     82    kb->CreateLikelyCondBr(kb->CreateIsNotNull(unnecessaryBytes), dropPages, checkRemaining);
    9483
    9584    kb->SetInsertPoint(dropPages);
     
    9786    kb->CreateMAdvise(readableBuffer, unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
    9887    kb->setScalarField("buffer", kb->CreateGEP(readableBuffer, unnecessaryBytes));
    99     kb->CreateBr(processSegment);
     88    kb->CreateBr(checkRemaining);
    10089
    10190    // determine whether or not we've exhausted the file buffer
    102     kb->SetInsertPoint(processSegment);
    103     Value * const fileSize = kb->getScalarField("fileSize");
    104     Value * const produced = kb->CreateAdd(kb->getProducedItemCount("sourceBuffer"), readSize);
    105     Value * const lessThanFullSegment = kb->CreateICmpULT(fileSize, produced);
    106     kb->CreateUnlikelyCondBr(lessThanFullSegment, setTermination, mmapSourceExit);
     91    kb->SetInsertPoint(checkRemaining);
     92    Value * const remaining = kb->CreateSub(fileSize, consumed);
     93    Value * const lastPage = kb->CreateICmpULE(remaining, pageSize);
     94    kb->CreateUnlikelyCondBr(lastPage, setTermination, exit);
    10795
    10896    kb->SetInsertPoint(setTermination);
    10997    kb->setTerminationSignal();
    110     kb->CreateBr(mmapSourceExit);
     98    kb->CreateBr(exit);
    11199
    112100    // finally, set the "produced" count to reflect current position in the file
    113     kb->SetInsertPoint(mmapSourceExit);
    114     PHINode * itemsRead = kb->CreatePHI(produced->getType(), 2);
    115     itemsRead->addIncoming(produced, processSegment);
    116     itemsRead->addIncoming(fileSize, setTermination);
    117     kb->setProducedItemCount("sourceBuffer", itemsRead);
    118 
     101    kb->SetInsertPoint(exit);
    119102}
    120103
     
    123106}
    124107
    125 MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, const unsigned stridesPerSegment, const unsigned codeUnitWidth)
    126 : SegmentOrientedKernel("mmap_source" + std::to_string(stridesPerSegment) + "@" + std::to_string(codeUnitWidth),
    127 {},
    128 {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer", FixedRate(), Deferred()}},
    129 {Binding{kb->getInt32Ty(), "fileDescriptor"}},
    130 {Binding{kb->getSizeTy(), "fileSize"}}, {Binding{kb->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
    131 , mStridesPerSegment(stridesPerSegment)
    132 , mCodeUnitWidth(codeUnitWidth)
    133 , mFileSizeFunction(nullptr) {
    134 
    135 }
    136 
    137108/// READ SOURCE KERNEL
    138109
    139 void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const unsigned segmentSize, const std::unique_ptr<KernelBuilder> & b) {
     110void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
    140111    const unsigned pageSize = getpagesize();
    141     const auto bufferSize = std::max(pageSize * 8, segmentSize * 4);
     112    const auto bufferSize = std::max(pageSize * 8, codegen::SegmentSize * stride * 4);
    142113    ConstantInt * const bufferItems = b->getSize(bufferSize);
    143114    const auto codeUnitSize = codeUnitWidth / 8;
     
    150121}
    151122
    152 void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned segmentSize, const std::unique_ptr<KernelBuilder> & b) {
     123void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
    153124
    154125    const unsigned pageSize = getpagesize();
    155     ConstantInt * const itemsToRead = b->getSize(std::max(pageSize, segmentSize * 2));
     126    ConstantInt * const itemsToRead = b->getSize(std::max(pageSize, codegen::SegmentSize * stride * 2));
    156127    ConstantInt * const codeUnitBytes = b->getSize(codeUnitWidth / 8);
    157     ConstantInt * const itemsPerSegment = b->getSize(segmentSize);
     128    ConstantInt * const itemsPerSegment = b->getSize(codegen::SegmentSize * stride);
    158129
    159130    BasicBlock * const entry = b->GetInsertBlock();
     
    213184    b->CreateMemCpy(expandedBuffer, unreadData, remainingBytes, blockSize);
    214185    b->setScalarField("buffer", expandedBuffer);
    215     b->setCapacity("sourceBuffer", expandedCapacity); 
     186    b->setCapacity("sourceBuffer", expandedCapacity);
    216187    b->CreateFree(baseBuffer);
    217188    b->CreateBr(prepareBuffer);
     
    236207    b->CreateUnlikelyCondBr(b->CreateICmpULT(itemsBuffered, itemsPending), setTermination, readExit);
    237208
    238     // ... set the termination signal.   
     209    // ... set the termination signal.
    239210    b->SetInsertPoint(setTermination);
    240211    Value * const bytesToZero = b->CreateMul(b->CreateSub(itemsPending, itemsBuffered), codeUnitBytes);
     
    254225void ReadSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & kb) {
    255226    kb->CreateFree(kb->getScalarField("buffer"));
    256 }
    257 
    258 ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned stridesPerSegment, const unsigned codeUnitWidth)
    259 : SegmentOrientedKernel("read_source"  + std::to_string(stridesPerSegment) + "@" + std::to_string(codeUnitWidth)
    260 , {}
    261 , {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer", FixedRate(), Deferred()}}
    262 , {Binding{b->getInt32Ty(), "fileDescriptor"}}
    263 , {}
    264 , {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
    265 , mStridesPerSegment(stridesPerSegment)
    266 , mCodeUnitWidth(codeUnitWidth) {
    267 
    268227}
    269228
     
    296255    kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(STDIN_FILENO)), initializeRead, initializeMMap);
    297256    kb->SetInsertPoint(initializeRead);
    298     ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, mStride * mStridesPerSegment, kb);
     257    ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, getStride(), kb);
    299258    kb->CreateBr(initializeDone);
    300259    kb->SetInsertPoint(initializeMMap);
    301     MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, mStride * mStridesPerSegment, kb);
     260    MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, kb);
    302261    kb->CreateBr(initializeDone);
    303262    kb->SetInsertPoint(initializeDone);
     
    311270    kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(STDIN_FILENO)), DoSegmentRead, DoSegmentMMap);
    312271    kb->SetInsertPoint(DoSegmentRead);
    313     ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride * mStridesPerSegment, kb);
     272    ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, getStride(), kb);
    314273    kb->CreateBr(DoSegmentDone);
    315274    kb->SetInsertPoint(DoSegmentMMap);
    316     MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride * mStridesPerSegment, kb);
     275    MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, kb);
    317276    kb->CreateBr(DoSegmentDone);
    318277    kb->SetInsertPoint(DoSegmentDone);
    319278}
    320279
    321 FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, const unsigned stridesPerSegment, const unsigned codeUnitWidth)
    322 : SegmentOrientedKernel("FD_source" + std::to_string(stridesPerSegment) + "@" + std::to_string(codeUnitWidth)
    323 , {}
    324 , {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer", FixedRate(), Deferred()}}
    325 , {Binding{kb->getInt32Ty(), "fileDescriptor"}}
    326 , {}
    327 , {Binding{kb->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}, Binding{kb->getSizeTy(), "fileSize"}})
    328 , mStridesPerSegment(stridesPerSegment)
    329 , mCodeUnitWidth(codeUnitWidth)
    330 , mFileSizeFunction(nullptr) {
    331 
    332 }
    333280
    334281/// MEMORY SOURCE KERNEL
     
    341288    kb->setBufferedSize("sourceBuffer", fileItems);
    342289    kb->setCapacity("sourceBuffer", fileItems);
     290    kb->setProducedItemCount("sourceBuffer", fileItems);
     291    kb->setTerminationSignal();
    343292}
    344293
    345294void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & kb) {
    346295
    347     BasicBlock * entryBlock = kb->GetInsertBlock();
    348     BasicBlock * setTermination = kb->CreateBasicBlock("setTermination");
    349     BasicBlock * mmapSourceExit = kb->CreateBasicBlock("sourceExit");
    350 
    351     ConstantInt * const segmentItems = kb->getSize(mStride * mStridesPerSegment);
    352     Value * const fileItems = kb->getBufferedSize("sourceBuffer");
    353     Value * const produced = kb->CreateAdd(kb->getProducedItemCount("sourceBuffer"), segmentItems);
    354 
    355     kb->CreateUnlikelyCondBr(kb->CreateICmpULT(fileItems, produced), setTermination, mmapSourceExit);
    356 
    357     kb->SetInsertPoint(setTermination);
    358     kb->setTerminationSignal();
    359     kb->CreateBr(mmapSourceExit);
    360 
    361     kb->SetInsertPoint(mmapSourceExit);
    362     PHINode * itemsRead = kb->CreatePHI(produced->getType(), 2);
    363     itemsRead->addIncoming(produced, entryBlock);
    364     itemsRead->addIncoming(fileItems, setTermination);
    365     kb->setProducedItemCount("sourceBuffer", itemsRead);
    366 }
    367 
    368 MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, Type * const type, const unsigned stridesPerSegment, const unsigned codeUnitWidth)
     296}
     297
     298MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
     299: SegmentOrientedKernel("mmap_source@" + std::to_string(codeUnitWidth),
     300{},
     301{Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer", FixedRate(), Deferred()}},
     302{Binding{b->getInt32Ty(), "fileDescriptor"}},
     303{Binding{b->getSizeTy(), "fileSize"}}, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
     304, mCodeUnitWidth(codeUnitWidth)
     305, mFileSizeFunction(nullptr) {
     306    addAttribute(CanTerminateEarly());
     307}
     308
     309
     310ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
     311: SegmentOrientedKernel("read_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(codeUnitWidth)
     312, {}
     313, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer", FixedRate(), Deferred()}}
     314, {Binding{b->getInt32Ty(), "fileDescriptor"}}
     315, {}
     316, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
     317, mCodeUnitWidth(codeUnitWidth) {
     318    addAttribute(CanTerminateEarly());
     319}
     320
     321
     322FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, const unsigned codeUnitWidth)
     323: SegmentOrientedKernel("FD_source@" + std::to_string(codeUnitWidth)
     324, {}
     325, {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer", FixedRate(), Deferred()}}
     326, {Binding{kb->getInt32Ty(), "fileDescriptor"}}
     327, {}
     328, {Binding{kb->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}, Binding{kb->getSizeTy(), "fileSize"}})
     329, mCodeUnitWidth(codeUnitWidth)
     330, mFileSizeFunction(nullptr) {
     331    addAttribute(CanTerminateEarly());
     332}
     333
     334MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, Type * const type, const unsigned codeUnitWidth)
    369335: SegmentOrientedKernel("memory_source",
    370336    {},
    371337    {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}},
    372338    {Binding{cast<PointerType>(type), "fileSource"}, Binding{kb->getSizeTy(), "fileSize"}}, {}, {})
    373 , mStridesPerSegment(stridesPerSegment)
    374339, mCodeUnitWidth(codeUnitWidth) {
    375 
    376 }
    377 
    378 }
     340    addAttribute(CanTerminateEarly());
     341}
     342
     343}
Note: See TracChangeset for help on using the changeset viewer.