Ignore:
Timestamp:
Dec 6, 2017, 5:41:27 PM (17 months ago)
Author:
nmedfort
Message:

Bug fixes + more assertions to prevent similar errors.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

    r5755 r5757  
    1212using namespace llvm;
    1313
    14 inline static size_t round_up_to_nearest(const size_t x, const size_t y) {
    15     return (((x - 1) | (y - 1)) + 1);
    16 }
    17 
    1814uint64_t file_size(const uint32_t fd) {
    1915    struct stat st;
     
    3228}
    3329
    34 void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & kb) {
     30void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const unsigned /* blocksRequiredPerSegment */, const std::unique_ptr<KernelBuilder> & kb) {
    3531    BasicBlock * const emptyFile = kb->CreateBasicBlock("EmptyFile");
    3632    BasicBlock * const nonEmptyFile = kb->CreateBasicBlock("NonEmptyFile");
     
    6359    size->addIncoming(pageSize, emptyFile);
    6460    size->addIncoming(fileSize, nonEmptyFile);
    65     Value * bufferPtr = kb->CreatePointerCast(buffer, kb->getInt8PtrTy());
     61
     62    PointerType * const codeUnitPtrTy = kb->getIntNTy(codeUnitWidth)->getPointerTo();
     63    Value * bufferPtr = kb->CreatePointerCast(buffer, codeUnitPtrTy);
    6664    kb->setBaseAddress("sourceBuffer", bufferPtr);
    6765    kb->setBufferedSize("sourceBuffer", size);
     
    7371}
    7472
    75 void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned segmentBlocks, const std::unique_ptr<KernelBuilder> & kb) {
     73void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned blocksRequiredPerSegment, const std::unique_ptr<KernelBuilder> & kb) {
    7674
    7775    BasicBlock * dropPages = kb->CreateBasicBlock("dropPages");
     
    8078    BasicBlock * mmapSourceExit = kb->CreateBasicBlock("mmapSourceExit");
    8179
     80    Constant * const segmentSize = kb->getSize(blocksRequiredPerSegment * kb->getBitBlockWidth());
     81    Constant * const pageSize = kb->getSize(getpagesize());
     82
     83    Value * consumed = kb->getConsumedItemCount("sourceBuffer");
     84    consumed = kb->CreateMul(consumed, kb->getSize(codeUnitWidth / 8));
     85    consumed = kb->CreateAnd(consumed, ConstantExpr::getNeg(pageSize));
     86
     87    Value * const consumedBuffer = kb->getRawOutputPointer("sourceBuffer", consumed);
     88    Value * const readableBuffer = kb->getScalarField("readableBuffer");
     89    Value * const unnecessaryBytes = kb->CreatePtrDiff(consumedBuffer, readableBuffer);
     90
     91    // avoid calling madvise unless an actual page table change could occur
     92    kb->CreateLikelyCondBr(kb->CreateIsNotNull(unnecessaryBytes), processSegment, dropPages);
     93
     94    kb->SetInsertPoint(dropPages);
    8295    // instruct the OS that it can safely drop any fully consumed pages
    83     Value * consumed = kb->getConsumedItemCount("sourceBuffer");
    84     IntegerType * const consumedTy = cast<IntegerType>(consumed->getType());
    85     Type * const int8PtrTy = kb->getInt8PtrTy();
    86 
    87     DataLayout DL(kb->getModule());
    88     IntegerType * const intAddrTy = kb->getIntPtrTy(DL);
    89 
    90     // multiply the consumed count by the code unit size then mask off any partial pages
    91     if (codeUnitWidth > 8) {
    92         consumed = kb->CreateMul(consumed, ConstantInt::get(consumedTy, codeUnitWidth / 8));
    93     }
    94     const auto pageSize = getpagesize();
    95     if (LLVM_LIKELY((pageSize & (pageSize - 1)) == 0)) {
    96         consumed = kb->CreateAnd(consumed, ConstantExpr::getNeg(ConstantInt::get(consumedTy, pageSize)));
    97     } else {
    98         consumed = kb->CreateSub(consumed, kb->CreateURem(consumed, ConstantInt::get(consumedTy, pageSize)));
    99     }
    100 
    101     Value * sourceBuffer = kb->getBaseAddress("sourceBuffer");
    102     sourceBuffer = kb->CreatePtrToInt(sourceBuffer, intAddrTy);
    103     if (LLVM_UNLIKELY(intAddrTy->getBitWidth() > consumedTy->getBitWidth())) {
    104         consumed = kb->CreateZExt(consumed, intAddrTy);
    105     } else if (LLVM_UNLIKELY(intAddrTy->getBitWidth() < consumedTy->getBitWidth())) {
    106         sourceBuffer = kb->CreateZExt(sourceBuffer, consumedTy);
    107     }
    108     Value * consumedBuffer = kb->CreateAdd(sourceBuffer, consumed);
    109     Value * readableBuffer = kb->getScalarField("readableBuffer");
    110     readableBuffer = kb->CreatePtrToInt(readableBuffer, consumedBuffer->getType());
    111     Value * unnecessaryBytes = kb->CreateSub(consumedBuffer, readableBuffer);
    112 
    113     // avoid calling madvise unless an actual page table change could occur
    114     Value * hasPagesToDrop = kb->CreateICmpEQ(unnecessaryBytes, ConstantInt::getNullValue(intAddrTy));
    115     kb->CreateLikelyCondBr(hasPagesToDrop, processSegment, dropPages);
    116 
    117     kb->SetInsertPoint(dropPages);
    118     kb->CreateMAdvise(kb->CreateIntToPtr(readableBuffer, int8PtrTy), unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
    119     readableBuffer = kb->CreateIntToPtr(kb->CreateAdd(readableBuffer, unnecessaryBytes), int8PtrTy);
    120     kb->setScalarField("readableBuffer", readableBuffer);
     96    kb->CreateMAdvise(readableBuffer, unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
     97    kb->setScalarField("readableBuffer", kb->CreateGEP(readableBuffer, unnecessaryBytes));
    12198    kb->CreateBr(processSegment);
    12299
    123100    // determine whether or not we've exhausted the file buffer
    124101    kb->SetInsertPoint(processSegment);
    125     ConstantInt * segmentItems = kb->getSize(segmentBlocks * kb->getBitBlockWidth());
    126102    Value * const fileSize = kb->getScalarField("fileSize");
    127     Value * const produced = kb->CreateAdd(kb->getProducedItemCount("sourceBuffer"), segmentItems);
     103    Value * const produced = kb->CreateAdd(kb->getProducedItemCount("sourceBuffer"), segmentSize);
    128104    Value * const lessThanFullSegment = kb->CreateICmpULT(fileSize, produced);
    129105    kb->CreateUnlikelyCondBr(lessThanFullSegment, setTermination, mmapSourceExit);
     
    146122}
    147123
    148 MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, unsigned blocksPerSegment, unsigned codeUnitWidth)
    149 : SegmentOrientedKernel("mmap_source" + std::to_string(blocksPerSegment) + "@" + std::to_string(codeUnitWidth),
     124MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, unsigned blocksRequiredPerSegment, unsigned codeUnitWidth)
     125: SegmentOrientedKernel("mmap_source" + std::to_string(blocksRequiredPerSegment) + "@" + std::to_string(codeUnitWidth),
    150126{},
    151 {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}},
     127{Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer", FixedRate(), Deferred()}},
    152128{Binding{kb->getInt32Ty(), "fileDescriptor"}},
    153 {Binding{kb->getSizeTy(), "fileSize"}}, {Binding{kb->getInt8PtrTy(), "readableBuffer"}})
    154 , mSegmentBlocks(blocksPerSegment)
     129{Binding{kb->getSizeTy(), "fileSize"}}, {Binding{kb->getIntNTy(codeUnitWidth)->getPointerTo(), "readableBuffer"}})
     130, mBlocksRequiredPerSegment(blocksRequiredPerSegment)
    155131, mCodeUnitWidth(codeUnitWidth)
    156132, mFileSizeFunction(nullptr) {
     
    160136/// READ SOURCE KERNEL
    161137
    162 void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & kb) {
    163     const size_t initialBufferSize = 8 * getpagesize() * codeUnitWidth;
    164     ConstantInt * const bufferBytes = kb->getSize(initialBufferSize * codeUnitWidth / 8);
    165     PointerType * const codeUnitPtrTy = IntegerType::get(kb->getContext(), codeUnitWidth)->getPointerTo();
    166     Value * const buffer = kb->CreatePointerCast(kb->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy);
    167     kb->setScalarField("buffer", buffer);
    168     kb->setScalarField("capacity", kb->getSize(initialBufferSize));
    169     kb->setBaseAddress("sourceBuffer", buffer);
    170     kb->setBufferedSize("sourceBuffer", kb->getSize(0));
    171     kb->setCapacity("sourceBuffer", kb->getSize(initialBufferSize));
    172 }
    173 
    174 void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned segmentBlocks, const std::unique_ptr<KernelBuilder> & kb) {
    175 
    176     ConstantInt * const readSize = kb->getSize(getpagesize() * 8/codeUnitWidth);
    177     PointerType * const codeUnitPtrTy = IntegerType::get(kb->getContext(), codeUnitWidth)->getPointerTo();
    178     PointerType * const i8PtrTy = IntegerType::get(kb->getContext(), 8)->getPointerTo();
    179     ConstantInt * const codeUnitBytes = kb->getSize(codeUnitWidth/8);
    180     BasicBlock * const entryBlock = kb->GetInsertBlock();
    181     BasicBlock * const exhaustedBuffer = kb->CreateBasicBlock("ExhaustedBuffer");
    182     BasicBlock * const waitOnConsumers = kb->CreateBasicBlock("WaitOnConsumers");
    183     BasicBlock * const readData = kb->CreateBasicBlock("ReadData");
    184     BasicBlock * const stdInExit = kb->CreateBasicBlock("StdInExit");
    185 
    186     // Check whether we need to read another page of data
    187     ConstantInt * const segmentSize = kb->getSize(segmentBlocks * kb->getBitBlockWidth());
    188     Value * bufferedSize = kb->getBufferedSize("sourceBuffer");
    189     Value * const produced = kb->getProducedItemCount("sourceBuffer");
    190     Value * unreadSize = kb->CreateSub(bufferedSize, produced);
    191 
    192     kb->CreateUnlikelyCondBr(kb->CreateICmpULT(unreadSize, segmentSize), exhaustedBuffer, stdInExit);
    193 
    194     // If so, it checks whether it can simply append another page to the existing buffer or whether
    195     // we need to perform a copyback.
    196 
    197     kb->SetInsertPoint(exhaustedBuffer);
    198 
    199     // Otherwise, we're going to have to perform a copy back...
    200 
    201     // Let L be the logical buffer address (i.e., the position of the "first code unit" of the input stream)
    202     // and B be the address pointing to the beginning of our actual buffer. Check whether:
    203 
    204     //     L + produced + readSize < B + capacity
    205 
    206     // If so, we can append to our existing buffer without impacting any subsequent kernel.
    207 
    208     Value * inputStream = kb->getRawOutputPointer("sourceBuffer", kb->getInt32(0));
    209     Value * const originalPtr = kb->CreateGEP(inputStream, produced);
    210 
    211     Value * const buffer = kb->getScalarField("buffer");
    212     Value * const capacity = kb->getScalarField("capacity");
    213 
    214     Value * L = kb->CreateGEP(originalPtr, readSize);
    215     Value * B = kb->CreateGEP(buffer, capacity);
    216     Value * const canAppend = kb->CreateICmpULT(L, B);
    217 
    218     kb->CreateLikelyCondBr(canAppend, readData, waitOnConsumers);
     138void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const unsigned blocksRequiredPerSegment, const std::unique_ptr<KernelBuilder> & b) {
     139    const unsigned pageSize = getpagesize();
     140    const unsigned segmentSize = blocksRequiredPerSegment * b->getBitBlockWidth();
     141    const auto bufferSize = std::max(pageSize * 8, segmentSize * 4);
     142    ConstantInt * const bufferItems = b->getSize(bufferSize);
     143    const auto codeUnitSize = codeUnitWidth / 8;
     144    ConstantInt * const bufferBytes = b->getSize(bufferSize * codeUnitSize);
     145    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
     146    Value * const buffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy);
     147    b->setBaseAddress("sourceBuffer", buffer);
     148    b->setScalarField("buffer", buffer);
     149    b->setCapacity("sourceBuffer", bufferItems);
     150}
     151
     152void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned blocksRequiredPerSegment, const std::unique_ptr<KernelBuilder> & b) {
     153
     154    const unsigned pageSize = getpagesize();
     155    const unsigned segmentSize = blocksRequiredPerSegment * b->getBitBlockWidth();
     156    ConstantInt * const itemsToRead = b->getSize(std::max(pageSize, segmentSize * 2));
     157    ConstantInt * const codeUnitBytes = b->getSize(codeUnitWidth / 8);
     158    ConstantInt * const itemsPerSegment = b->getSize(segmentSize);
     159
     160    BasicBlock * const entry = b->GetInsertBlock();
     161    BasicBlock * const checkData = b->CreateBasicBlock("CheckData");
     162    BasicBlock * const moveData = b->CreateBasicBlock("MoveData");
     163    BasicBlock * const prepareBuffer = b->CreateBasicBlock("PrepareBuffer");
     164    BasicBlock * const readData = b->CreateBasicBlock("ReadData");
     165    BasicBlock * const setTermination = b->CreateBasicBlock("SetTermination");
     166    BasicBlock * const readExit = b->CreateBasicBlock("ReadExit");
     167
     168    // Do we have enough unread data to support a segments worth of processing?
     169    Value * const produced = b->getProducedItemCount("sourceBuffer");
     170    Value * const buffered = b->getBufferedSize("sourceBuffer");
     171    Value * const itemsPending = b->CreateAdd(produced, itemsPerSegment);
     172
     173    b->CreateLikelyCondBr(b->CreateICmpULT(itemsPending, buffered), readExit, checkData);
     174
     175    // Can we append to our existing buffer without impacting any subsequent kernel?
     176    b->SetInsertPoint(checkData);
     177    Value * const capacity = b->getCapacity("sourceBuffer");
     178    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", b->CreateAdd(buffered, itemsToRead));
     179    Value * const baseBuffer = b->getScalarField("buffer");
     180    Value * const bufferLimit = b->CreateGEP(baseBuffer, capacity);
     181    b->CreateLikelyCondBr(b->CreateICmpULE(readEnd, bufferLimit), readData, moveData);
    219182
    220183    // First wait on any consumers to finish processing then check how much data has been consumed.
    221     kb->SetInsertPoint(waitOnConsumers);
    222     kb->CreateConsumerWait();
     184    b->SetInsertPoint(moveData);
     185    b->CreateConsumerWait();
    223186
    224187    // Then determine how much data has been consumed and how much needs to be copied back, noting
    225188    // that our "unproduced" data must be block aligned.
    226     const size_t blockAlignment = kb->getBitBlockWidth() / 8;
    227     Constant * const alignmentMask = kb->getSize(-(blockAlignment * 8 / codeUnitWidth));
    228     Value * const consumed = kb->CreateAnd(kb->getConsumedItemCount("sourceBuffer"), alignmentMask);
    229     Value * const remaining = kb->CreateSub(bufferedSize, consumed);
    230     Value * const unconsumedPtr = kb->CreateGEP(inputStream, consumed);
    231     Value * const consumedMajority = kb->CreateICmpULT(kb->CreateGEP(buffer, remaining), unconsumedPtr);
    232     Value * target = buffer;
    233     Value * source = unconsumedPtr;
    234     Value * toCopy = remaining;
    235     if (codeUnitWidth != 8) {
    236         source = kb->CreatePointerCast(unconsumedPtr, i8PtrTy);
    237         toCopy = kb->CreateMul(remaining, codeUnitBytes);
    238     }
    239 
    240     BasicBlock * const copyBack = kb->CreateBasicBlock("CopyBack");
    241     BasicBlock * const expandAndCopyBack = kb->CreateBasicBlock("ExpandAndCopyBack");
    242     BasicBlock * const calculateLogicalAddress = kb->CreateBasicBlock("CalculateLogicalAddress");
    243 
    244     // Have we consumed enough data that we can safely copy back the unconsumed data without needing
    245     // a temporary buffer? (i.e., B + remaining < L + consumed)
    246     kb->CreateLikelyCondBr(consumedMajority, copyBack, expandAndCopyBack);
    247     kb->SetInsertPoint(copyBack);
     189    BasicBlock * const copyBack = b->CreateBasicBlock("CopyBack");
     190    BasicBlock * const expandAndCopyBack = b->CreateBasicBlock("ExpandAndCopyBack");
     191
     192    const auto blockSize = b->getBitBlockWidth() / 8;
     193    Constant * const blockSizeAlignmentMask = ConstantExpr::getNeg(b->getSize(blockSize));
     194    Value * const consumed = b->getConsumedItemCount("sourceBuffer");
     195    Value * const offset = b->CreateAnd(consumed, blockSizeAlignmentMask);
     196    Value * const unreadData = b->getRawOutputPointer("sourceBuffer", offset);
     197    Value * const remainingItems = b->CreateSub(buffered, offset);
     198    Value * const remainingBytes = b->CreateMul(remainingItems, codeUnitBytes);
     199
     200    // Have we consumed enough data that we can safely copy back the unconsumed data without needing a temporary buffer?
     201    Value * const canCopy = b->CreateICmpULT(b->CreateGEP(baseBuffer, remainingItems), b->getRawOutputPointer("sourceBuffer", offset));
     202    b->CreateLikelyCondBr(canCopy, copyBack, expandAndCopyBack);
     203
    248204    // If so, just copy the data ...
    249     if (codeUnitWidth != 8) {
    250         target = kb->CreatePointerCast(buffer, i8PtrTy);
    251     }
    252     kb->CreateMemCpy(target, source, toCopy, 1);
    253     kb->CreateBr(calculateLogicalAddress);
    254    
     205    b->SetInsertPoint(copyBack);
     206    b->CreateMemCpy(baseBuffer, unreadData, remainingBytes, blockSize);
     207    b->CreateBr(prepareBuffer);
     208
    255209    // Otherwise, allocate a buffer with twice the capacity and copy the unconsumed data back into it
    256     kb->SetInsertPoint(expandAndCopyBack);
    257     Value * const expandedCapacity = kb->CreateShl(capacity, 1);
    258     Value * const expandedBytes = codeUnitWidth == 8 ? expandedCapacity : kb->CreateMul(expandedCapacity, codeUnitBytes);
    259     Value * const expandedBuffer = kb->CreatePointerCast(kb->CreateCacheAlignedMalloc(expandedBytes), codeUnitPtrTy);
    260     target = codeUnitWidth == 8 ? expandedBuffer : kb->CreatePointerCast(expandedBuffer, i8PtrTy);
    261     kb->CreateMemCpy(target, source, toCopy, 1);
    262     kb->CreateFree(buffer);
    263     kb->setScalarField("buffer", expandedBuffer);
    264     kb->setScalarField("capacity", expandedCapacity);
    265     kb->setCapacity("sourceBuffer", expandedCapacity);
    266     kb->CreateBr(calculateLogicalAddress);
    267 
    268     // Update the logical address for this buffer....
    269     kb->SetInsertPoint(calculateLogicalAddress);
    270     PHINode * const baseAddress = kb->CreatePHI(codeUnitPtrTy, 2);
    271     baseAddress->addIncoming(buffer, copyBack);
    272     baseAddress->addIncoming(expandedBuffer, expandAndCopyBack);
    273     Value * const logicalAddress = kb->CreateGEP(baseAddress, kb->CreateNeg(consumed));
    274     Value * const modifiedPtr = kb->CreateGEP(baseAddress, remaining);
    275     kb->setBaseAddress("sourceBuffer", logicalAddress);
    276     kb->CreateBr(readData);
     210    b->SetInsertPoint(expandAndCopyBack);
     211    Value * const expandedCapacity = b->CreateShl(capacity, 1);
     212    Value * const expandedBytes = b->CreateMul(expandedCapacity, codeUnitBytes);
     213    Value * const expandedBuffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(expandedBytes), unreadData->getType());
     214    b->CreateMemCpy(expandedBuffer, unreadData, remainingBytes, blockSize);
     215    b->CreateFree(baseBuffer);
     216    b->setScalarField("buffer", expandedBuffer);
     217    b->setCapacity("sourceBuffer", expandedCapacity);
     218    b->CreateBr(prepareBuffer);
     219
     220    b->SetInsertPoint(prepareBuffer);
     221    PHINode * newBaseBuffer = b->CreatePHI(baseBuffer->getType(), 2);
     222    newBaseBuffer->addIncoming(baseBuffer, copyBack);
     223    newBaseBuffer->addIncoming(expandedBuffer, expandAndCopyBack);
     224    b->setBaseAddress("sourceBuffer", b->CreateGEP(newBaseBuffer, b->CreateNeg(offset)));
     225    b->CreateBr(readData);
    277226
    278227    // Regardless of whether we're simply appending data or had to allocate a new buffer, read a new page
    279     // of data into the input source buffer. If we fail to read a full segment ...
    280     readData->moveAfter(calculateLogicalAddress);
    281     kb->SetInsertPoint(readData);
    282     calculateLogicalAddress->moveAfter(calculateLogicalAddress);
    283     PHINode * const addr = kb->CreatePHI(codeUnitPtrTy, 2);
    284     addr->addIncoming(originalPtr, exhaustedBuffer);
    285     addr->addIncoming(modifiedPtr, calculateLogicalAddress);
    286     Value * const fd = kb->getScalarField("fileDescriptor");
    287 
    288     Value * itemsRead = kb->CreateReadCall(fd, addr, readSize);
    289     if (codeUnitWidth != 8) {
    290         itemsRead = kb->CreateUDiv(itemsRead, codeUnitBytes);
    291     }
    292     unreadSize = kb->CreateAdd(unreadSize, itemsRead);
    293     bufferedSize = kb->CreateAdd(bufferedSize, itemsRead);
    294     kb->setBufferedSize("sourceBuffer", bufferedSize);
    295     Value * const exhaustedInputSource = kb->CreateICmpULT(unreadSize, segmentSize);
    296     BasicBlock * const setTermination = kb->CreateBasicBlock("SetTermination");
    297     kb->CreateUnlikelyCondBr(exhaustedInputSource, setTermination, stdInExit);
    298 
    299     // ... zero out the remaining bytes and set the termination signal.
    300     kb->SetInsertPoint(setTermination);
    301     Value * bytesToZero = kb->CreateSub(segmentSize, unreadSize);
    302     Value * unreadPtr = kb->CreateGEP(addr, unreadSize);
    303     bytesToZero = codeUnitWidth == 8 ? bytesToZero : kb->CreateMul(bytesToZero, codeUnitBytes);
    304     if (codeUnitWidth != 8) {
    305         bytesToZero = kb->CreateMul(bytesToZero, codeUnitBytes);
    306         unreadPtr = kb->CreatePointerCast(unreadPtr, i8PtrTy);
    307     }
    308     kb->CreateMemZero(unreadPtr, bytesToZero);
    309     kb->setCapacity("sourceBuffer", bufferedSize);
    310     kb->setTerminationSignal(kb->CreateICmpEQ(unreadSize, Constant::getNullValue(itemsRead->getType())));
    311     kb->CreateBr(stdInExit);
    312 
    313     // finally add the segment item count to the produced item count to inform the subsequent kernels how
    314     // much data is available for processing
    315     kb->SetInsertPoint(stdInExit);
    316     stdInExit->moveAfter(setTermination);
    317     PHINode * const items = kb->CreatePHI(produced->getType(), 3);
    318     items->addIncoming(segmentSize, entryBlock);
    319     items->addIncoming(segmentSize, readData);
    320     items->addIncoming(unreadSize, setTermination);
    321     kb->setProducedItemCount("sourceBuffer", kb->CreateAdd(produced, items));
     228    // of data into the input source buffer. If we fail to read a full page ...
     229    b->SetInsertPoint(readData);
     230    Value * const sourceBuffer = b->getRawOutputPointer("sourceBuffer", buffered);
     231    Value * const fd = b->getScalarField("fileDescriptor");
     232    Constant * const bytesToRead = ConstantExpr::getMul(itemsToRead, codeUnitBytes);
     233    Value * const bytesRead = b->CreateReadCall(fd, sourceBuffer, bytesToRead);
     234    Value * const itemsRead = b->CreateUDiv(bytesRead, codeUnitBytes);
     235    b->CreateAssert(b->CreateICmpULE(itemsRead, itemsToRead), "read more items than expected");
     236    Value * const itemsBuffered = b->CreateAdd(buffered, itemsRead);
     237    b->setBufferedSize("sourceBuffer", itemsBuffered);
     238    b->CreateUnlikelyCondBr(b->CreateICmpULT(itemsBuffered, itemsPending), setTermination, readExit);
     239
     240    // ... set the termination signal.   
     241    b->SetInsertPoint(setTermination);
     242    Value * const bytesToZero = b->CreateMul(b->CreateSub(itemsPending, itemsBuffered), codeUnitBytes);
     243    b->CreateMemZero(b->getRawOutputPointer("sourceBuffer", itemsBuffered), bytesToZero);
     244    b->setTerminationSignal();
     245    b->CreateBr(readExit);
     246
     247    readExit->moveAfter(setTermination);
     248    b->SetInsertPoint(readExit);
     249    PHINode * const itemsProduced = b->CreatePHI(itemsPending->getType(), 3);
     250    itemsProduced->addIncoming(itemsPending, entry);
     251    itemsProduced->addIncoming(itemsPending, readData);
     252    itemsProduced->addIncoming(itemsBuffered, setTermination);
     253    b->setProducedItemCount("sourceBuffer", itemsProduced);
    322254}
    323255
     
    326258}
    327259
    328 ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, unsigned blocksPerSegment, unsigned codeUnitWidth)
    329 : SegmentOrientedKernel("read_source"  + std::to_string(blocksPerSegment) + "@" + std::to_string(codeUnitWidth)
     260ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned blocksRequiredPerSegment, const unsigned codeUnitWidth)
     261: SegmentOrientedKernel("read_source"  + std::to_string(blocksRequiredPerSegment) + "@" + std::to_string(codeUnitWidth)
    330262, {}
    331 , {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
    332 , {Binding{kb->getInt32Ty(), "fileDescriptor"}}
     263, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer", FixedRate(), Deferred()}}
     264, {Binding{b->getInt32Ty(), "fileDescriptor"}}
    333265, {}
    334 , {Binding{IntegerType::get(kb->getContext(), codeUnitWidth)->getPointerTo(), "buffer"}, Binding{kb->getSizeTy(), "capacity"}})
    335 , mSegmentBlocks(blocksPerSegment)
     266, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
     267, mBlocksRequiredPerSegment(blocksRequiredPerSegment)
    336268, mCodeUnitWidth(codeUnitWidth) {
    337269
     
    366298    kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(STDIN_FILENO)), initializeRead, initializeMMap);
    367299    kb->SetInsertPoint(initializeRead);
    368     ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, kb);
     300    ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, mBlocksRequiredPerSegment, kb);
    369301    kb->CreateBr(initializeDone);
    370302    kb->SetInsertPoint(initializeMMap);
    371     MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, kb);
     303    MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, mBlocksRequiredPerSegment, kb);
    372304    kb->CreateBr(initializeDone);
    373305    kb->SetInsertPoint(initializeDone);
     
    381313    kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(STDIN_FILENO)), DoSegmentRead, DoSegmentMMap);
    382314    kb->SetInsertPoint(DoSegmentRead);
    383     ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mSegmentBlocks, kb);
     315    ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mBlocksRequiredPerSegment, kb);
    384316    kb->CreateBr(DoSegmentDone);
    385317    kb->SetInsertPoint(DoSegmentMMap);
    386     MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mSegmentBlocks, kb);
     318    MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mBlocksRequiredPerSegment, kb);
    387319    kb->CreateBr(DoSegmentDone);
    388320    kb->SetInsertPoint(DoSegmentDone);
    389321}
    390322
    391 FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, unsigned blocksPerSegment, unsigned codeUnitWidth)
    392 : SegmentOrientedKernel("FD_source" + std::to_string(blocksPerSegment) + "@" + std::to_string(codeUnitWidth)
     323FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, const unsigned blocksRequiredPerSegment, const unsigned codeUnitWidth)
     324: SegmentOrientedKernel("FD_source" + std::to_string(blocksRequiredPerSegment) + "@" + std::to_string(codeUnitWidth)
    393325, {}
    394326, {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
    395327, {Binding{kb->getInt32Ty(), "fileDescriptor"}}
    396328, {}
    397 , {Binding{IntegerType::get(kb->getContext(), codeUnitWidth)->getPointerTo(), "buffer"}, Binding{kb->getSizeTy(), "capacity"},
     329, {Binding{kb->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"},
    398330    Binding{kb->getSizeTy(), "fileSize"}, Binding{kb->getInt8PtrTy(), "readableBuffer"}})
    399 , mSegmentBlocks(blocksPerSegment)
     331, mBlocksRequiredPerSegment(blocksRequiredPerSegment)
    400332, mCodeUnitWidth(codeUnitWidth)
    401333, mFileSizeFunction(nullptr) {
     
    418350    BasicBlock * setTermination = kb->CreateBasicBlock("setTermination");
    419351    BasicBlock * mmapSourceExit = kb->CreateBasicBlock("sourceExit");
    420     ConstantInt * segmentItems = kb->getSize(mSegmentBlocks * kb->getBitBlockWidth());
     352    ConstantInt * segmentItems = kb->getSize(mBlocksRequiredPerSegment * kb->getBitBlockWidth());
    421353    Value * fileItems = kb->getScalarField("fileSize");
    422354    if (mCodeUnitWidth > 8) {
     
    439371}
    440372
    441 MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, Type * type, unsigned blocksPerSegment, unsigned codeUnitWidth)
     373MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, Type * const type, const unsigned blocksRequiredPerSegment, const unsigned codeUnitWidth)
    442374: SegmentOrientedKernel("memory_source",
    443375    {},
    444376    {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}},
    445377    {Binding{cast<PointerType>(type), "fileSource"}, Binding{kb->getSizeTy(), "fileSize"}}, {}, {})
    446 , mSegmentBlocks(blocksPerSegment)
     378, mBlocksRequiredPerSegment(blocksRequiredPerSegment)
    447379, mCodeUnitWidth(codeUnitWidth) {
    448380
Note: See TracChangeset for help on using the changeset viewer.