Ignore:
Timestamp:
Nov 2, 2018, 7:18:31 PM (9 months ago)
Author:
nmedfort
Message:

Initial version of PipelineKernel? + revised StreamSet? model.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

    r6155 r6184  
    1313using namespace llvm;
    1414
     15
    1516extern "C" uint64_t file_size(const uint32_t fd) {
    1617    struct stat st;
     
    2627
    2728Function * MMapSourceKernel::linkFileSizeMethod(const std::unique_ptr<kernel::KernelBuilder> & b) {
    28     return b->LinkFunction("file_size", &file_size);
    29 }
    30 
    31 void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
     29    return b->LinkFunction("file_size", file_size);
     30}
     31
     32void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
    3233
    3334    BasicBlock * const emptyFile = b->CreateBasicBlock("emptyFile");
     
    3536    BasicBlock * const exit = b->CreateBasicBlock("Exit");
    3637    IntegerType * const sizeTy = b->getSizeTy();
    37     ConstantInt * const PAGE_SIZE = b->getSize(getpagesize());
    38     Constant * const PAGE_ITEMS = b->getSize(getpagesize()/(codeUnitWidth/8));
     38    ConstantInt * const PAGE_SIZE = b->getSize(stride);
     39    Constant * const PAGE_ITEMS = b->getSize(stride /(codeUnitWidth/8));
    3940    Value * const fd = b->getScalarField("fileDescriptor");
     41
     42    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
     43    b->setScalarField("ancillaryBuffer", ConstantPointerNull::get(codeUnitPtrTy));
    4044    assert (fileSizeMethod);
    4145    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(fileSizeMethod, fd), sizeTy);
     
    4347
    4448    b->SetInsertPoint(nonEmptyFile);
    45     PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
    4649    Value * const fileBuffer = b->CreatePointerCast(b->CreateFileSourceMMap(fd, fileSize), codeUnitPtrTy);
    4750    b->setScalarField("buffer", fileBuffer);
     
    5356    }
    5457    b->setScalarField("fileItems", fileItems);
     58    b->setCapacity("sourceBuffer", fileItems);
    5559    b->CreateBr(exit);
    5660
     
    6771
    6872
    69 void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
     73void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
    7074
    7175    BasicBlock * const dropPages = b->CreateBasicBlock("dropPages");
     
    7478    BasicBlock * const exit = b->CreateBasicBlock("mmapSourceExit");
    7579
    76     Constant * const PAGE_SIZE = b->getSize(getpagesize());
    77     Constant * const PAGE_ITEMS = b->getSize(getpagesize()/(codeUnitWidth/8));
     80    Constant * const PAGE_SIZE = b->getSize(stride);
     81    Constant * const PAGE_ITEMS = b->getSize((8 * stride) / codeUnitWidth);
    7882    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
    7983    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
    80 
    81     Value * const fileItems = b->getScalarField("fileItems");
     84    Constant * const PADDING_SIZE = b->getSize(b->getBitBlockWidth() * codeUnitWidth / 8);
     85
    8286    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
    8387    Value * const consumedBytes = b->CreateMul(consumedItems, CODE_UNIT_BYTES);
     
    100104    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
    101105    Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_ITEMS);
     106    Value * const fileItems = b->getScalarField("fileItems");
    102107    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
    103108    b->CreateUnlikelyCondBr(lastPage, setTermination, exit);
     
    109114    Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
    110115    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
    111     Value * unconsumedBytes = b->CreateTrunc(b->CreatePtrDiff(readEnd, readStart), b->getSizeTy());
    112     if (LLVM_UNLIKELY(codeUnitWidth > 8)) {
    113         unconsumedBytes = b->CreateMul(unconsumedBytes, CODE_UNIT_BYTES);
    114     }
    115     Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
     116
     117    DataLayout DL(b->getModule());
     118    Type * const intPtrTy = DL.getIntPtrType(readEnd->getType());
     119    Value * const readEndInt = b->CreatePtrToInt(readEnd, intPtrTy);
     120    Value * const readStartInt = b->CreatePtrToInt(readStart, intPtrTy);
     121    Value * unconsumedBytes = b->CreateSub(readEndInt, readStartInt);
     122    unconsumedBytes = b->CreateTrunc(unconsumedBytes, b->getSizeTy());
     123
     124    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, PADDING_SIZE), PAGE_SIZE);
    116125    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
     126
    117127    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
    118128    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
    119129    // get the difference between our base and from position then compute an offsetted temporary buffer address
    120130    Value * const base = b->getBaseAddress("sourceBuffer");
    121     Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
     131    Value * const baseInt = b->CreatePtrToInt(base, intPtrTy);
     132    Value * const diff = b->CreateSub(baseInt, readStartInt);
    122133    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
    123     b->CreateConsumerWait();
    124     // Unmap the file buffer and set the temporary buffer as the new source buffer
    125     Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES);
    126     b->CreateMUnmap(base, fileSize);
    127134    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
    128     b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
     135    // set the original base address as the buffer address.
     136    b->setScalarField("buffer", b->CreatePointerCast(base, codeUnitPtrTy));
     137    b->setScalarField("ancillaryBuffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
    129138    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
    130139    b->setTerminationSignal();
     
    139148    b->setProducedItemCount("sourceBuffer", newProducedItems);
    140149}
    141 
    142 void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
    143     b->CreateFree(b->getScalarField("buffer"));
     150void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b, const unsigned codeUnitWidth) {
     151    b->CreateFree(b->getScalarField("ancillaryBuffer"));
     152    Value * const fileItems = b->getScalarField("fileItems");
     153    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
     154    Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES);
     155    b->CreateMUnmap(b->getScalarField("buffer"), fileSize);
    144156}
    145157
    146158/// READ SOURCE KERNEL
    147159
    148 void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
    149     const unsigned pageSize = getpagesize();
    150     ConstantInt * const bufferItems = b->getSize(pageSize * 4);
     160void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
     161    ConstantInt * const bufferItems = b->getSize(stride * 4);
    151162    const auto codeUnitSize = codeUnitWidth / 8;
    152     ConstantInt * const bufferBytes = b->getSize(pageSize * 4 * codeUnitSize);
     163    ConstantInt * const bufferBytes = b->getSize(stride * 4 * codeUnitSize);
    153164    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
    154165    Value * const buffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy);
    155166    b->setBaseAddress("sourceBuffer", buffer);
     167    b->setScalarField("ancillaryBuffer", ConstantPointerNull::get(codeUnitPtrTy));
    156168    b->setScalarField("buffer", buffer);
    157169    b->setCapacity("sourceBuffer", bufferItems);
    158170}
    159171
    160 void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
    161 
    162     const unsigned pageSize = getpagesize();
    163     ConstantInt * const itemsToRead = b->getSize(pageSize);
     172void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
     173
     174    ConstantInt * const itemsToRead = b->getSize(stride);
    164175    ConstantInt * const codeUnitBytes = b->getSize(codeUnitWidth / 8);
    165176
     
    170181    BasicBlock * const readExit = b->CreateBasicBlock("ReadExit");
    171182
    172     // Do we have enough unread data to support one segment?
     183    // Can we append to our existing buffer without impacting any subsequent kernel?
    173184    Value * const produced = b->getProducedItemCount("sourceBuffer");
    174185    Value * const itemsPending = b->CreateAdd(produced, itemsToRead);
    175 
    176     // Can we append to our existing buffer without impacting any subsequent kernel?
    177186    Value * const capacity = b->getCapacity("sourceBuffer");
    178187    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", itemsPending);
     
    181190    b->CreateLikelyCondBr(b->CreateICmpULE(readEnd, bufferLimit), readData, moveData);
    182191
    183     // First wait on any consumers to finish processing then check how much data has been consumed.
     192    // No. If we can copy the unconsumed data back to the start of the buffer *and* write a full
     193    // segment of data without overwriting the currently unconsumed data, do so since it won't
     194    // affect any potential consumer that could be using the "stale" output base pointer.
    184195    b->SetInsertPoint(moveData);
    185     b->CreateConsumerWait();
    186 
    187     // Then determine how much data has been consumed and how much needs to be copied back, noting
     196
     197    // Determine how much data has been consumed and how much needs to be copied back, noting
    188198    // that our "unproduced" data must be block aligned.
    189199    BasicBlock * const copyBack = b->CreateBasicBlock("CopyBack");
    190200    BasicBlock * const expandAndCopyBack = b->CreateBasicBlock("ExpandAndCopyBack");
    191 
    192201    const auto blockSize = b->getBitBlockWidth() / 8;
    193202    Constant * const blockSizeAlignmentMask = ConstantExpr::getNeg(b->getSize(blockSize));
     
    196205    Value * const unreadData = b->getRawOutputPointer("sourceBuffer", offset);
    197206    Value * const remainingItems = b->CreateSub(produced, offset);
     207    Value * const potentialItems = b->CreateAdd(remainingItems, itemsToRead);
    198208    Value * const remainingBytes = b->CreateMul(remainingItems, codeUnitBytes);
    199 
    200     // Have we consumed enough data that we can safely copy back the unconsumed data without needing a temporary buffer?
    201     Value * const canCopy = b->CreateICmpULT(b->CreateGEP(baseBuffer, remainingItems), b->getRawOutputPointer("sourceBuffer", offset));
     209    // Have we consumed enough data that we can safely copy back the unconsumed data and still leave enough space
     210    // for one segment without needing a temporary buffer?
     211    Value * const canCopy = b->CreateICmpULT(b->CreateGEP(baseBuffer, potentialItems), unreadData);
    202212    b->CreateLikelyCondBr(canCopy, copyBack, expandAndCopyBack);
    203213
     
    213223    Value * const expandedBuffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(expandedBytes), unreadData->getType());
    214224    b->CreateMemCpy(expandedBuffer, unreadData, remainingBytes, blockSize);
     225    // Free the prior buffer if it exists
     226    Value * const ancillaryBuffer = b->getScalarField("ancillaryBuffer");
     227    b->CreateFree(ancillaryBuffer);
     228    b->setScalarField("ancillaryBuffer", baseBuffer);
    215229    b->setScalarField("buffer", expandedBuffer);
    216230    b->setCapacity("sourceBuffer", expandedCapacity);
    217     b->CreateFree(baseBuffer);
    218231    b->CreateBr(prepareBuffer);
    219232
     
    222235    newBaseBuffer->addIncoming(baseBuffer, copyBack);
    223236    newBaseBuffer->addIncoming(expandedBuffer, expandAndCopyBack);
    224     b->setBaseAddress("sourceBuffer", b->CreateGEP(newBaseBuffer, b->CreateNeg(offset)));
     237    Value * const newBaseAddress = b->CreateGEP(newBaseBuffer, b->CreateNeg(offset));
     238    b->setBaseAddress("sourceBuffer", newBaseAddress);
    225239    b->CreateBr(readData);
    226240
     
    240254    Value * const bytesToZero = b->CreateMul(b->CreateSub(itemsPending, itemsBuffered), codeUnitBytes);
    241255    b->CreateMemZero(b->getRawOutputPointer("sourceBuffer", itemsBuffered), bytesToZero);
     256    b->setScalarField("fileItems", itemsBuffered);
    242257    b->setTerminationSignal();
    243258    b->CreateBr(readExit);
     
    251266
    252267void ReadSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
     268    b->CreateFree(b->getScalarField("ancillaryBuffer"));
    253269    b->CreateFree(b->getScalarField("buffer"));
    254270}
     
    266282    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), finalizeMMap, finalizeRead);
    267283    b->SetInsertPoint(finalizeMMap);
    268     MMapSourceKernel::freeBuffer(b);
     284    MMapSourceKernel::freeBuffer(b, mCodeUnitWidth);
    269285    b->CreateBr(finalizeDone);
    270286    b->SetInsertPoint(finalizeRead);
     
    293309    useMMap = b->CreateICmpNE(fileSize, b->getSize(0));
    294310    b->CreateCondBr(useMMap, initializeMMap, initializeRead);
     311
    295312    b->SetInsertPoint(initializeMMap);
    296     MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, b);
     313    MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, mStride, b);
    297314    b->CreateBr(initializeDone);
    298315
     
    300317    // Ensure that readSource logic is used throughout.
    301318    b->setScalarField("useMMap", b->getInt8(0));
    302     ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth,b);
     319    ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, mStride,b);
    303320    b->CreateBr(initializeDone);
    304321    b->SetInsertPoint(initializeDone);
     
    311328    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), DoSegmentMMap, DoSegmentRead);
    312329    b->SetInsertPoint(DoSegmentMMap);
    313     MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, b);
     330    MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride, b);
    314331    b->CreateBr(DoSegmentDone);
    315332    b->SetInsertPoint(DoSegmentRead);
    316     ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, b);
     333    ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride, b);
    317334    b->CreateBr(DoSegmentDone);
    318335    b->SetInsertPoint(DoSegmentDone);
     
    324341    Value * const fileSource = b->getScalarField("fileSource");
    325342    b->setBaseAddress("sourceBuffer", fileSource);
     343    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
     344        b->CreateAssert(fileSource, getName() + " fileSource cannot be null");
     345    }
    326346    Value * const fileItems = b->getScalarField("fileItems");
    327347    b->setCapacity("sourceBuffer", fileItems);
    328     if (mStreamSetCount > 1) {
    329         b->setProducedItemCount("sourceBuffer", fileItems);
    330         b->setTerminationSignal();
     348}
     349
     350void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
     351
     352    Constant * const STRIDE_ITEMS = b->getSize(getStride());
     353    Constant * const STRIDE_SIZE = b->getSize(getStride() * mCodeUnitWidth);
     354    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
     355
     356    BasicBlock * const entry = b->GetInsertBlock();
     357    BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary");
     358    BasicBlock * const exit = b->CreateBasicBlock("exit");
     359
     360    Value * const fileItems = b->getScalarField("fileItems");
     361    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
     362    Value * const nextProducedItems = b->CreateAdd(producedItems, STRIDE_ITEMS);
     363    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
     364    b->CreateUnlikelyCondBr(lastPage, createTemporary, exit);
     365
     366    b->SetInsertPoint(createTemporary);
     367    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
     368    Value * readStart = nullptr;
     369    Value * readEnd = nullptr;
     370
     371    // compute the range of our unconsumed buffer slice
     372    if (LLVM_UNLIKELY(mStreamSetCount > 1)) {
     373        Constant * const ZERO = b->getSize(0);
     374        const StreamSetBuffer * const sourceBuffer = getOutputStreamSetBuffer("sourceBuffer");
     375        Value * const fromIndex = b->CreateUDiv(consumedItems, BLOCK_WIDTH);
     376        readStart = sourceBuffer->getStreamBlockPtr(b.get(), ZERO, fromIndex);
     377        Value * const toIndex = b->CreateUDivCeil(fileItems, BLOCK_WIDTH);
     378        // since we know this is an ExternalBuffer, we don't need to consider any potential modulus calculations.
     379        readEnd = sourceBuffer->getStreamBlockPtr(b.get(), ZERO, toIndex);
     380    } else {
     381        // make sure our copy is block-aligned
     382        Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
     383        readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
     384        readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
    331385    }
    332 }
    333 
    334 void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
    335     if (mStreamSetCount == 1) {
    336         Constant * const PAGE_SIZE = b->getSize(getStride());
    337         Constant * const PAGE_ITEMS = b->getSize(getStride()/(mCodeUnitWidth/8));
    338         Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
    339         Constant * const CODE_UNIT_BYTES = b->getSize(mCodeUnitWidth / 8);
    340 
    341         BasicBlock * const entry = b->GetInsertBlock();
    342         BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary");
    343         BasicBlock * const exit = b->CreateBasicBlock("exit");
    344 
    345         Value * const fileItems = b->getScalarField("fileItems");
    346         Value * const producedItems = b->getProducedItemCount("sourceBuffer");
    347         Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_ITEMS);
    348         Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
    349         b->CreateUnlikelyCondBr(lastPage, createTemporary, exit);
    350 
    351         b->SetInsertPoint(createTemporary);
    352         Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
    353         Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
    354         Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
    355         Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
    356         Value * unconsumedBytes = b->CreateTrunc(b->CreatePtrDiff(readEnd, readStart), b->getSizeTy());
    357         if (LLVM_UNLIKELY(mCodeUnitWidth > 8)) {
    358             unconsumedBytes = b->CreateMul(unconsumedBytes, CODE_UNIT_BYTES);
    359         }
    360         Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
    361         Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
    362         b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
    363         b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
    364 
    365         // get the difference between our base and from position then compute an offsetted temporary buffer address
    366         Value * const base = b->getBaseAddress("sourceBuffer");
    367         Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
    368         Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
    369         b->CreateConsumerWait();
    370 
    371         // set the temporary buffer as the new source buffer
    372         PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
    373         b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
    374         b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
    375         b->setTerminationSignal();
    376         BasicBlock * const terminationExit = b->GetInsertBlock();
    377         b->CreateBr(exit);
    378 
    379         b->SetInsertPoint(exit);
    380         PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
    381         newProducedItems->addIncoming(nextProducedItems, entry);
    382         newProducedItems->addIncoming(fileItems, terminationExit);
    383         b->setProducedItemCount("sourceBuffer", newProducedItems);
    384     }
     386
     387    DataLayout DL(b->getModule());
     388    Type * const intPtrTy = DL.getIntPtrType(readEnd->getType());
     389    Value * const readEndInt = b->CreatePtrToInt(readEnd, intPtrTy);
     390    Value * const readStartInt = b->CreatePtrToInt(readStart, intPtrTy);
     391    Value * const unconsumedBytes = b->CreateTrunc(b->CreateSub(readEndInt, readStartInt), b->getSizeTy());
     392    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), STRIDE_SIZE);
     393    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
     394    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
     395    b->setScalarField("ancillaryBuffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
     396    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
     397    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
     398
     399    // get the difference between our base and from position then compute an offsetted temporary buffer address
     400    Value * const base = b->getBaseAddress("sourceBuffer");
     401    Value * const baseInt = b->CreatePtrToInt(base, intPtrTy);
     402    Value * const diff = b->CreateSub(baseInt, readStartInt);
     403    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
     404    // set the temporary buffer as the new source buffer
     405    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
     406    b->setTerminationSignal();
     407    BasicBlock * const terminationExit = b->GetInsertBlock();
     408    b->CreateBr(exit);
     409
     410    b->SetInsertPoint(exit);
     411    PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
     412    newProducedItems->addIncoming(nextProducedItems, entry);
     413    newProducedItems->addIncoming(fileItems, terminationExit);
     414    b->setProducedItemCount("sourceBuffer", newProducedItems);   
    385415}
    386416
    387417void MemorySourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
    388     if (mStreamSetCount == 1) {
    389         b->CreateFree(b->getScalarField("buffer"));
    390     }
    391 }
    392 
    393 MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
    394 : SegmentOrientedKernel("mmap_source@" + std::to_string(codeUnitWidth)
     418    b->CreateFree(b->getScalarField("ancillaryBuffer"));
     419}
     420
     421MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const fd, StreamSet * const outputStream)
     422: SegmentOrientedKernel("mmap_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
    395423// input streams
    396 , {}
     424,{}
    397425// output streams
    398 , {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
     426,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
    399427// input scalars
    400 , {Binding{b->getInt32Ty(), "fileDescriptor"}}
     428,{Binding{"fileDescriptor", fd}}
    401429// output scalars
    402 , {Binding{b->getSizeTy(), "fileItems"}}
     430,{Binding{b->getSizeTy(), "fileItems"}}
    403431// internal scalars
    404 , {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
    405 , mCodeUnitWidth(codeUnitWidth)
     432,{})
     433, mCodeUnitWidth(outputStream->getFieldWidth())
    406434, mFileSizeFunction(nullptr) {
     435    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
     436    addInternalScalar(codeUnitPtrTy, "buffer");
     437    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
    407438    addAttribute(MustExplicitlyTerminate());
    408     setStride(getpagesize());
    409 }
    410 
    411 ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
    412 : SegmentOrientedKernel("read_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(codeUnitWidth)
     439    setStride(codegen::SegmentSize);
     440}
     441
     442ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const fd, StreamSet * const outputStream)
     443: SegmentOrientedKernel("read_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
    413444// input streams
    414 , {}
     445,{}
    415446// output streams
    416 , {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
     447,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
    417448// input scalars
    418 , {Binding{b->getInt32Ty(), "fileDescriptor"}}
     449,{Binding{"fileDescriptor", fd}}
    419450// output scalars
    420 , {}
     451,{Binding{b->getSizeTy(), "fileItems"}}
    421452// internal scalars
    422 , {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
    423 , mCodeUnitWidth(codeUnitWidth) {
     453,{})
     454, mCodeUnitWidth(outputStream->getFieldWidth()) {
     455    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
     456    addInternalScalar(codeUnitPtrTy, "buffer");
     457    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
    424458    addAttribute(MustExplicitlyTerminate());
    425     setStride(getpagesize());
    426 }
    427 
    428 
    429 FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
    430 : SegmentOrientedKernel("FD_source@" + std::to_string(codeUnitWidth)
     459    setStride(codegen::SegmentSize);
     460}
     461
     462
     463FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const useMMap, Scalar * const fd, StreamSet * const outputStream)
     464: SegmentOrientedKernel("FD_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
    431465// input streams
    432 , {}
     466,{}
    433467// output stream
    434 , {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
     468,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
    435469// input scalar
    436 , {Binding{b->getInt8Ty(), "useMMap"}, Binding{b->getInt32Ty(), "fileDescriptor"}}
    437 // output scalars
    438 , {Binding{b->getSizeTy(), "fileItems"}}
     470,{Binding{"useMMap", useMMap}
     471, Binding{"fileDescriptor", fd}}
     472// output scalar
     473,{Binding{b->getSizeTy(), "fileItems"}}
    439474// internal scalars
    440 , {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
    441 , mCodeUnitWidth(codeUnitWidth)
     475,{})
     476, mCodeUnitWidth(outputStream->getFieldWidth())
    442477, mFileSizeFunction(nullptr) {
     478    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
     479    addInternalScalar(codeUnitPtrTy, "buffer");
     480    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
    443481    addAttribute(MustExplicitlyTerminate());
    444     setStride(getpagesize());
    445 }
    446 
    447 MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned streamSetCount, const unsigned codeUnitWidth)
    448 : SegmentOrientedKernel("memory_source@" + std::to_string(streamSetCount) + ":" + std::to_string(codeUnitWidth),
     482    setStride(codegen::SegmentSize);
     483}
     484
     485MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> &, Scalar * fileSource, Scalar * fileItems, StreamSet * const outputStream)
     486: SegmentOrientedKernel("memory_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth()) + ":" + std::to_string(outputStream->getNumElements()),
    449487// input streams
    450488{},
    451489// output stream
    452 {Binding{b->getStreamSetTy(streamSetCount, codeUnitWidth), "sourceBuffer"}},
     490{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}},
    453491// input scalar
    454 {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "fileSource"}, Binding{b->getSizeTy(), "fileItems"}},
     492{Binding{"fileSource", fileSource}, Binding{"fileItems", fileItems}},
    455493{},
    456494// internal scalar
    457 {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
    458 , mStreamSetCount(streamSetCount)
    459 , mCodeUnitWidth(codeUnitWidth) {
     495{Binding{fileSource->getType(), "buffer"}
     496,Binding{fileSource->getType(), "ancillaryBuffer"}
     497})
     498, mStreamSetCount(outputStream->getNumElements())
     499, mCodeUnitWidth(outputStream->getFieldWidth()) {
    460500    addAttribute(MustExplicitlyTerminate());
    461     setStride(getpagesize());
    462 }
    463 
    464 }
     501    setStride(codegen::SegmentSize);
     502}
     503
     504}
Note: See TracChangeset for help on using the changeset viewer.