Ignore:
Timestamp:
Jun 10, 2017, 7:21:21 AM (2 years ago)
Author:
cameron
Message:

Read source kernel restructure calculations using logical code units

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

    r5499 r5500  
    2828/// MMAP SOURCE KERNEL
    2929
    30 void MMapSourceKernel::linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    31     mFileSizeFunction = iBuilder->LinkFunction("file_size", &file_size);
    32 }
    33 
    34 void MMapSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & iBuilder) {
    35     BasicBlock * const emptyFile = iBuilder->CreateBasicBlock("EmptyFile");
    36     BasicBlock * const nonEmptyFile = iBuilder->CreateBasicBlock("NonEmptyFile");
    37     BasicBlock * const exit = iBuilder->CreateBasicBlock("Exit");
    38     IntegerType * const sizeTy = iBuilder->getSizeTy();
    39     assert (iBuilder->getKernel() == this);
    40     Value * const fd = iBuilder->getScalarField("fileDescriptor");
     30void MMapSourceKernel::linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & kb) {
     31    mFileSizeFunction = kb->LinkFunction("file_size", &file_size);
     32}
     33
     34void MMapSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & kb) {
     35    BasicBlock * const emptyFile = kb->CreateBasicBlock("EmptyFile");
     36    BasicBlock * const nonEmptyFile = kb->CreateBasicBlock("NonEmptyFile");
     37    BasicBlock * const exit = kb->CreateBasicBlock("Exit");
     38    IntegerType * const sizeTy = kb->getSizeTy();
     39    assert (kb->getKernel() == this);
     40    Value * const fd = kb->getScalarField("fileDescriptor");
    4141    assert (mFileSizeFunction);
    42     Value * fileSize = iBuilder->CreateCall(mFileSizeFunction, fd);
    43     fileSize = iBuilder->CreateZExtOrTrunc(fileSize, sizeTy);
     42    Value * fileSize = kb->CreateCall(mFileSizeFunction, fd);
     43    fileSize = kb->CreateZExtOrTrunc(fileSize, sizeTy);
    4444    if (mCodeUnitWidth > 8) {
    45         fileSize = iBuilder->CreateUDiv(fileSize, iBuilder->getSize(mCodeUnitWidth / 8));
    46     }
    47     Value * const isEmpty = iBuilder->CreateICmpEQ(fileSize, ConstantInt::getNullValue(fileSize->getType()));
    48     iBuilder->CreateUnlikelyCondBr(isEmpty, emptyFile, nonEmptyFile);
     45        fileSize = kb->CreateUDiv(fileSize, kb->getSize(mCodeUnitWidth / 8));
     46    }
     47    Value * const isEmpty = kb->CreateICmpEQ(fileSize, ConstantInt::getNullValue(fileSize->getType()));
     48    kb->CreateUnlikelyCondBr(isEmpty, emptyFile, nonEmptyFile);
    4949    // we cannot mmap a 0 length file; just create a 1-page sized fake file buffer for simplicity
    50     iBuilder->SetInsertPoint(emptyFile);
    51     Constant * pageSize = iBuilder->getSize(getpagesize());
    52     Value * fakeFileBuffer = iBuilder->CreateAnonymousMMap(pageSize);
    53     iBuilder->CreateBr(exit);
    54 
    55     iBuilder->SetInsertPoint(nonEmptyFile);
    56     Value * fileBackedBuffer = iBuilder->CreateFileSourceMMap(fd, fileSize);
    57     iBuilder->CreateBr(exit);
    58 
    59     iBuilder->SetInsertPoint(exit);
    60     PHINode * buffer = iBuilder->CreatePHI(fileBackedBuffer->getType(), 2);
     50    kb->SetInsertPoint(emptyFile);
     51    Constant * pageSize = kb->getSize(getpagesize());
     52    Value * fakeFileBuffer = kb->CreateAnonymousMMap(pageSize);
     53    kb->CreateBr(exit);
     54
     55    kb->SetInsertPoint(nonEmptyFile);
     56    Value * fileBackedBuffer = kb->CreateFileSourceMMap(fd, fileSize);
     57    kb->CreateBr(exit);
     58
     59    kb->SetInsertPoint(exit);
     60    PHINode * buffer = kb->CreatePHI(fileBackedBuffer->getType(), 2);
    6161    buffer->addIncoming(fakeFileBuffer, emptyFile);
    6262    buffer->addIncoming(fileBackedBuffer, nonEmptyFile);
    63     PHINode * size = iBuilder->CreatePHI(sizeTy, 2);
     63    PHINode * size = kb->CreatePHI(sizeTy, 2);
    6464    size->addIncoming(pageSize, emptyFile);
    6565    size->addIncoming(fileSize, nonEmptyFile);
    6666
    67     iBuilder->setBaseAddress("sourceBuffer", buffer);
    68     iBuilder->setBufferedSize("sourceBuffer", size);
    69     iBuilder->setScalarField("readableBuffer", buffer);
    70     iBuilder->setScalarField("fileSize", fileSize);
    71     iBuilder->CreateMAdvise(buffer, fileSize, CBuilder::ADVICE_WILLNEED);
    72 
    73 }
    74 
    75 void MMapSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & iBuilder) {
    76 
    77     BasicBlock * dropPages = iBuilder->CreateBasicBlock("dropPages");
    78     BasicBlock * processSegment = iBuilder->CreateBasicBlock("produceData");
    79     BasicBlock * setTermination = iBuilder->CreateBasicBlock("setTermination");
    80     BasicBlock * mmapSourceExit = iBuilder->CreateBasicBlock("mmapSourceExit");
     67    kb->setBaseAddress("sourceBuffer", buffer);
     68    kb->setBufferedSize("sourceBuffer", size);
     69    kb->setScalarField("readableBuffer", buffer);
     70    kb->setScalarField("fileSize", fileSize);
     71    kb->CreateMAdvise(buffer, fileSize, CBuilder::ADVICE_WILLNEED);
     72
     73}
     74
     75void MMapSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & kb) {
     76
     77    BasicBlock * dropPages = kb->CreateBasicBlock("dropPages");
     78    BasicBlock * processSegment = kb->CreateBasicBlock("produceData");
     79    BasicBlock * setTermination = kb->CreateBasicBlock("setTermination");
     80    BasicBlock * mmapSourceExit = kb->CreateBasicBlock("mmapSourceExit");
    8181
    8282    // instruct the OS that it can safely drop any fully consumed pages
    83     Value * consumed = iBuilder->getConsumedItemCount("sourceBuffer");
     83    Value * consumed = kb->getConsumedItemCount("sourceBuffer");
    8484    IntegerType * const consumedTy = cast<IntegerType>(consumed->getType());
    85     Type * const voidPtrTy = iBuilder->getVoidPtrTy();
    86 
    87     DataLayout DL(iBuilder->getModule());
    88     IntegerType * const intAddrTy = iBuilder->getIntPtrTy(DL);
     85    Type * const voidPtrTy = kb->getVoidPtrTy();
     86
     87    DataLayout DL(kb->getModule());
     88    IntegerType * const intAddrTy = kb->getIntPtrTy(DL);
    8989
    9090    // multiply the consumed count by the code unit size then mask off any partial pages
    9191    if (mCodeUnitWidth > 8) {
    92         consumed = iBuilder->CreateMul(consumed, ConstantInt::get(consumedTy, mCodeUnitWidth / 8));
     92        consumed = kb->CreateMul(consumed, ConstantInt::get(consumedTy, mCodeUnitWidth / 8));
    9393    }
    9494    const auto pageSize = getpagesize();
    9595    if (LLVM_LIKELY((pageSize & (pageSize - 1)) == 0)) {
    96         consumed = iBuilder->CreateAnd(consumed, ConstantExpr::getNeg(ConstantInt::get(consumedTy, pageSize)));
     96        consumed = kb->CreateAnd(consumed, ConstantExpr::getNeg(ConstantInt::get(consumedTy, pageSize)));
    9797    } else {
    98         consumed = iBuilder->CreateSub(consumed, iBuilder->CreateURem(consumed, ConstantInt::get(consumedTy, pageSize)));
    99     }
    100 
    101     Value * sourceBuffer = iBuilder->getBaseAddress("sourceBuffer");
    102     sourceBuffer = iBuilder->CreatePtrToInt(sourceBuffer, intAddrTy);
     98        consumed = kb->CreateSub(consumed, kb->CreateURem(consumed, ConstantInt::get(consumedTy, pageSize)));
     99    }
     100
     101    Value * sourceBuffer = kb->getBaseAddress("sourceBuffer");
     102    sourceBuffer = kb->CreatePtrToInt(sourceBuffer, intAddrTy);
    103103    if (LLVM_UNLIKELY(intAddrTy->getBitWidth() > consumedTy->getBitWidth())) {
    104         consumed = iBuilder->CreateZExt(consumed, intAddrTy);
     104        consumed = kb->CreateZExt(consumed, intAddrTy);
    105105    } else if (LLVM_UNLIKELY(intAddrTy->getBitWidth() < consumedTy->getBitWidth())) {
    106         sourceBuffer = iBuilder->CreateZExt(sourceBuffer, consumedTy);
    107     }
    108     Value * consumedBuffer = iBuilder->CreateAdd(sourceBuffer, consumed);
    109     Value * readableBuffer = iBuilder->getScalarField("readableBuffer");
    110     readableBuffer = iBuilder->CreatePtrToInt(readableBuffer, consumedBuffer->getType());
    111     Value * unnecessaryBytes = iBuilder->CreateSub(consumedBuffer, readableBuffer);
     106        sourceBuffer = kb->CreateZExt(sourceBuffer, consumedTy);
     107    }
     108    Value * consumedBuffer = kb->CreateAdd(sourceBuffer, consumed);
     109    Value * readableBuffer = kb->getScalarField("readableBuffer");
     110    readableBuffer = kb->CreatePtrToInt(readableBuffer, consumedBuffer->getType());
     111    Value * unnecessaryBytes = kb->CreateSub(consumedBuffer, readableBuffer);
    112112
    113113    // avoid calling madvise unless an actual page table change could occur
    114     Value * hasPagesToDrop = iBuilder->CreateICmpEQ(unnecessaryBytes, ConstantInt::getNullValue(intAddrTy));
    115     iBuilder->CreateLikelyCondBr(hasPagesToDrop, processSegment, dropPages);
    116 
    117     iBuilder->SetInsertPoint(dropPages);
    118     iBuilder->CreateMAdvise(iBuilder->CreateIntToPtr(readableBuffer, voidPtrTy), unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
    119     readableBuffer = iBuilder->CreateIntToPtr(iBuilder->CreateAdd(readableBuffer, unnecessaryBytes), voidPtrTy);
    120     iBuilder->setScalarField("readableBuffer", readableBuffer);
    121     iBuilder->CreateBr(processSegment);
     114    Value * hasPagesToDrop = kb->CreateICmpEQ(unnecessaryBytes, ConstantInt::getNullValue(intAddrTy));
     115    kb->CreateLikelyCondBr(hasPagesToDrop, processSegment, dropPages);
     116
     117    kb->SetInsertPoint(dropPages);
     118    kb->CreateMAdvise(kb->CreateIntToPtr(readableBuffer, voidPtrTy), unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
     119    readableBuffer = kb->CreateIntToPtr(kb->CreateAdd(readableBuffer, unnecessaryBytes), voidPtrTy);
     120    kb->setScalarField("readableBuffer", readableBuffer);
     121    kb->CreateBr(processSegment);
    122122
    123123    // determine whether or not we've exhausted the file buffer
    124     iBuilder->SetInsertPoint(processSegment);
    125     ConstantInt * segmentItems = iBuilder->getSize(mSegmentBlocks * iBuilder->getBitBlockWidth());
    126     Value * const fileSize = iBuilder->getScalarField("fileSize");
    127     Value * const produced = iBuilder->CreateAdd(iBuilder->getProducedItemCount("sourceBuffer"), segmentItems);
    128     Value * const lessThanFullSegment = iBuilder->CreateICmpULT(fileSize, produced);
    129     iBuilder->CreateUnlikelyCondBr(lessThanFullSegment, setTermination, mmapSourceExit);
    130     iBuilder->SetInsertPoint(setTermination);
    131 
    132     iBuilder->setTerminationSignal();
    133     iBuilder->CreateBr(mmapSourceExit);
     124    kb->SetInsertPoint(processSegment);
     125    ConstantInt * segmentItems = kb->getSize(mSegmentBlocks * kb->getBitBlockWidth());
     126    Value * const fileSize = kb->getScalarField("fileSize");
     127    Value * const produced = kb->CreateAdd(kb->getProducedItemCount("sourceBuffer"), segmentItems);
     128    Value * const lessThanFullSegment = kb->CreateICmpULT(fileSize, produced);
     129    kb->CreateUnlikelyCondBr(lessThanFullSegment, setTermination, mmapSourceExit);
     130    kb->SetInsertPoint(setTermination);
     131
     132    kb->setTerminationSignal();
     133    kb->CreateBr(mmapSourceExit);
    134134
    135135    // finally, set the "produced" count to reflect current position in the file
    136     iBuilder->SetInsertPoint(mmapSourceExit);
    137     PHINode * itemsRead = iBuilder->CreatePHI(produced->getType(), 2);
     136    kb->SetInsertPoint(mmapSourceExit);
     137    PHINode * itemsRead = kb->CreatePHI(produced->getType(), 2);
    138138    itemsRead->addIncoming(produced, processSegment);
    139139    itemsRead->addIncoming(fileSize, setTermination);
    140     iBuilder->setProducedItemCount("sourceBuffer", itemsRead);
    141 }
    142 
    143 void MMapSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & iBuilder) {
    144     iBuilder->CreateMUnmap(iBuilder->getBaseAddress("sourceBuffer"), iBuilder->getBufferedSize("sourceBuffer"));
    145 }
    146 
    147 MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, unsigned blocksPerSegment, unsigned codeUnitWidth)
     140    kb->setProducedItemCount("sourceBuffer", itemsRead);
     141}
     142
     143void MMapSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & kb) {
     144    kb->CreateMUnmap(kb->getBaseAddress("sourceBuffer"), kb->getBufferedSize("sourceBuffer"));
     145}
     146
     147MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, unsigned blocksPerSegment, unsigned codeUnitWidth)
    148148: SegmentOrientedKernel("mmap_source" + std::to_string(blocksPerSegment) + "@" + std::to_string(codeUnitWidth),
    149149{},
    150 {Binding{iBuilder->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}},
    151 {Binding{iBuilder->getInt32Ty(), "fileDescriptor"}},
    152 {Binding{iBuilder->getSizeTy(), "fileSize"}}, {Binding{iBuilder->getVoidPtrTy(), "readableBuffer"}})
     150{Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}},
     151{Binding{kb->getInt32Ty(), "fileDescriptor"}},
     152{Binding{kb->getSizeTy(), "fileSize"}}, {Binding{kb->getVoidPtrTy(), "readableBuffer"}})
    153153, mSegmentBlocks(blocksPerSegment)
    154154, mCodeUnitWidth(codeUnitWidth)
     
    159159/// READ SOURCE KERNEL
    160160
    161 void ReadSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & iBuilder) {
    162     ConstantInt * const bufferSize = iBuilder->getSize(64 * getpagesize());
    163     Value * const buffer = iBuilder->CreateCacheAlignedMalloc(bufferSize);
    164     iBuilder->setScalarField("buffer", buffer);
    165     iBuilder->setScalarField("capacity", bufferSize);
    166     iBuilder->setBaseAddress("sourceBuffer", buffer);
    167     iBuilder->setBufferedSize("sourceBuffer", iBuilder->getSize(0));
    168 }
    169 
    170 void ReadSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & iBuilder) {
    171 
    172     ConstantInt * const pageSize = iBuilder->getSize(getpagesize());
    173     PointerType * const codeUnitPtrTy = IntegerType::get(iBuilder->getContext(), mCodeUnitWidth)->getPointerTo();
    174     BasicBlock * const entryBlock = iBuilder->GetInsertBlock();
    175     BasicBlock * const exhaustedBuffer = iBuilder->CreateBasicBlock("ExhaustedBuffer");
    176     BasicBlock * const waitOnConsumers = iBuilder->CreateBasicBlock("WaitOnConsumers");
    177     BasicBlock * const readData = iBuilder->CreateBasicBlock("ReadData");
    178     BasicBlock * const stdInExit = iBuilder->CreateBasicBlock("StdInExit");
    179 
    180     assert(iBuilder->getKernel() == this);
     161void ReadSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & kb) {
     162    const size_t initialBufferSize = 8 * getpagesize() * mCodeUnitWidth;
     163    ConstantInt * const bufferBytes = kb->getSize(initialBufferSize * mCodeUnitWidth/8);
     164    PointerType * const codeUnitPtrTy = IntegerType::get(kb->getContext(), mCodeUnitWidth)->getPointerTo();
     165    Value * const buffer = kb->CreatePointerCast(kb->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy);
     166    kb->setScalarField("buffer", buffer);
     167    kb->setScalarField("capacity", kb->getSize(initialBufferSize));
     168    kb->setBaseAddress("sourceBuffer", buffer);
     169    kb->setBufferedSize("sourceBuffer", kb->getSize(0));
     170}
     171
     172void ReadSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & kb) {
     173
     174    ConstantInt * const readSize = kb->getSize(getpagesize() * 8/mCodeUnitWidth);
     175    PointerType * const codeUnitPtrTy = IntegerType::get(kb->getContext(), mCodeUnitWidth)->getPointerTo();
     176    PointerType * const i8PtrTy = IntegerType::get(kb->getContext(), 8)->getPointerTo();
     177    ConstantInt * const codeUnitBytes = kb->getSize(mCodeUnitWidth/8);
     178    BasicBlock * const entryBlock = kb->GetInsertBlock();
     179    BasicBlock * const exhaustedBuffer = kb->CreateBasicBlock("ExhaustedBuffer");
     180    BasicBlock * const waitOnConsumers = kb->CreateBasicBlock("WaitOnConsumers");
     181    BasicBlock * const readData = kb->CreateBasicBlock("ReadData");
     182    BasicBlock * const stdInExit = kb->CreateBasicBlock("StdInExit");
     183
     184    assert(kb->getKernel() == this);
    181185
    182186    // Check whether we need to read another page of data
    183     ConstantInt * const segmentSize = iBuilder->getSize(mSegmentBlocks * iBuilder->getBitBlockWidth());
    184     Value * bufferedSize = iBuilder->getBufferedSize("sourceBuffer");
    185     Value * const produced = iBuilder->getProducedItemCount("sourceBuffer");
    186     Value * unreadSize = iBuilder->CreateSub(bufferedSize, produced);
    187     iBuilder->CreateUnlikelyCondBr(iBuilder->CreateICmpULT(unreadSize, segmentSize), exhaustedBuffer, stdInExit);
     187    ConstantInt * const segmentSize = kb->getSize(mSegmentBlocks * kb->getBitBlockWidth());
     188    Value * bufferedSize = kb->getBufferedSize("sourceBuffer");
     189    Value * const produced = kb->getProducedItemCount("sourceBuffer");
     190    Value * unreadSize = kb->CreateSub(bufferedSize, produced);
     191    kb->CreateUnlikelyCondBr(kb->CreateICmpULT(unreadSize, segmentSize), exhaustedBuffer, stdInExit);
    188192
    189193    // If so, it checks whether it can simply append another page to the existing buffer or whether
    190194    // we need to perform a copyback.
    191195
    192     iBuilder->SetInsertPoint(exhaustedBuffer);
     196    kb->SetInsertPoint(exhaustedBuffer);
    193197
    194198    // Otherwise, we're going to have to perform a copy back...
    195199
    196     // Let L be the logical buffer address (i.e., the position of the "first byte" of the input stream)
     200    // Let L be the logical buffer address (i.e., the position of the "first code unit" of the input stream)
    197201    // and B be the address pointing to the beginning of our actual buffer. Check whether:
    198202
    199     //     L + produced + pagesize < B + capacity
     203    //     L + produced + readSize < B + capacity
    200204
    201205    // If so, we can append to our existing buffer without impacting any subsequent kernel.
    202206
    203     Value * inputStream = iBuilder->getRawOutputPointer("sourceBuffer", iBuilder->getInt32(0), iBuilder->getInt32(0));
    204     inputStream = iBuilder->CreatePointerCast(inputStream, codeUnitPtrTy);
    205     Value * const originalPtr = iBuilder->CreateGEP(inputStream, produced);
    206 
    207     Value * const buffer = iBuilder->CreatePointerCast(iBuilder->getScalarField("buffer"), codeUnitPtrTy);
    208     Value * const capacity = iBuilder->getScalarField("capacity");
    209 
    210     Value * L = iBuilder->CreateGEP(originalPtr, pageSize);
    211     Value * B = iBuilder->CreateGEP(buffer, capacity);
    212     Value * const canAppend = iBuilder->CreateICmpULT(L, B);
    213     iBuilder->CreateLikelyCondBr(canAppend, readData, waitOnConsumers);
     207    Value * inputStream = kb->getRawOutputPointer("sourceBuffer", kb->getInt32(0), kb->getInt32(0));
     208    Value * const originalPtr = kb->CreateGEP(inputStream, produced);
     209
     210    Value * const buffer = kb->getScalarField("buffer");
     211    Value * const capacity = kb->getScalarField("capacity");
     212
     213    Value * L = kb->CreateGEP(originalPtr, readSize);
     214    Value * B = kb->CreateGEP(buffer, capacity);
     215    Value * const canAppend = kb->CreateICmpULT(L, B);
     216    kb->CreateLikelyCondBr(canAppend, readData, waitOnConsumers);
    214217
    215218    // First wait on any consumers to finish processing then check how much data has been consumed.
    216     iBuilder->SetInsertPoint(waitOnConsumers);
    217     iBuilder->CreateConsumerWait();
     219    kb->SetInsertPoint(waitOnConsumers);
     220    kb->CreateConsumerWait();
    218221
    219222    // Then determine how much data has been consumed and how much needs to be copied back, noting
    220223    // that our "unproduced" data must be block aligned.
    221     const size_t blockAlignment = iBuilder->getBitBlockWidth() / 8;
    222     Constant * const alignmentMask = iBuilder->getSize(-blockAlignment);
    223     Value * const consumed = iBuilder->CreateAnd(iBuilder->getConsumedItemCount("sourceBuffer"), alignmentMask);
    224     Value * const remaining = iBuilder->CreateSub(bufferedSize, consumed);
    225     Value * const unconsumedPtr = iBuilder->CreateGEP(inputStream, consumed);
    226     Value * const consumedMajority = iBuilder->CreateICmpULT(iBuilder->CreateGEP(buffer, remaining), unconsumedPtr);
    227 
    228     BasicBlock * const copyBack = iBuilder->CreateBasicBlock("CopyBack");
    229     BasicBlock * const expandAndCopyBack = iBuilder->CreateBasicBlock("ExpandAndCopyBack");
    230     BasicBlock * const calculateLogicalAddress = iBuilder->CreateBasicBlock("CalculateLogicalAddress");
     224    const size_t blockAlignment = kb->getBitBlockWidth() / 8;
     225    Constant * const alignmentMask = kb->getSize(-(blockAlignment * 8 / mCodeUnitWidth));
     226    Value * const consumed = kb->CreateAnd(kb->getConsumedItemCount("sourceBuffer"), alignmentMask);
     227    Value * const remaining = kb->CreateSub(bufferedSize, consumed);
     228    Value * const unconsumedPtr = kb->CreateGEP(inputStream, consumed);
     229    Value * const consumedMajority = kb->CreateICmpULT(kb->CreateGEP(buffer, remaining), unconsumedPtr);
     230    Value * target = buffer;
     231    Value * source = unconsumedPtr;
     232    Value * toCopy = remaining;
     233    if (mCodeUnitWidth != 8) {
     234        source = kb->CreatePointerCast(unconsumedPtr, i8PtrTy);
     235        toCopy = kb->CreateMul(remaining, codeUnitBytes);
     236    }
     237
     238    BasicBlock * const copyBack = kb->CreateBasicBlock("CopyBack");
     239    BasicBlock * const expandAndCopyBack = kb->CreateBasicBlock("ExpandAndCopyBack");
     240    BasicBlock * const calculateLogicalAddress = kb->CreateBasicBlock("CalculateLogicalAddress");
    231241
    232242    // Have we consumed enough data that we can safely copy back the unconsumed data without needing
    233243    // a temporary buffer? (i.e., B + remaining < L + consumed)
    234     iBuilder->CreateLikelyCondBr(consumedMajority, copyBack, expandAndCopyBack);
    235     iBuilder->SetInsertPoint(copyBack);
    236 
     244    kb->CreateLikelyCondBr(consumedMajority, copyBack, expandAndCopyBack);
     245    kb->SetInsertPoint(copyBack);
    237246    // If so, just copy the data ...
    238     iBuilder->CreateMemCpy(buffer, unconsumedPtr, remaining, 1);
    239     iBuilder->CreateBr(calculateLogicalAddress);
    240 
     247    if (mCodeUnitWidth != 8) {
     248        target = kb->CreatePointerCast(buffer, i8PtrTy);
     249    }
     250    kb->CreateMemCpy(target, source, toCopy, 1);
     251    kb->CreateBr(calculateLogicalAddress);
     252   
    241253    // Otherwise, allocate a buffer with twice the capacity and copy the unconsumed data back into it
    242     iBuilder->SetInsertPoint(expandAndCopyBack);
    243 
    244     Value * const expandedCapacity = iBuilder->CreateShl(capacity, 1);
    245     Value * const expandedBuffer = iBuilder->CreateCacheAlignedMalloc(expandedCapacity);
    246     Value * const expandedBufferPtr = iBuilder->CreatePointerCast(expandedBuffer, codeUnitPtrTy);
    247     iBuilder->CreateMemCpy(expandedBufferPtr, unconsumedPtr, remaining, 1);
    248     iBuilder->CreateFree(buffer);
    249     iBuilder->setScalarField("buffer", expandedBuffer);
    250     iBuilder->setScalarField("capacity", expandedCapacity);
    251     iBuilder->CreateBr(calculateLogicalAddress);
     254    kb->SetInsertPoint(expandAndCopyBack);
     255    Value * const expandedCapacity = kb->CreateShl(capacity, 1);
     256    Value * const expandedBytes = mCodeUnitWidth == 8 ? expandedCapacity : kb->CreateMul(expandedCapacity, codeUnitBytes);
     257    Value * const expandedBuffer = kb->CreatePointerCast(kb->CreateCacheAlignedMalloc(expandedBytes), codeUnitPtrTy);
     258    target = mCodeUnitWidth == 8 ? expandedBuffer : kb->CreatePointerCast(expandedBuffer, i8PtrTy);
     259    kb->CreateMemCpy(target, source, toCopy, 1);
     260    kb->CreateFree(buffer);
     261    kb->setScalarField("buffer", expandedBuffer);
     262    kb->setScalarField("capacity", expandedCapacity);
     263    kb->CreateBr(calculateLogicalAddress);
    252264
    253265    // Update the logical address for this buffer....
    254     iBuilder->SetInsertPoint(calculateLogicalAddress);
    255     PHINode * const baseAddress = iBuilder->CreatePHI(codeUnitPtrTy, 2);
     266    kb->SetInsertPoint(calculateLogicalAddress);
     267    PHINode * const baseAddress = kb->CreatePHI(codeUnitPtrTy, 2);
    256268    baseAddress->addIncoming(buffer, copyBack);
    257     baseAddress->addIncoming(expandedBufferPtr, expandAndCopyBack);
    258     Value * const modifiedPtr = iBuilder->CreateGEP(baseAddress, remaining);
    259     Value * const logicalAddress = iBuilder->CreateGEP(baseAddress, iBuilder->CreateNeg(consumed));
    260     iBuilder->setBaseAddress("sourceBuffer", logicalAddress);
    261     iBuilder->CreateBr(readData);
     269    baseAddress->addIncoming(expandedBuffer, expandAndCopyBack);
     270    Value * const logicalAddress = kb->CreateGEP(baseAddress, kb->CreateNeg(consumed));
     271    Value * const modifiedPtr = kb->CreateGEP(baseAddress, remaining);
     272    kb->setBaseAddress("sourceBuffer", logicalAddress);
     273    kb->CreateBr(readData);
    262274
    263275    // Regardless of whether we're simply appending data or had to allocate a new buffer, read a new page
    264276    // of data into the input source buffer. If we fail to read a full segment ...
    265277    readData->moveAfter(calculateLogicalAddress);
    266     iBuilder->SetInsertPoint(readData);
     278    kb->SetInsertPoint(readData);
    267279    calculateLogicalAddress->moveAfter(calculateLogicalAddress);
    268     PHINode * const addr = iBuilder->CreatePHI(codeUnitPtrTy, 2);
     280    PHINode * const addr = kb->CreatePHI(codeUnitPtrTy, 2);
    269281    addr->addIncoming(originalPtr, exhaustedBuffer);
    270282    addr->addIncoming(modifiedPtr, calculateLogicalAddress);
    271     assert(iBuilder->getKernel() == this);
    272     Value * const fd = iBuilder->getScalarField("fileDescriptor");
    273     Value * bytesRead = iBuilder->CreateReadCall(fd, addr, pageSize);
    274     unreadSize = iBuilder->CreateAdd(unreadSize, bytesRead);
    275     bufferedSize = iBuilder->CreateAdd(bufferedSize, bytesRead);
    276     iBuilder->setBufferedSize("sourceBuffer", bufferedSize);
    277     Value * const exhaustedInputSource = iBuilder->CreateICmpULT(unreadSize, segmentSize);
    278     BasicBlock * const setTermination = iBuilder->CreateBasicBlock("SetTermination");
    279     iBuilder->CreateUnlikelyCondBr(exhaustedInputSource, setTermination, stdInExit);
     283    assert(kb->getKernel() == this);
     284    Value * const fd = kb->getScalarField("fileDescriptor");
     285    Value * toRead = readSize;
     286    if (mCodeUnitWidth != 8) {
     287        toRead = kb->CreateMul(toRead, codeUnitBytes);
     288    }
     289    Value * bytesRead = kb->CreateReadCall(fd, addr, toRead);
     290    Value * itemsRead = bytesRead;
     291    if (mCodeUnitWidth != 8) {
     292        itemsRead = kb->CreateUDiv(bytesRead, codeUnitBytes);
     293    }
     294    unreadSize = kb->CreateAdd(unreadSize, itemsRead);
     295    bufferedSize = kb->CreateAdd(bufferedSize, itemsRead);
     296    kb->setBufferedSize("sourceBuffer", bufferedSize);
     297    Value * const exhaustedInputSource = kb->CreateICmpULT(unreadSize, segmentSize);
     298    BasicBlock * const setTermination = kb->CreateBasicBlock("SetTermination");
     299    kb->CreateUnlikelyCondBr(exhaustedInputSource, setTermination, stdInExit);
    280300
    281301    // ... zero out the remaining bytes and set the termination signal.
    282     iBuilder->SetInsertPoint(setTermination);
    283     Value * const bytesToZero = iBuilder->CreateSub(segmentSize, unreadSize);
    284     iBuilder->CreateMemZero(iBuilder->CreateGEP(addr, unreadSize), bytesToZero);
    285     iBuilder->setTerminationSignal();
    286     iBuilder->CreateBr(stdInExit);
     302    kb->SetInsertPoint(setTermination);
     303    Value * bytesToZero = kb->CreateSub(segmentSize, unreadSize);
     304    Value * unreadPtr = kb->CreateGEP(addr, unreadSize);
     305    bytesToZero = mCodeUnitWidth == 8 ? bytesToZero : kb->CreateMul(bytesToZero, codeUnitBytes);
     306    if (mCodeUnitWidth != 8) {
     307        bytesToZero = kb->CreateMul(bytesToZero, codeUnitBytes);
     308        unreadPtr = kb->CreatePointerCast(unreadPtr, i8PtrTy);
     309    }
     310    kb->CreateMemZero(unreadPtr, bytesToZero);
     311    kb->setTerminationSignal();
     312    kb->CreateBr(stdInExit);
    287313
    288314    // finally add the segment item count to the produced item count to inform the subsequent kernels how
    289315    // much data is available for processing
    290     iBuilder->SetInsertPoint(stdInExit);
     316    kb->SetInsertPoint(stdInExit);
    291317    stdInExit->moveAfter(setTermination);
    292     PHINode * const items = iBuilder->CreatePHI(produced->getType(), 3);
     318    PHINode * const items = kb->CreatePHI(produced->getType(), 3);
    293319    items->addIncoming(segmentSize, entryBlock);
    294320    items->addIncoming(segmentSize, readData);
    295321    items->addIncoming(unreadSize, setTermination);
    296     iBuilder->setProducedItemCount("sourceBuffer", iBuilder->CreateAdd(produced, items));
    297 }
    298 
    299 void ReadSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & iBuilder) {
    300     iBuilder->CreateFree(iBuilder->getScalarField("buffer"));
    301 }
    302 
    303 ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, unsigned blocksPerSegment, unsigned codeUnitWidth)
     322    kb->setProducedItemCount("sourceBuffer", kb->CreateAdd(produced, items));
     323}
     324
     325void ReadSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & kb) {
     326    kb->CreateFree(kb->getScalarField("buffer"));
     327}
     328
     329ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, unsigned blocksPerSegment, unsigned codeUnitWidth)
    304330: SegmentOrientedKernel("read_source"
    305331, {}
    306 , {Binding{iBuilder->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
    307 , {Binding{iBuilder->getInt32Ty(), "fileDescriptor"}}
     332, {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
     333, {Binding{kb->getInt32Ty(), "fileDescriptor"}}
    308334, {}
    309 , {Binding{iBuilder->getVoidPtrTy(), "buffer"}, Binding{iBuilder->getSizeTy(), "capacity"}})
     335, {Binding{IntegerType::get(kb->getContext(), codeUnitWidth)->getPointerTo(), "buffer"}, Binding{kb->getSizeTy(), "capacity"}})
    310336, mSegmentBlocks(blocksPerSegment)
    311337, mCodeUnitWidth(codeUnitWidth) {
     
    315341/// MEMORY SOURCE KERNEL
    316342
    317 void MemorySourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & iBuilder) {
    318     iBuilder->setBaseAddress("sourceBuffer", iBuilder->CreatePointerCast(iBuilder->getScalarField("fileSource"), iBuilder->getVoidPtrTy()));
    319     iBuilder->setBufferedSize("sourceBuffer", iBuilder->getScalarField("fileSize"));
    320 }
    321 
    322 void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & iBuilder) {
    323 
    324     BasicBlock * entryBlock = iBuilder->GetInsertBlock();
    325     BasicBlock * setTermination = iBuilder->CreateBasicBlock("setTermination");
    326     BasicBlock * mmapSourceExit = iBuilder->CreateBasicBlock("sourceExit");
    327     ConstantInt * segmentItems = iBuilder->getSize(mSegmentBlocks * iBuilder->getBitBlockWidth());
    328     Value * fileItems = iBuilder->getScalarField("fileSize");
     343void MemorySourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & kb) {
     344    kb->setBaseAddress("sourceBuffer", kb->CreatePointerCast(kb->getScalarField("fileSource"), kb->getVoidPtrTy()));
     345    kb->setBufferedSize("sourceBuffer", kb->getScalarField("fileSize"));
     346}
     347
     348void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & kb) {
     349
     350    BasicBlock * entryBlock = kb->GetInsertBlock();
     351    BasicBlock * setTermination = kb->CreateBasicBlock("setTermination");
     352    BasicBlock * mmapSourceExit = kb->CreateBasicBlock("sourceExit");
     353    ConstantInt * segmentItems = kb->getSize(mSegmentBlocks * kb->getBitBlockWidth());
     354    Value * fileItems = kb->getScalarField("fileSize");
    329355    if (mCodeUnitWidth > 8) {
    330         fileItems = iBuilder->CreateUDiv(fileItems, iBuilder->getSize(mCodeUnitWidth / 8));
    331     }
    332     Value * produced = iBuilder->getProducedItemCount("sourceBuffer");
    333     produced = iBuilder->CreateAdd(produced, segmentItems);
    334     Value * lessThanFullSegment = iBuilder->CreateICmpULT(fileItems, produced);
    335     iBuilder->CreateCondBr(lessThanFullSegment, setTermination, mmapSourceExit);
    336     iBuilder->SetInsertPoint(setTermination);
    337     iBuilder->setTerminationSignal();
    338     iBuilder->CreateBr(mmapSourceExit);
    339 
    340     iBuilder->SetInsertPoint(mmapSourceExit);
    341 
    342     PHINode * itemsRead = iBuilder->CreatePHI(produced->getType(), 2);
     356        fileItems = kb->CreateUDiv(fileItems, kb->getSize(mCodeUnitWidth / 8));
     357    }
     358    Value * produced = kb->getProducedItemCount("sourceBuffer");
     359    produced = kb->CreateAdd(produced, segmentItems);
     360    Value * lessThanFullSegment = kb->CreateICmpULT(fileItems, produced);
     361    kb->CreateCondBr(lessThanFullSegment, setTermination, mmapSourceExit);
     362    kb->SetInsertPoint(setTermination);
     363    kb->setTerminationSignal();
     364    kb->CreateBr(mmapSourceExit);
     365
     366    kb->SetInsertPoint(mmapSourceExit);
     367
     368    PHINode * itemsRead = kb->CreatePHI(produced->getType(), 2);
    343369    itemsRead->addIncoming(produced, entryBlock);
    344370    itemsRead->addIncoming(fileItems, setTermination);
    345     iBuilder->setProducedItemCount("sourceBuffer", itemsRead);
    346 }
    347 
    348 MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, Type * type, unsigned blocksPerSegment, unsigned codeUnitWidth)
     371    kb->setProducedItemCount("sourceBuffer", itemsRead);
     372}
     373
     374MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, Type * type, unsigned blocksPerSegment, unsigned codeUnitWidth)
    349375: SegmentOrientedKernel("memory_source",
    350376    {},
    351     {Binding{iBuilder->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}},
    352     {Binding{cast<PointerType>(type), "fileSource"}, Binding{iBuilder->getSizeTy(), "fileSize"}}, {}, {})
     377    {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}},
     378    {Binding{cast<PointerType>(type), "fileSource"}, Binding{kb->getSizeTy(), "fileSize"}}, {}, {})
    353379, mSegmentBlocks(blocksPerSegment)
    354380, mCodeUnitWidth(codeUnitWidth) {
Note: See TracChangeset for help on using the changeset viewer.