source: icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp @ 6047

Last change on this file since 6047 was 6047, checked in by nmedfort, 15 months ago

Major refactoring of buffer types. Static buffers replace Circular and CircularCopyback?. External buffers unify Source/External?.

File size: 22.6 KB
RevLine 
[5429]1/*
[5927]2 *  Copyright (c) 2018 International Characters.
[5429]3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5#include "source_kernel.h"
[5436]6#include <kernels/kernel_builder.h>
[5429]7#include <kernels/streamset.h>
[5493]8#include <llvm/IR/Module.h>
[5429]9#include <sys/stat.h>
10#include <fcntl.h>
[5856]11#include <toolchain/toolchain.h>
[5429]12
13using namespace llvm;
14
[5915]15extern "C" uint64_t file_size(const uint32_t fd) {
[5429]16    struct stat st;
17    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
18        st.st_size = 0;
19    }
20    return st.st_size;
21}
22
23namespace kernel {
24
25/// MMAP SOURCE KERNEL
26
[5985]27Function * MMapSourceKernel::linkFileSizeMethod(const std::unique_ptr<kernel::KernelBuilder> & b) {
28    return b->LinkFunction("file_size", &file_size);
[5429]29}
30
[5985]31void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
[5856]32
[5985]33    BasicBlock * const emptyFile = b->CreateBasicBlock("emptyFile");
34    BasicBlock * const nonEmptyFile = b->CreateBasicBlock("NonEmptyFile");
35    BasicBlock * const exit = b->CreateBasicBlock("Exit");
36    IntegerType * const sizeTy = b->getSizeTy();
[6025]37    ConstantInt * const PAGE_SIZE = b->getSize(getpagesize());
[5985]38    Value * const fd = b->getScalarField("fileDescriptor");
[5706]39    assert (fileSizeMethod);
[5985]40    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(fileSizeMethod, fd), sizeTy);
41    b->CreateLikelyCondBr(b->CreateIsNotNull(fileSize), nonEmptyFile, emptyFile);
[5856]42
[5985]43    b->SetInsertPoint(nonEmptyFile);
44    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
45    Value * const fileBuffer = b->CreatePointerCast(b->CreateFileSourceMMap(fd, fileSize), codeUnitPtrTy);
46    b->setScalarField("buffer", fileBuffer);
47    b->setBaseAddress("sourceBuffer", fileBuffer);
48    b->CreateMAdvise(fileBuffer, fileSize, CBuilder::ADVICE_WILLNEED);
49    if (LLVM_UNLIKELY(codeUnitWidth > 8)) {
50        fileSize = b->CreateUDiv(fileSize, b->getSize(codeUnitWidth / 8));
[5429]51    }
[5985]52    b->setScalarField("fileSize", fileSize);
53    b->CreateBr(exit);
[5429]54
[5985]55    b->SetInsertPoint(emptyFile);
56    Value * const emptyFilePtr = b->CreatePointerCast(b->CreateAnonymousMMap(PAGE_SIZE), codeUnitPtrTy);
57    b->setScalarField("buffer", emptyFilePtr);
58    b->setBaseAddress("sourceBuffer", emptyFilePtr);
[6047]59    b->setScalarField("fileSize", PAGE_SIZE);
[5985]60    b->setTerminationSignal();
61    b->CreateBr(exit);
[5429]62
[5985]63    b->SetInsertPoint(exit);
[5429]64}
65
[6008]66
[5985]67void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
[5429]68
[5985]69    BasicBlock * const dropPages = b->CreateBasicBlock("dropPages");
70    BasicBlock * const checkRemaining = b->CreateBasicBlock("checkRemaining");
71    BasicBlock * const setTermination = b->CreateBasicBlock("setTermination");
72    BasicBlock * const exit = b->CreateBasicBlock("mmapSourceExit");
[5429]73
[5985]74    Constant * const PAGE_SIZE = b->getSize(getpagesize());
[6008]75    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
76    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
[5757]77
[6008]78    Value * const fileItems = b->getScalarField("fileSize");
[5985]79    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
[6008]80    Value * const consumedBytes = b->CreateMul(consumedItems, CODE_UNIT_BYTES);
[5985]81    Value * const consumedPageOffset = b->CreateAnd(consumedBytes, ConstantExpr::getNeg(PAGE_SIZE));
82    Value * const consumedBuffer = b->getRawOutputPointer("sourceBuffer", consumedPageOffset);
83    Value * const readableBuffer = b->getScalarField("buffer");
84    Value * const unnecessaryBytes = b->CreatePtrDiff(consumedBuffer, readableBuffer);
[5454]85
[5429]86    // avoid calling madvise unless an actual page table change could occur
[5985]87    b->CreateLikelyCondBr(b->CreateIsNotNull(unnecessaryBytes), dropPages, checkRemaining);
[5429]88
[5985]89    b->SetInsertPoint(dropPages);
[5757]90    // instruct the OS that it can safely drop any fully consumed pages
[5985]91    b->CreateMAdvise(readableBuffer, unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
92    b->setScalarField("buffer", b->CreateGEP(readableBuffer, unnecessaryBytes));
93    b->CreateBr(checkRemaining);
[5429]94
95    // determine whether or not we've exhausted the file buffer
[5985]96    b->SetInsertPoint(checkRemaining);
97    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
[6008]98    Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE);
99    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
[5985]100    b->CreateUnlikelyCondBr(lastPage, setTermination, exit);
[5706]101
[6008]102    // If this is the last page, create a temporary buffer of up to two pages size, copy the unconsumed data
103    // and zero any bytes that are not used.
[5985]104    b->SetInsertPoint(setTermination);
[6008]105    Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
106    Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
107    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
[6010]108    Value * const unconsumedBytes = b->CreateTrunc(b->CreatePtrDiff(readEnd, readStart), b->getSizeTy());
[6008]109    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
110    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
111    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
112    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
113    // get the difference between our base and from position then compute an offsetted temporary buffer address
114    Value * const base = b->getBaseAddress("sourceBuffer");
115    Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
116    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
117    b->CreateConsumerWait();
118    // Unmap the file buffer and set the temporary buffer as the new source buffer
119    Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES);
120    b->CreateMUnmap(base, fileSize);
121    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
122    b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
123    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
[5985]124    b->setTerminationSignal();
[6008]125    BasicBlock * const terminationExit = b->GetInsertBlock();
[5985]126    b->CreateBr(exit);
[5429]127
128    // finally, set the "produced" count to reflect current position in the file
[5985]129    b->SetInsertPoint(exit);
130    PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
131    newProducedItems->addIncoming(nextProducedItems, checkRemaining);
[6008]132    newProducedItems->addIncoming(fileItems, terminationExit);
[5985]133    b->setProducedItemCount("sourceBuffer", newProducedItems);
[5429]134}
135
[6008]136void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
137    b->CreateFree(b->getScalarField("buffer"));
[5429]138}
139
140/// READ SOURCE KERNEL
141
[5856]142void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
[5757]143    const unsigned pageSize = getpagesize();
[5856]144    const auto bufferSize = std::max(pageSize * 8, codegen::SegmentSize * stride  * 4);
[5757]145    ConstantInt * const bufferItems = b->getSize(bufferSize);
146    const auto codeUnitSize = codeUnitWidth / 8;
147    ConstantInt * const bufferBytes = b->getSize(bufferSize * codeUnitSize);
148    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
149    Value * const buffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy);
150    b->setBaseAddress("sourceBuffer", buffer);
151    b->setScalarField("buffer", buffer);
152    b->setCapacity("sourceBuffer", bufferItems);
[6047]153    b->setScalarField("fileSize", b->getSize(0));
[5429]154}
155
[5856]156void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
[5429]157
[5757]158    const unsigned pageSize = getpagesize();
[5856]159    ConstantInt * const itemsToRead = b->getSize(std::max(pageSize, codegen::SegmentSize * stride * 2));
[5757]160    ConstantInt * const codeUnitBytes = b->getSize(codeUnitWidth / 8);
[5856]161    ConstantInt * const itemsPerSegment = b->getSize(codegen::SegmentSize * stride);
[5429]162
[5757]163    BasicBlock * const entry = b->GetInsertBlock();
164    BasicBlock * const checkData = b->CreateBasicBlock("CheckData");
165    BasicBlock * const moveData = b->CreateBasicBlock("MoveData");
166    BasicBlock * const prepareBuffer = b->CreateBasicBlock("PrepareBuffer");
167    BasicBlock * const readData = b->CreateBasicBlock("ReadData");
168    BasicBlock * const setTermination = b->CreateBasicBlock("SetTermination");
169    BasicBlock * const readExit = b->CreateBasicBlock("ReadExit");
[5706]170
[5761]171    // Do we have enough unread data to support one segment?
[5757]172    Value * const produced = b->getProducedItemCount("sourceBuffer");
[6047]173    Value * const buffered = b->getScalarField("fileSize");
[5757]174    Value * const itemsPending = b->CreateAdd(produced, itemsPerSegment);
[5429]175
[5757]176    b->CreateLikelyCondBr(b->CreateICmpULT(itemsPending, buffered), readExit, checkData);
[5429]177
[5757]178    // Can we append to our existing buffer without impacting any subsequent kernel?
179    b->SetInsertPoint(checkData);
180    Value * const capacity = b->getCapacity("sourceBuffer");
181    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", b->CreateAdd(buffered, itemsToRead));
182    Value * const baseBuffer = b->getScalarField("buffer");
183    Value * const bufferLimit = b->CreateGEP(baseBuffer, capacity);
184    b->CreateLikelyCondBr(b->CreateICmpULE(readEnd, bufferLimit), readData, moveData);
[5429]185
186    // First wait on any consumers to finish processing then check how much data has been consumed.
[5757]187    b->SetInsertPoint(moveData);
188    b->CreateConsumerWait();
[5464]189
[5429]190    // Then determine how much data has been consumed and how much needs to be copied back, noting
191    // that our "unproduced" data must be block aligned.
[5757]192    BasicBlock * const copyBack = b->CreateBasicBlock("CopyBack");
193    BasicBlock * const expandAndCopyBack = b->CreateBasicBlock("ExpandAndCopyBack");
[5464]194
[5757]195    const auto blockSize = b->getBitBlockWidth() / 8;
196    Constant * const blockSizeAlignmentMask = ConstantExpr::getNeg(b->getSize(blockSize));
197    Value * const consumed = b->getConsumedItemCount("sourceBuffer");
198    Value * const offset = b->CreateAnd(consumed, blockSizeAlignmentMask);
199    Value * const unreadData = b->getRawOutputPointer("sourceBuffer", offset);
200    Value * const remainingItems = b->CreateSub(buffered, offset);
201    Value * const remainingBytes = b->CreateMul(remainingItems, codeUnitBytes);
[5493]202
[5757]203    // Have we consumed enough data that we can safely copy back the unconsumed data without needing a temporary buffer?
204    Value * const canCopy = b->CreateICmpULT(b->CreateGEP(baseBuffer, remainingItems), b->getRawOutputPointer("sourceBuffer", offset));
205    b->CreateLikelyCondBr(canCopy, copyBack, expandAndCopyBack);
206
[5429]207    // If so, just copy the data ...
[5757]208    b->SetInsertPoint(copyBack);
209    b->CreateMemCpy(baseBuffer, unreadData, remainingBytes, blockSize);
210    b->CreateBr(prepareBuffer);
211
[5429]212    // Otherwise, allocate a buffer with twice the capacity and copy the unconsumed data back into it
[5757]213    b->SetInsertPoint(expandAndCopyBack);
214    Value * const expandedCapacity = b->CreateShl(capacity, 1);
215    Value * const expandedBytes = b->CreateMul(expandedCapacity, codeUnitBytes);
216    Value * const expandedBuffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(expandedBytes), unreadData->getType());
217    b->CreateMemCpy(expandedBuffer, unreadData, remainingBytes, blockSize);
218    b->setScalarField("buffer", expandedBuffer);
[5856]219    b->setCapacity("sourceBuffer", expandedCapacity);
[5761]220    b->CreateFree(baseBuffer);
[5757]221    b->CreateBr(prepareBuffer);
[5497]222
[5757]223    b->SetInsertPoint(prepareBuffer);
224    PHINode * newBaseBuffer = b->CreatePHI(baseBuffer->getType(), 2);
225    newBaseBuffer->addIncoming(baseBuffer, copyBack);
226    newBaseBuffer->addIncoming(expandedBuffer, expandAndCopyBack);
227    b->setBaseAddress("sourceBuffer", b->CreateGEP(newBaseBuffer, b->CreateNeg(offset)));
228    b->CreateBr(readData);
[5493]229
[5429]230    // Regardless of whether we're simply appending data or had to allocate a new buffer, read a new page
[5757]231    // of data into the input source buffer. If we fail to read a full page ...
232    b->SetInsertPoint(readData);
233    Value * const sourceBuffer = b->getRawOutputPointer("sourceBuffer", buffered);
234    Value * const fd = b->getScalarField("fileDescriptor");
235    Constant * const bytesToRead = ConstantExpr::getMul(itemsToRead, codeUnitBytes);
236    Value * const bytesRead = b->CreateReadCall(fd, sourceBuffer, bytesToRead);
237    Value * const itemsRead = b->CreateUDiv(bytesRead, codeUnitBytes);
238    Value * const itemsBuffered = b->CreateAdd(buffered, itemsRead);
[6047]239    b->setScalarField("fileSize", itemsBuffered);
[5757]240    b->CreateUnlikelyCondBr(b->CreateICmpULT(itemsBuffered, itemsPending), setTermination, readExit);
[5706]241
[5856]242    // ... set the termination signal.
[5757]243    b->SetInsertPoint(setTermination);
244    Value * const bytesToZero = b->CreateMul(b->CreateSub(itemsPending, itemsBuffered), codeUnitBytes);
245    b->CreateMemZero(b->getRawOutputPointer("sourceBuffer", itemsBuffered), bytesToZero);
246    b->setTerminationSignal();
247    b->CreateBr(readExit);
[5429]248
[5757]249    b->SetInsertPoint(readExit);
250    PHINode * const itemsProduced = b->CreatePHI(itemsPending->getType(), 3);
251    itemsProduced->addIncoming(itemsPending, entry);
252    itemsProduced->addIncoming(itemsPending, readData);
253    itemsProduced->addIncoming(itemsBuffered, setTermination);
254    b->setProducedItemCount("sourceBuffer", itemsProduced);
[5429]255}
256
[5985]257void ReadSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
258    b->CreateFree(b->getScalarField("buffer"));
[5429]259}
260
[5706]261/// Hybrid MMap/Read source kernel
262
[5985]263void FDSourceKernel::linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & b) {
264    mFileSizeFunction = MMapSourceKernel::linkFileSizeMethod(b);
[5693]265}
266
[5985]267void FDSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
268    BasicBlock * finalizeRead = b->CreateBasicBlock("finalizeRead");
269    BasicBlock * finalizeMMap = b->CreateBasicBlock("finalizeMMap");
270    BasicBlock * finalizeDone = b->CreateBasicBlock("finalizeDone");
271    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), finalizeMMap, finalizeRead);
272    b->SetInsertPoint(finalizeMMap);
[6008]273    MMapSourceKernel::freeBuffer(b);
[5985]274    b->CreateBr(finalizeDone);
275    b->SetInsertPoint(finalizeRead);
276    ReadSourceKernel::freeBuffer(b);
277    b->CreateBr(finalizeDone);
278    b->SetInsertPoint(finalizeDone);
[5693]279}
280
[5985]281void FDSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
282    BasicBlock * initializeRead = b->CreateBasicBlock("initializeRead");
283    BasicBlock * tryMMap = b->CreateBasicBlock("tryMMap");
284    BasicBlock * initializeMMap = b->CreateBasicBlock("initializeMMap");
285    BasicBlock * initializeDone = b->CreateBasicBlock("initializeDone");
286
[5927]287    // The source will use MMapSource or readSoure kernel logic depending on the useMMap
288    // parameter, possibly overridden.
[5985]289    Value * useMMap = b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty());
[5927]290    // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic.
[5985]291    Value * fd = b->getScalarField("fileDescriptor");
292    useMMap = b->CreateAnd(useMMap, b->CreateICmpNE(fd, b->getInt32(STDIN_FILENO)));
293    b->CreateCondBr(useMMap, tryMMap, initializeRead);
294
295    b->SetInsertPoint(tryMMap);
[5927]296    // If the fileSize is 0, we may have a virtual file such as /proc/cpuinfo
[5985]297    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(mFileSizeFunction, fd), b->getSizeTy());
298    useMMap = b->CreateICmpNE(fileSize, b->getSize(0));
299    b->CreateCondBr(useMMap, initializeMMap, initializeRead);
300    b->SetInsertPoint(initializeMMap);
301    MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, b);
302    b->CreateBr(initializeDone);
[5927]303
[5985]304    b->SetInsertPoint(initializeRead);
[5927]305    // Ensure that readSource logic is used throughout.
[5985]306    b->setScalarField("useMMap", b->getInt8(0));
307    ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, getStride(), b);
308    b->CreateBr(initializeDone);
309    b->SetInsertPoint(initializeDone);
[5693]310}
311
[5985]312void FDSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
313    BasicBlock * DoSegmentRead = b->CreateBasicBlock("DoSegmentRead");
314    BasicBlock * DoSegmentMMap = b->CreateBasicBlock("DoSegmentMMap");
315    BasicBlock * DoSegmentDone = b->CreateBasicBlock("DoSegmentDone");
316    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), DoSegmentMMap, DoSegmentRead);
317    b->SetInsertPoint(DoSegmentMMap);
318    MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, b);
319    b->CreateBr(DoSegmentDone);
320    b->SetInsertPoint(DoSegmentRead);
321    ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, getStride(), b);
322    b->CreateBr(DoSegmentDone);
323    b->SetInsertPoint(DoSegmentDone);
[5693]324}
325
[5429]326/// MEMORY SOURCE KERNEL
327
[5985]328void MemorySourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
329    Value * const fileSource = b->getScalarField("fileSource");
330    b->setBaseAddress("sourceBuffer", fileSource);
331    Value * const fileSize = b->getScalarField("fileSize");
[6008]332    b->setCapacity("sourceBuffer", fileSize);
333    if (mStreamSetCount > 1) {
334        b->setProducedItemCount("sourceBuffer", fileSize);
335        b->setTerminationSignal();
336    }
[5429]337}
338
[5985]339void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
[6008]340    if (mStreamSetCount == 1) {
341        Constant * const PAGE_SIZE = b->getSize(getStride());
342        Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
[5429]343
[6008]344        BasicBlock * const entry = b->GetInsertBlock();
345        BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary");
346        BasicBlock * const exit = b->CreateBasicBlock("exit");
347
348        Value * const fileItems = b->getScalarField("fileSize");
349        Value * const producedItems = b->getProducedItemCount("sourceBuffer");
350        Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE);
351        Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
352        b->CreateUnlikelyCondBr(lastPage, createTemporary, exit);
353
354        b->SetInsertPoint(createTemporary);
355        Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
356        Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
357        Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
358        Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
[6010]359        Value * const unconsumedBytes = b->CreateTrunc(b->CreatePtrDiff(readEnd, readStart), b->getSizeTy());
[6008]360        Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
361        Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
362        b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
363        b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
364
365        // get the difference between our base and from position then compute an offsetted temporary buffer address
366        Value * const base = b->getBaseAddress("sourceBuffer");
367        Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
368        Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
369        b->CreateConsumerWait();
370
371        // set the temporary buffer as the new source buffer
372        PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
373        b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
374        b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
375        b->setTerminationSignal();
376        BasicBlock * const terminationExit = b->GetInsertBlock();
377        b->CreateBr(exit);
378
379        b->SetInsertPoint(exit);
380        PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
381        newProducedItems->addIncoming(nextProducedItems, entry);
382        newProducedItems->addIncoming(fileItems, terminationExit);
383        b->setProducedItemCount("sourceBuffer", newProducedItems);
384    }
[5856]385}
[5761]386
[6008]387void MemorySourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
388    if (mStreamSetCount == 1) {
389        b->CreateFree(b->getScalarField("buffer"));
390    }
391}
392
[5856]393MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
[5985]394: SegmentOrientedKernel("mmap_source@" + std::to_string(codeUnitWidth)
[6047]395// input streams
[5985]396, {}
[6047]397// output streams
[5985]398, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
[6047]399// input scalars
[5985]400, {Binding{b->getInt32Ty(), "fileDescriptor"}}
[6047]401// output scalars
402, {}
403// internal scalars
404, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}
405,  Binding{b->getSizeTy(), "fileSize"}})
[5856]406, mCodeUnitWidth(codeUnitWidth)
407, mFileSizeFunction(nullptr) {
[5883]408    addAttribute(MustExplicitlyTerminate());
[6008]409    setStride(getpagesize());
[5856]410}
[5761]411
[5856]412ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
413: SegmentOrientedKernel("read_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(codeUnitWidth)
[6047]414// input streams
[5856]415, {}
[6047]416// output streams
[5985]417, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
[6047]418// input scalars
[5856]419, {Binding{b->getInt32Ty(), "fileDescriptor"}}
[6047]420// output scalars
[5856]421, {}
[6047]422// internal scalars
423, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}
424,  Binding{b->getSizeTy(), "fileSize"}})
[5856]425, mCodeUnitWidth(codeUnitWidth) {
[5883]426    addAttribute(MustExplicitlyTerminate());
[6008]427    setStride(getpagesize());
[5856]428}
[5429]429
[5856]430
[5985]431FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
[5856]432: SegmentOrientedKernel("FD_source@" + std::to_string(codeUnitWidth)
[6047]433// input streams
[5856]434, {}
[6008]435// output stream
[5985]436, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
437// input scalar
438, {Binding{b->getInt8Ty(), "useMMap"}, Binding{b->getInt32Ty(), "fileDescriptor"}}
[5856]439, {}
[6047]440// internal scalars
441, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"},
442   Binding{b->getSizeTy(), "fileSize"}})
[5856]443, mCodeUnitWidth(codeUnitWidth)
444, mFileSizeFunction(nullptr) {
[5883]445    addAttribute(MustExplicitlyTerminate());
[6008]446    setStride(getpagesize());
[5429]447}
448
[6008]449MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned streamSetCount, const unsigned codeUnitWidth)
450: SegmentOrientedKernel("memory_source@" + std::to_string(streamSetCount) + ":" + std::to_string(codeUnitWidth),
[6047]451// input streams
[6008]452{},
453// output stream
454{Binding{b->getStreamSetTy(streamSetCount, codeUnitWidth), "sourceBuffer"}},
455// input scalar
456{Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "fileSource"}, Binding{b->getSizeTy(), "fileSize"}},
457{},
458// internal scalar
459{Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
460, mStreamSetCount(streamSetCount)
[5429]461, mCodeUnitWidth(codeUnitWidth) {
[5883]462    addAttribute(MustExplicitlyTerminate());
[6008]463    setStride(getpagesize());
[5429]464}
465
466}
Note: See TracBrowser for help on using the repository browser.