source: icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp @ 6241

Last change on this file since 6241 was 6241, checked in by nmedfort, 7 months ago

Bug fix for kernel state malloc; minor optimization of pipeline state; grep printing bug fix

File size: 25.1 KB
Line 
1/*
2 *  Copyright (c) 2018 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5#include "source_kernel.h"
6#include <kernels/kernel_builder.h>
7#include <kernels/streamset.h>
8#include <llvm/IR/Module.h>
9#include <sys/stat.h>
10#include <fcntl.h>
11#include <toolchain/toolchain.h>
12#include <boost/interprocess/mapped_region.hpp>
13
14using namespace llvm;
15
16inline unsigned getPageSize() {
17    return boost::interprocess::mapped_region::get_page_size();
18}
19
20extern "C" uint64_t file_size(const uint32_t fd) {
21    struct stat st;
22    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
23        st.st_size = 0;
24    }
25    return st.st_size;
26}
27
28namespace kernel {
29
30/// MMAP SOURCE KERNEL
31
32Function * MMapSourceKernel::linkFileSizeMethod(const std::unique_ptr<kernel::KernelBuilder> & b) {
33    return b->LinkFunction("file_size", file_size);
34}
35
36void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
37
38    BasicBlock * const emptyFile = b->CreateBasicBlock("emptyFile");
39    BasicBlock * const nonEmptyFile = b->CreateBasicBlock("NonEmptyFile");
40    BasicBlock * const exit = b->CreateBasicBlock("Exit");
41    IntegerType * const sizeTy = b->getSizeTy();
42    ConstantInt * const STRIDE_SIZE = b->getSize(stride);
43    Constant * const PAGE_ITEMS = b->getSize(stride /(codeUnitWidth/8));
44    Value * const fd = b->getScalarField("fileDescriptor");
45
46    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
47    b->setScalarField("ancillaryBuffer", ConstantPointerNull::get(codeUnitPtrTy));
48    assert (fileSizeMethod);
49    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(fileSizeMethod, fd), sizeTy);
50    b->CreateLikelyCondBr(b->CreateIsNotNull(fileSize), nonEmptyFile, emptyFile);
51
52    b->SetInsertPoint(nonEmptyFile);
53    Value * const fileBuffer = b->CreatePointerCast(b->CreateFileSourceMMap(fd, fileSize), codeUnitPtrTy);
54    b->setScalarField("buffer", fileBuffer);
55    b->setBaseAddress("sourceBuffer", fileBuffer);
56    b->CreateMAdvise(fileBuffer, fileSize, CBuilder::ADVICE_WILLNEED);
57    Value * fileItems = fileSize;
58    if (LLVM_UNLIKELY(codeUnitWidth > 8)) {
59        fileItems = b->CreateUDiv(fileSize, b->getSize(codeUnitWidth / 8));
60    }
61    b->setScalarField("fileItems", fileItems);
62    b->setCapacity("sourceBuffer", fileItems);
63    b->CreateBr(exit);
64
65    b->SetInsertPoint(emptyFile);
66    Value * const emptyFilePtr = b->CreatePointerCast(b->CreateAnonymousMMap(STRIDE_SIZE), codeUnitPtrTy);
67    b->setScalarField("buffer", emptyFilePtr);
68    b->setBaseAddress("sourceBuffer", emptyFilePtr);
69    b->setScalarField("fileItems", PAGE_ITEMS);
70    b->setTerminationSignal();
71    b->CreateBr(exit);
72
73    b->SetInsertPoint(exit);
74}
75
76
77void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
78
79    BasicBlock * const dropPages = b->CreateBasicBlock("dropPages");
80    BasicBlock * const checkRemaining = b->CreateBasicBlock("checkRemaining");
81    BasicBlock * const setTermination = b->CreateBasicBlock("setTermination");
82    BasicBlock * const exit = b->CreateBasicBlock("mmapSourceExit");
83
84    Constant * const MMAP_PAGE_SIZE = b->getSize(getPageSize());
85
86    Constant * const STRIDE_SIZE = b->getSize(stride);
87
88    Constant * const PAGE_ITEMS = b->getSize((8 * stride) / codeUnitWidth);
89    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
90    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
91    Constant * const PADDING_SIZE = b->getSize(b->getBitBlockWidth() * codeUnitWidth / 8);
92
93    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
94    Value * const consumedBytes = b->CreateMul(consumedItems, CODE_UNIT_BYTES);
95    Value * const consumedPageOffset = b->CreateAnd(consumedBytes, ConstantExpr::getNeg(MMAP_PAGE_SIZE));
96    Value * const consumedBuffer = b->getRawOutputPointer("sourceBuffer", consumedPageOffset);
97    Value * const readableBuffer = b->getScalarField("buffer");
98    Value * const unnecessaryBytes = b->CreatePtrDiff(consumedBuffer, readableBuffer);
99
100    // avoid calling madvise unless an actual page table change could occur
101    b->CreateLikelyCondBr(b->CreateIsNotNull(unnecessaryBytes), dropPages, checkRemaining);
102
103    b->SetInsertPoint(dropPages);
104    // instruct the OS that it can safely drop any fully consumed pages
105    b->CreateMAdvise(readableBuffer, unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
106    //b->setScalarField("buffer", b->CreateGEP(readableBuffer, unnecessaryBytes));
107    b->CreateBr(checkRemaining);
108
109    // determine whether or not we've exhausted the file buffer
110    b->SetInsertPoint(checkRemaining);
111    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
112    Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_ITEMS);
113    Value * const fileItems = b->getScalarField("fileItems");
114    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
115    b->CreateUnlikelyCondBr(lastPage, setTermination, exit);
116
117    // If this is the last page, create a temporary buffer of up to two pages size, copy the unconsumed data
118    // and zero any bytes that are not used.
119    b->SetInsertPoint(setTermination);
120    Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
121    Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
122    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
123
124    DataLayout DL(b->getModule());
125    Type * const intPtrTy = DL.getIntPtrType(readEnd->getType());
126    Value * const readEndInt = b->CreatePtrToInt(readEnd, intPtrTy);
127    Value * const readStartInt = b->CreatePtrToInt(readStart, intPtrTy);
128    Value * unconsumedBytes = b->CreateSub(readEndInt, readStartInt);
129    unconsumedBytes = b->CreateTrunc(unconsumedBytes, b->getSizeTy());
130    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, PADDING_SIZE), STRIDE_SIZE);
131    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
132    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
133    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
134    // get the difference between our base and from position then compute an offsetted temporary buffer address
135    Value * const base = b->getBaseAddress("sourceBuffer");
136    Value * const baseInt = b->CreatePtrToInt(base, intPtrTy);
137    Value * const diff = b->CreateSub(baseInt, readStartInt);
138    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
139    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
140    b->setScalarField("ancillaryBuffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
141    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
142    b->setTerminationSignal();
143
144
145    BasicBlock * const terminationExit = b->GetInsertBlock();
146    b->CreateBr(exit);
147
148    // finally, set the "produced" count to reflect current position in the file
149    b->SetInsertPoint(exit);
150    PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
151    newProducedItems->addIncoming(nextProducedItems, checkRemaining);
152    newProducedItems->addIncoming(fileItems, terminationExit);
153    b->setProducedItemCount("sourceBuffer", newProducedItems);
154}
155void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b, const unsigned codeUnitWidth) {
156    b->CreateFree(b->getScalarField("ancillaryBuffer"));
157    Value * const fileItems = b->getScalarField("fileItems");
158    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
159    Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES);
160    b->CreateMUnmap(b->getScalarField("buffer"), fileSize);
161}
162
163/// READ SOURCE KERNEL
164
165void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
166    ConstantInt * const bufferItems = b->getSize(stride * 4);
167    const auto codeUnitSize = codeUnitWidth / 8;
168    ConstantInt * const bufferBytes = b->getSize(stride * 4 * codeUnitSize);
169    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
170    Value * const buffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy);
171    b->setBaseAddress("sourceBuffer", buffer);
172    b->setScalarField("ancillaryBuffer", ConstantPointerNull::get(codeUnitPtrTy));
173    b->setScalarField("buffer", buffer);
174    b->setCapacity("sourceBuffer", bufferItems);
175}
176
177void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
178
179    ConstantInt * const itemsToRead = b->getSize(stride);
180    ConstantInt * const codeUnitBytes = b->getSize(codeUnitWidth / 8);
181
182    BasicBlock * const moveData = b->CreateBasicBlock("MoveData");
183    BasicBlock * const prepareBuffer = b->CreateBasicBlock("PrepareBuffer");
184    BasicBlock * const readData = b->CreateBasicBlock("ReadData");
185    BasicBlock * const setTermination = b->CreateBasicBlock("SetTermination");
186    BasicBlock * const readExit = b->CreateBasicBlock("ReadExit");
187
188    // Can we append to our existing buffer without impacting any subsequent kernel?
189    Value * const produced = b->getProducedItemCount("sourceBuffer");
190    Value * const itemsPending = b->CreateAdd(produced, itemsToRead);
191    Value * const capacity = b->getCapacity("sourceBuffer");
192    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", itemsPending);
193    Value * const baseBuffer = b->getScalarField("buffer");
194    Value * const bufferLimit = b->CreateGEP(baseBuffer, capacity);
195    b->CreateLikelyCondBr(b->CreateICmpULE(readEnd, bufferLimit), readData, moveData);
196
197    // No. If we can copy the unconsumed data back to the start of the buffer *and* write a full
198    // segment of data without overwriting the currently unconsumed data, do so since it won't
199    // affect any potential consumer that could be using the "stale" output base pointer.
200    b->SetInsertPoint(moveData);
201
202    // Determine how much data has been consumed and how much needs to be copied back, noting
203    // that our "unproduced" data must be block aligned.
204    BasicBlock * const copyBack = b->CreateBasicBlock("CopyBack");
205    BasicBlock * const expandAndCopyBack = b->CreateBasicBlock("ExpandAndCopyBack");
206    const auto blockSize = b->getBitBlockWidth() / 8;
207    Constant * const blockSizeAlignmentMask = ConstantExpr::getNeg(b->getSize(blockSize));
208    Value * const consumed = b->getConsumedItemCount("sourceBuffer");
209    Value * const offset = b->CreateAnd(consumed, blockSizeAlignmentMask);
210    Value * const unreadData = b->getRawOutputPointer("sourceBuffer", offset);
211    Value * const remainingItems = b->CreateSub(produced, offset);
212    Value * const potentialItems = b->CreateAdd(remainingItems, itemsToRead);
213    Value * const remainingBytes = b->CreateMul(remainingItems, codeUnitBytes);
214    // Have we consumed enough data that we can safely copy back the unconsumed data and still leave enough space
215    // for one segment without needing a temporary buffer?
216    Value * const canCopy = b->CreateICmpULT(b->CreateGEP(baseBuffer, potentialItems), unreadData);
217    b->CreateLikelyCondBr(canCopy, copyBack, expandAndCopyBack);
218
219    // If so, just copy the data ...
220    b->SetInsertPoint(copyBack);
221    b->CreateMemCpy(baseBuffer, unreadData, remainingBytes, blockSize);
222    b->CreateBr(prepareBuffer);
223
224    // Otherwise, allocate a buffer with twice the capacity and copy the unconsumed data back into it
225    b->SetInsertPoint(expandAndCopyBack);
226    Value * const expandedCapacity = b->CreateShl(capacity, 1);
227    Value * const expandedBytes = b->CreateMul(expandedCapacity, codeUnitBytes);
228    Value * const expandedBuffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(expandedBytes), unreadData->getType());
229    b->CreateMemCpy(expandedBuffer, unreadData, remainingBytes, blockSize);
230    // Free the prior buffer if it exists
231    Value * const ancillaryBuffer = b->getScalarField("ancillaryBuffer");
232    b->CreateFree(ancillaryBuffer);
233    b->setScalarField("ancillaryBuffer", baseBuffer);
234    b->setScalarField("buffer", expandedBuffer);
235    b->setCapacity("sourceBuffer", expandedCapacity);
236    b->CreateBr(prepareBuffer);
237
238    b->SetInsertPoint(prepareBuffer);
239    PHINode * const newBaseBuffer = b->CreatePHI(baseBuffer->getType(), 2);
240    newBaseBuffer->addIncoming(baseBuffer, copyBack);
241    newBaseBuffer->addIncoming(expandedBuffer, expandAndCopyBack);
242    Value * const newBaseAddress = b->CreateGEP(newBaseBuffer, b->CreateNeg(offset));
243    b->setBaseAddress("sourceBuffer", newBaseAddress);
244    b->CreateBr(readData);
245
246    // Regardless of whether we're simply appending data or had to allocate a new buffer, read a new page
247    // of data into the input source buffer. If we fail to read a full page ...
248    b->SetInsertPoint(readData);
249    Value * const sourceBuffer = b->getRawOutputPointer("sourceBuffer", produced);
250    Value * const fd = b->getScalarField("fileDescriptor");
251    Constant * const bytesToRead = ConstantExpr::getMul(itemsToRead, codeUnitBytes);
252    Value * const bytesRead = b->CreateReadCall(fd, sourceBuffer, bytesToRead);
253    Value * const itemsRead = b->CreateUDiv(bytesRead, codeUnitBytes);
254    Value * const itemsBuffered = b->CreateAdd(produced, itemsRead);
255    b->CreateUnlikelyCondBr(b->CreateICmpULT(itemsBuffered, itemsPending), setTermination, readExit);
256
257    // ... set the termination signal.
258    b->SetInsertPoint(setTermination);
259    Value * const bytesToZero = b->CreateMul(b->CreateSub(itemsPending, itemsBuffered), codeUnitBytes);
260    b->CreateMemZero(b->getRawOutputPointer("sourceBuffer", itemsBuffered), bytesToZero);
261    b->setScalarField("fileItems", itemsBuffered);
262    b->setTerminationSignal();
263    b->CreateBr(readExit);
264
265    b->SetInsertPoint(readExit);
266    PHINode * const itemsProduced = b->CreatePHI(itemsPending->getType(), 2);
267    itemsProduced->addIncoming(itemsPending, readData);
268    itemsProduced->addIncoming(itemsBuffered, setTermination);
269    b->setProducedItemCount("sourceBuffer", itemsProduced);
270}
271
272void ReadSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
273    b->CreateFree(b->getScalarField("ancillaryBuffer"));
274    b->CreateFree(b->getScalarField("buffer"));
275}
276
277/// Hybrid MMap/Read source kernel
278
279void FDSourceKernel::linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & b) {
280    mFileSizeFunction = MMapSourceKernel::linkFileSizeMethod(b);
281}
282
283void FDSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
284    BasicBlock * finalizeRead = b->CreateBasicBlock("finalizeRead");
285    BasicBlock * finalizeMMap = b->CreateBasicBlock("finalizeMMap");
286    BasicBlock * finalizeDone = b->CreateBasicBlock("finalizeDone");
287    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), finalizeMMap, finalizeRead);
288    b->SetInsertPoint(finalizeMMap);
289    MMapSourceKernel::freeBuffer(b, mCodeUnitWidth);
290    b->CreateBr(finalizeDone);
291    b->SetInsertPoint(finalizeRead);
292    ReadSourceKernel::freeBuffer(b);
293    b->CreateBr(finalizeDone);
294    b->SetInsertPoint(finalizeDone);
295}
296
297void FDSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
298    BasicBlock * initializeRead = b->CreateBasicBlock("initializeRead");
299    BasicBlock * tryMMap = b->CreateBasicBlock("tryMMap");
300    BasicBlock * initializeMMap = b->CreateBasicBlock("initializeMMap");
301    BasicBlock * initializeDone = b->CreateBasicBlock("initializeDone");
302
303    // The source will use MMapSource or readSoure kernel logic depending on the useMMap
304    // parameter, possibly overridden.
305    Value * useMMap = b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty());
306    // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic.
307    Value * fd = b->getScalarField("fileDescriptor");
308    useMMap = b->CreateAnd(useMMap, b->CreateICmpNE(fd, b->getInt32(STDIN_FILENO)));
309    b->CreateCondBr(useMMap, tryMMap, initializeRead);
310
311    b->SetInsertPoint(tryMMap);
312    // If the fileSize is 0, we may have a virtual file such as /proc/cpuinfo
313    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(mFileSizeFunction, fd), b->getSizeTy());
314    useMMap = b->CreateICmpNE(fileSize, b->getSize(0));
315    b->CreateCondBr(useMMap, initializeMMap, initializeRead);
316
317    b->SetInsertPoint(initializeMMap);
318    MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, mStride, b);
319    b->CreateBr(initializeDone);
320
321    b->SetInsertPoint(initializeRead);
322    // Ensure that readSource logic is used throughout.
323    b->setScalarField("useMMap", b->getInt8(0));
324    ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, mStride,b);
325    b->CreateBr(initializeDone);
326    b->SetInsertPoint(initializeDone);
327}
328
329void FDSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
330    BasicBlock * DoSegmentRead = b->CreateBasicBlock("DoSegmentRead");
331    BasicBlock * DoSegmentMMap = b->CreateBasicBlock("DoSegmentMMap");
332    BasicBlock * DoSegmentDone = b->CreateBasicBlock("DoSegmentDone");
333    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), DoSegmentMMap, DoSegmentRead);
334    b->SetInsertPoint(DoSegmentMMap);
335    MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride, b);
336    b->CreateBr(DoSegmentDone);
337    b->SetInsertPoint(DoSegmentRead);
338    ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride, b);
339    b->CreateBr(DoSegmentDone);
340    b->SetInsertPoint(DoSegmentDone);
341}
342
343/// MEMORY SOURCE KERNEL
344
345void MemorySourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
346    Value * const fileSource = b->getScalarField("fileSource");
347    b->setBaseAddress("sourceBuffer", fileSource);
348    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
349        b->CreateAssert(fileSource, getName() + " fileSource cannot be null");
350    }
351    Value * const fileItems = b->getScalarField("fileItems");
352    b->setCapacity("sourceBuffer", fileItems);
353}
354
355void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
356
357    Constant * const STRIDE_ITEMS = b->getSize(getStride());
358    Constant * const STRIDE_SIZE = b->getSize(getStride() * mCodeUnitWidth);
359    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
360
361    BasicBlock * const entry = b->GetInsertBlock();
362    BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary");
363    BasicBlock * const exit = b->CreateBasicBlock("exit");
364
365    Value * const fileItems = b->getScalarField("fileItems");
366    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
367    Value * const nextProducedItems = b->CreateAdd(producedItems, STRIDE_ITEMS);
368    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
369    b->CreateUnlikelyCondBr(lastPage, createTemporary, exit);
370
371    b->SetInsertPoint(createTemporary);
372    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
373    Value * readStart = nullptr;
374    Value * readEnd = nullptr;
375
376    // compute the range of our unconsumed buffer slice
377    if (LLVM_UNLIKELY(mStreamSetCount > 1)) {
378        Constant * const ZERO = b->getSize(0);
379        const StreamSetBuffer * const sourceBuffer = getOutputStreamSetBuffer("sourceBuffer");
380        Value * const fromIndex = b->CreateUDiv(consumedItems, BLOCK_WIDTH);
381        readStart = sourceBuffer->getStreamBlockPtr(b.get(), ZERO, fromIndex);
382        Value * const toIndex = b->CreateUDivCeil(fileItems, BLOCK_WIDTH);
383        // since we know this is an ExternalBuffer, we don't need to consider any potential modulus calculations.
384        readEnd = sourceBuffer->getStreamBlockPtr(b.get(), ZERO, toIndex);
385    } else {
386        // make sure our copy is block-aligned
387        Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
388        readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
389        readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
390    }
391
392    DataLayout DL(b->getModule());
393    Type * const intPtrTy = DL.getIntPtrType(readEnd->getType());
394    Value * const readEndInt = b->CreatePtrToInt(readEnd, intPtrTy);
395    Value * const readStartInt = b->CreatePtrToInt(readStart, intPtrTy);
396    Value * const unconsumedBytes = b->CreateTrunc(b->CreateSub(readEndInt, readStartInt), b->getSizeTy());
397    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), STRIDE_SIZE);
398    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
399    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
400    b->setScalarField("ancillaryBuffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
401    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
402    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
403
404    // get the difference between our base and from position then compute an offsetted temporary buffer address
405    Value * const base = b->getBaseAddress("sourceBuffer");
406    Value * const baseInt = b->CreatePtrToInt(base, intPtrTy);
407    Value * const diff = b->CreateSub(baseInt, readStartInt);
408    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
409    // set the temporary buffer as the new source buffer
410    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
411    b->setTerminationSignal();
412    BasicBlock * const terminationExit = b->GetInsertBlock();
413    b->CreateBr(exit);
414
415    b->SetInsertPoint(exit);
416    PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
417    newProducedItems->addIncoming(nextProducedItems, entry);
418    newProducedItems->addIncoming(fileItems, terminationExit);
419    b->setProducedItemCount("sourceBuffer", newProducedItems);
420}
421
422void MemorySourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
423    b->CreateFree(b->getScalarField("ancillaryBuffer"));
424}
425
426MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const fd, StreamSet * const outputStream)
427: SegmentOrientedKernel("mmap_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
428// input streams
429,{}
430// output streams
431,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
432// input scalars
433,{Binding{"fileDescriptor", fd}}
434// output scalars
435,{Binding{b->getSizeTy(), "fileItems"}}
436// internal scalars
437,{})
438, mCodeUnitWidth(outputStream->getFieldWidth())
439, mFileSizeFunction(nullptr) {
440    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
441    addInternalScalar(codeUnitPtrTy, "buffer");
442    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
443    addAttribute(MustExplicitlyTerminate());
444    setStride(codegen::SegmentSize);
445}
446
447ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const fd, StreamSet * const outputStream)
448: SegmentOrientedKernel("read_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
449// input streams
450,{}
451// output streams
452,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
453// input scalars
454,{Binding{"fileDescriptor", fd}}
455// output scalars
456,{Binding{b->getSizeTy(), "fileItems"}}
457// internal scalars
458,{})
459, mCodeUnitWidth(outputStream->getFieldWidth()) {
460    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
461    addInternalScalar(codeUnitPtrTy, "buffer");
462    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
463    addAttribute(MustExplicitlyTerminate());
464    setStride(codegen::SegmentSize);
465}
466
467
468FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const useMMap, Scalar * const fd, StreamSet * const outputStream)
469: SegmentOrientedKernel("FD_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
470// input streams
471,{}
472// output stream
473,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
474// input scalar
475,{Binding{"useMMap", useMMap}
476, Binding{"fileDescriptor", fd}}
477// output scalar
478,{Binding{b->getSizeTy(), "fileItems"}}
479// internal scalars
480,{})
481, mCodeUnitWidth(outputStream->getFieldWidth())
482, mFileSizeFunction(nullptr) {
483    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
484    addInternalScalar(codeUnitPtrTy, "buffer");
485    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
486    addAttribute(MustExplicitlyTerminate());
487    setStride(codegen::SegmentSize);
488}
489
490MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> &, Scalar * fileSource, Scalar * fileItems, StreamSet * const outputStream)
491: SegmentOrientedKernel("memory_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth()) + ":" + std::to_string(outputStream->getNumElements()),
492// input streams
493{},
494// output stream
495{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}},
496// input scalar
497{Binding{"fileSource", fileSource}, Binding{"fileItems", fileItems}},
498{},
499// internal scalar
500{Binding{fileSource->getType(), "buffer"}
501,Binding{fileSource->getType(), "ancillaryBuffer"}
502})
503, mStreamSetCount(outputStream->getNumElements())
504, mCodeUnitWidth(outputStream->getFieldWidth()) {
505    addAttribute(MustExplicitlyTerminate());
506    setStride(codegen::SegmentSize);
507}
508
509}
Note: See TracBrowser for help on using the repository browser.