source: icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp @ 6297

Last change on this file since 6297 was 6261, checked in by nmedfort, 7 months ago

Work on OptimizationBranch?; revisited pipeline termination

File size: 24.5 KB
Line 
1/*
2 *  Copyright (c) 2018 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5#include "source_kernel.h"
6#include <kernels/kernel_builder.h>
7#include <kernels/streamset.h>
8#include <llvm/IR/Module.h>
9#include <sys/stat.h>
10#include <fcntl.h>
11#include <toolchain/toolchain.h>
12#include <boost/interprocess/mapped_region.hpp>
13
14using namespace llvm;
15
16inline unsigned getPageSize() {
17    return boost::interprocess::mapped_region::get_page_size();
18}
19
20extern "C" uint64_t file_size(const uint32_t fd) {
21    struct stat st;
22    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
23        st.st_size = 0;
24    }
25    return st.st_size;
26}
27
28namespace kernel {
29
30/// MMAP SOURCE KERNEL
31
32Function * MMapSourceKernel::linkFileSizeMethod(const std::unique_ptr<kernel::KernelBuilder> & b) {
33    return b->LinkFunction("file_size", file_size);
34}
35
36void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
37
38    BasicBlock * const emptyFile = b->CreateBasicBlock("emptyFile");
39    BasicBlock * const nonEmptyFile = b->CreateBasicBlock("NonEmptyFile");
40    BasicBlock * const exit = b->CreateBasicBlock("Exit");
41    IntegerType * const sizeTy = b->getSizeTy();
42    ConstantInt * const STRIDE_SIZE = b->getSize(stride);
43    Constant * const PAGE_ITEMS = b->getSize((8 * stride) / codeUnitWidth);
44    Value * const fd = b->getScalarField("fileDescriptor");
45    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
46    b->setScalarField("ancillaryBuffer", ConstantPointerNull::get(codeUnitPtrTy));
47    assert (fileSizeMethod);
48    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(fileSizeMethod, fd), sizeTy);
49    b->CreateLikelyCondBr(b->CreateIsNotNull(fileSize), nonEmptyFile, emptyFile);
50
51    b->SetInsertPoint(nonEmptyFile);
52    Value * const fileBuffer = b->CreatePointerCast(b->CreateFileSourceMMap(fd, fileSize), codeUnitPtrTy);
53    b->setScalarField("buffer", fileBuffer);
54    b->setBaseAddress("sourceBuffer", fileBuffer);
55    b->CreateMAdvise(fileBuffer, fileSize, CBuilder::ADVICE_WILLNEED);
56    Value * fileItems = fileSize;
57    if (LLVM_UNLIKELY(codeUnitWidth > 8)) {
58        fileItems = b->CreateUDiv(fileSize, b->getSize(codeUnitWidth / 8));
59    }
60    b->setScalarField("fileItems", fileItems);
61    b->setCapacity("sourceBuffer", fileItems);
62    b->CreateBr(exit);
63
64    b->SetInsertPoint(emptyFile);
65    Value * const emptyFilePtr = b->CreatePointerCast(b->CreateAnonymousMMap(STRIDE_SIZE), codeUnitPtrTy);
66    b->setScalarField("buffer", emptyFilePtr);
67    b->setBaseAddress("sourceBuffer", emptyFilePtr);
68    b->setScalarField("fileItems", PAGE_ITEMS);
69    b->setTerminationSignal();
70    b->CreateBr(exit);
71
72    b->SetInsertPoint(exit);
73}
74
75
76void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
77
78    BasicBlock * const dropPages = b->CreateBasicBlock("dropPages");
79    BasicBlock * const checkRemaining = b->CreateBasicBlock("checkRemaining");
80    BasicBlock * const setTermination = b->CreateBasicBlock("setTermination");
81    BasicBlock * const exit = b->CreateBasicBlock("mmapSourceExit");
82
83    ConstantInt * const MMAP_PAGE_SIZE = b->getSize(getPageSize());
84    ConstantInt * const STRIDE_ITEMS = b->getSize(stride);
85    ConstantInt * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
86    ConstantInt * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
87
88    ConstantInt * const STRIDE_BYTES = b->getSize((codeUnitWidth * stride) / 8);
89    ConstantInt * const PADDING_SIZE = b->getSize(b->getBitBlockWidth() * codeUnitWidth / 8);
90
91    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
92    Value * const consumedBytes = b->CreateMul(consumedItems, CODE_UNIT_BYTES);
93    Value * const consumedPageOffset = b->CreateAnd(consumedBytes, ConstantExpr::getNeg(MMAP_PAGE_SIZE));
94    Value * const consumedBuffer = b->getRawOutputPointer("sourceBuffer", consumedPageOffset);
95    Value * const readableBuffer = b->getScalarField("buffer");
96    Value * const unnecessaryBytes = b->CreatePtrDiff(consumedBuffer, readableBuffer);
97
98    // avoid calling madvise unless an actual page table change could occur
99    b->CreateLikelyCondBr(b->CreateIsNotNull(unnecessaryBytes), dropPages, checkRemaining);
100
101    b->SetInsertPoint(dropPages);
102    // instruct the OS that it can safely drop any fully consumed pages
103    b->CreateMAdvise(readableBuffer, unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
104    b->CreateBr(checkRemaining);
105
106    // determine whether or not we've exhausted the "safe" region of the file buffer
107    b->SetInsertPoint(checkRemaining);
108    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
109    Value * const nextProducedItems = b->CreateAdd(producedItems, STRIDE_ITEMS);
110    Value * const fileItems = b->getScalarField("fileItems");
111    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
112    b->CreateUnlikelyCondBr(lastPage, setTermination, exit);
113
114    // If this is the last page, create a temporary buffer of up to two pages size, copy the unconsumed data
115    // and zero any bytes that are not used.
116    b->SetInsertPoint(setTermination);
117    Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
118    Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
119    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
120
121    DataLayout DL(b->getModule());
122    Type * const intPtrTy = DL.getIntPtrType(readEnd->getType());
123    Value * const readEndInt = b->CreatePtrToInt(readEnd, intPtrTy);
124    Value * const readStartInt = b->CreatePtrToInt(readStart, intPtrTy);
125    Value * unconsumedBytes = b->CreateSub(readEndInt, readStartInt);
126    unconsumedBytes = b->CreateTrunc(unconsumedBytes, b->getSizeTy());
127    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, PADDING_SIZE), STRIDE_BYTES);
128    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
129    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
130    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
131    // get the difference between our base and from position then compute an offsetted temporary buffer address
132    Value * const base = b->getBaseAddress("sourceBuffer");
133    Value * const baseInt = b->CreatePtrToInt(base, intPtrTy);
134    Value * const diff = b->CreateSub(baseInt, readStartInt);
135    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
136    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
137    b->setScalarField("ancillaryBuffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
138    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
139    b->setTerminationSignal();
140    b->setProducedItemCount("sourceBuffer", fileItems);
141    b->CreateBr(exit);
142
143    b->SetInsertPoint(exit);
144}
145void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b, const unsigned codeUnitWidth) {
146    b->CreateFree(b->getScalarField("ancillaryBuffer"));
147    Value * const fileItems = b->getScalarField("fileItems");
148    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
149    Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES);
150    b->CreateMUnmap(b->getScalarField("buffer"), fileSize);
151}
152
153/// READ SOURCE KERNEL
154
155void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
156    ConstantInt * const bufferItems = b->getSize(stride * 4);
157    const auto codeUnitSize = codeUnitWidth / 8;
158    ConstantInt * const bufferBytes = b->getSize(stride * 4 * codeUnitSize);
159    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
160    Value * const buffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy);
161    b->setBaseAddress("sourceBuffer", buffer);
162    b->setScalarField("ancillaryBuffer", ConstantPointerNull::get(codeUnitPtrTy));
163    b->setScalarField("buffer", buffer);
164    b->setCapacity("sourceBuffer", bufferItems);
165}
166
167void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
168
169    ConstantInt * const itemsToRead = b->getSize(stride);
170    ConstantInt * const codeUnitBytes = b->getSize(codeUnitWidth / 8);
171
172    BasicBlock * const moveData = b->CreateBasicBlock("MoveData");
173    BasicBlock * const prepareBuffer = b->CreateBasicBlock("PrepareBuffer");
174    BasicBlock * const readData = b->CreateBasicBlock("ReadData");
175    BasicBlock * const setTermination = b->CreateBasicBlock("SetTermination");
176    BasicBlock * const readExit = b->CreateBasicBlock("ReadExit");
177
178    // Can we append to our existing buffer without impacting any subsequent kernel?
179    Value * const produced = b->getProducedItemCount("sourceBuffer");
180    Value * const itemsPending = b->CreateAdd(produced, itemsToRead);
181    Value * const capacity = b->getCapacity("sourceBuffer");
182    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", itemsPending);
183    Value * const baseBuffer = b->getScalarField("buffer");
184    Value * const bufferLimit = b->CreateGEP(baseBuffer, capacity);
185    b->CreateLikelyCondBr(b->CreateICmpULE(readEnd, bufferLimit), readData, moveData);
186
187    // No. If we can copy the unconsumed data back to the start of the buffer *and* write a full
188    // segment of data without overwriting the currently unconsumed data, do so since it won't
189    // affect any potential consumer that could be using the "stale" output base pointer.
190    b->SetInsertPoint(moveData);
191
192    // Determine how much data has been consumed and how much needs to be copied back, noting
193    // that our "unproduced" data must be block aligned.
194    BasicBlock * const copyBack = b->CreateBasicBlock("CopyBack");
195    BasicBlock * const expandAndCopyBack = b->CreateBasicBlock("ExpandAndCopyBack");
196    const auto blockSize = b->getBitBlockWidth() / 8;
197    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
198    ConstantInt * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
199    Constant * const ALIGNMENT_MASK = ConstantExpr::getNeg(BLOCK_WIDTH);
200    Value * const consumed = b->CreateAnd(consumedItems, ALIGNMENT_MASK);
201    Value * const unreadData = b->getRawOutputPointer("sourceBuffer", consumed);
202    Value * const remainingItems = b->CreateSub(produced, consumed);
203    Value * const potentialItems = b->CreateAdd(remainingItems, itemsToRead);
204    Value * const remainingBytes = b->CreateMul(remainingItems, codeUnitBytes);
205    // Have we consumed enough data that we can safely copy back the unconsumed data and still
206    // leave enough space for one segment without needing a temporary buffer?
207    Value * const canCopy = b->CreateICmpULT(b->CreateGEP(baseBuffer, potentialItems), unreadData);
208    b->CreateLikelyCondBr(canCopy, copyBack, expandAndCopyBack);
209
210    // If so, just copy the data ...
211    b->SetInsertPoint(copyBack);
212    b->CreateMemCpy(baseBuffer, unreadData, remainingBytes, blockSize);
213    b->CreateBr(prepareBuffer);
214
215    // Otherwise, allocate a buffer with twice the capacity and copy the unconsumed data back into it
216    b->SetInsertPoint(expandAndCopyBack);
217    Value * const expandedCapacity = b->CreateShl(capacity, 1);
218    Value * const expandedBytes = b->CreateMul(expandedCapacity, codeUnitBytes);
219    Value * const expandedBuffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(expandedBytes), unreadData->getType());
220    b->CreateMemCpy(expandedBuffer, unreadData, remainingBytes, blockSize);
221    // Free the prior buffer if it exists
222    Value * const ancillaryBuffer = b->getScalarField("ancillaryBuffer");
223    b->setScalarField("ancillaryBuffer", baseBuffer);
224    b->CreateFree(ancillaryBuffer);
225    b->setScalarField("buffer", expandedBuffer);
226    b->setCapacity("sourceBuffer", expandedCapacity);
227    b->CreateBr(prepareBuffer);
228
229    b->SetInsertPoint(prepareBuffer);
230    PHINode * const newBaseBuffer = b->CreatePHI(baseBuffer->getType(), 2);
231    newBaseBuffer->addIncoming(baseBuffer, copyBack);
232    newBaseBuffer->addIncoming(expandedBuffer, expandAndCopyBack);
233    Value * const newBaseAddress = b->CreateGEP(newBaseBuffer, b->CreateNeg(consumed));
234    b->setBaseAddress("sourceBuffer", newBaseAddress);
235    b->CreateBr(readData);
236
237    // Regardless of whether we're simply appending data or had to allocate a new buffer, read a new page
238    // of data into the input source buffer. If we fail to read a full page ...
239    b->SetInsertPoint(readData);
240    Value * const sourceBuffer = b->getRawOutputPointer("sourceBuffer", produced);
241    Value * const fd = b->getScalarField("fileDescriptor");
242    Constant * const bytesToRead = ConstantExpr::getMul(itemsToRead, codeUnitBytes);
243    Value * const bytesRead = b->CreateReadCall(fd, sourceBuffer, bytesToRead);
244    Value * const itemsRead = b->CreateUDiv(bytesRead, codeUnitBytes);
245    Value * const itemsBuffered = b->CreateAdd(produced, itemsRead);
246    b->CreateUnlikelyCondBr(b->CreateICmpULT(itemsBuffered, itemsPending), setTermination, readExit);
247
248    // ... set the termination signal.
249    b->SetInsertPoint(setTermination);
250    Value * const bytesToZero = b->CreateMul(b->CreateSub(itemsPending, itemsBuffered), codeUnitBytes);
251    b->CreateMemZero(b->getRawOutputPointer("sourceBuffer", itemsBuffered), bytesToZero);
252    b->setScalarField("fileItems", itemsBuffered);
253    b->setTerminationSignal();
254    b->setProducedItemCount("sourceBuffer", itemsBuffered);
255    b->CreateBr(readExit);
256
257    b->SetInsertPoint(readExit);
258}
259
260void ReadSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
261    b->CreateFree(b->getScalarField("ancillaryBuffer"));
262    b->CreateFree(b->getScalarField("buffer"));
263}
264
265/// Hybrid MMap/Read source kernel
266
267void FDSourceKernel::linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & b) {
268    mFileSizeFunction = MMapSourceKernel::linkFileSizeMethod(b);
269}
270
271void FDSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
272    BasicBlock * finalizeRead = b->CreateBasicBlock("finalizeRead");
273    BasicBlock * finalizeMMap = b->CreateBasicBlock("finalizeMMap");
274    BasicBlock * finalizeDone = b->CreateBasicBlock("finalizeDone");
275    Value * const useMMap = b->CreateIsNotNull(b->getScalarField("useMMap"));
276    b->CreateCondBr(useMMap, finalizeMMap, finalizeRead);
277    b->SetInsertPoint(finalizeMMap);
278    MMapSourceKernel::freeBuffer(b, mCodeUnitWidth);
279    b->CreateBr(finalizeDone);
280    b->SetInsertPoint(finalizeRead);
281    ReadSourceKernel::freeBuffer(b);
282    b->CreateBr(finalizeDone);
283    b->SetInsertPoint(finalizeDone);
284}
285
286void FDSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
287    BasicBlock * initializeRead = b->CreateBasicBlock("initializeRead");
288    BasicBlock * checkFileSize = b->CreateBasicBlock("checkFileSize");
289    BasicBlock * initializeMMap = b->CreateBasicBlock("initializeMMap");
290    BasicBlock * initializeDone = b->CreateBasicBlock("initializeDone");
291
292    // The source will use MMapSource or readSoure kernel logic depending on the useMMap
293    // parameter, possibly overridden.
294
295    Value * const useMMap = b->getScalarField("useMMap");
296    Constant * const ZERO = ConstantInt::getNullValue(useMMap->getType());
297    // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic.
298    Value * const fd = b->getScalarField("fileDescriptor");
299    Value * const notStdIn = b->CreateICmpNE(fd, b->getInt32(STDIN_FILENO));
300    Value * const tryMMap = b->CreateICmpNE(useMMap, ZERO);
301    b->CreateCondBr(b->CreateAnd(tryMMap, notStdIn), checkFileSize, initializeRead);
302
303    b->SetInsertPoint(checkFileSize);
304    // If the fileSize is 0, we may have a virtual file such as /proc/cpuinfo
305    Value * const fileSize = b->CreateCall(mFileSizeFunction, fd);
306    Value * const emptyFile = b->CreateIsNotNull(fileSize);
307    b->CreateUnlikelyCondBr(emptyFile, initializeRead, initializeMMap);
308
309    b->SetInsertPoint(initializeMMap);
310    MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, mStride, b);
311    b->CreateBr(initializeDone);
312
313    b->SetInsertPoint(initializeRead);
314    // Ensure that readSource logic is used throughout.
315    b->setScalarField("useMMap", ZERO);
316    ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, mStride,b);
317    b->CreateBr(initializeDone);
318
319    b->SetInsertPoint(initializeDone);
320}
321
322void FDSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
323    BasicBlock * DoSegmentRead = b->CreateBasicBlock("DoSegmentRead");
324    BasicBlock * DoSegmentMMap = b->CreateBasicBlock("DoSegmentMMap");
325    BasicBlock * DoSegmentDone = b->CreateBasicBlock("DoSegmentDone");
326    Value * const useMMap = b->CreateIsNotNull(b->getScalarField("useMMap"));
327    b->CreateCondBr(useMMap, DoSegmentMMap, DoSegmentRead);
328    b->SetInsertPoint(DoSegmentMMap);
329    MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride, b);
330    b->CreateBr(DoSegmentDone);
331    b->SetInsertPoint(DoSegmentRead);
332    ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride, b);
333    b->CreateBr(DoSegmentDone);
334    b->SetInsertPoint(DoSegmentDone);
335}
336
337/// MEMORY SOURCE KERNEL
338
339void MemorySourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
340    Value * const fileSource = b->getScalarField("fileSource");
341    b->setBaseAddress("sourceBuffer", fileSource);
342    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
343        b->CreateAssert(fileSource, getName() + " fileSource cannot be null");
344    }
345    Value * const fileItems = b->getScalarField("fileItems");
346    b->setCapacity("sourceBuffer", fileItems);
347}
348
349void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
350
351    Constant * const STRIDE_ITEMS = b->getSize(getStride());
352    Constant * const STRIDE_SIZE = b->getSize(getStride() * mCodeUnitWidth);
353    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
354
355    BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary");
356    BasicBlock * const exit = b->CreateBasicBlock("exit");
357
358    Value * const fileItems = b->getScalarField("fileItems");
359    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
360    Value * const nextProducedItems = b->CreateAdd(producedItems, STRIDE_ITEMS);
361    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
362    b->CreateUnlikelyCondBr(lastPage, createTemporary, exit);
363
364    b->SetInsertPoint(createTemporary);
365    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
366    Value * readStart = nullptr;
367    Value * readEnd = nullptr;
368
369    // compute the range of our unconsumed buffer slice
370    if (LLVM_UNLIKELY(mStreamSetCount > 1)) {
371        Constant * const ZERO = b->getSize(0);
372        const StreamSetBuffer * const sourceBuffer = getOutputStreamSetBuffer("sourceBuffer");
373        Value * const fromIndex = b->CreateUDiv(consumedItems, BLOCK_WIDTH);
374        readStart = sourceBuffer->getStreamBlockPtr(b.get(), ZERO, fromIndex);
375        Value * const toIndex = b->CreateUDivCeil(fileItems, BLOCK_WIDTH);
376        // since we know this is an ExternalBuffer, we don't need to consider any potential modulus calculations.
377        readEnd = sourceBuffer->getStreamBlockPtr(b.get(), ZERO, toIndex);
378    } else {
379        // make sure our copy is block-aligned
380        Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
381        readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
382        readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
383    }
384
385    DataLayout DL(b->getModule());
386    Type * const intPtrTy = DL.getIntPtrType(readEnd->getType());
387    Value * const readEndInt = b->CreatePtrToInt(readEnd, intPtrTy);
388    Value * const readStartInt = b->CreatePtrToInt(readStart, intPtrTy);
389    Value * const unconsumedBytes = b->CreateTrunc(b->CreateSub(readEndInt, readStartInt), b->getSizeTy());
390    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), STRIDE_SIZE);
391    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
392    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
393    b->setScalarField("ancillaryBuffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
394    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
395    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
396
397    // get the difference between our base and from position then compute an offsetted temporary buffer address
398    Value * const base = b->getBaseAddress("sourceBuffer");
399    Value * const baseInt = b->CreatePtrToInt(base, intPtrTy);
400    Value * const diff = b->CreateSub(baseInt, readStartInt);
401    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
402    // set the temporary buffer as the new source buffer
403    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
404    b->setTerminationSignal();
405    b->setProducedItemCount("sourceBuffer", fileItems);
406    b->CreateBr(exit);
407
408    b->SetInsertPoint(exit);
409}
410
411void MemorySourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
412    b->CreateFree(b->getScalarField("ancillaryBuffer"));
413}
414
415MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const fd, StreamSet * const outputStream)
416: SegmentOrientedKernel(b, "mmap_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
417// input streams
418,{}
419// output streams
420,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
421// input scalars
422,{Binding{"fileDescriptor", fd}}
423// output scalars
424,{Binding{b->getSizeTy(), "fileItems"}}
425// internal scalars
426,{})
427, mCodeUnitWidth(outputStream->getFieldWidth())
428, mFileSizeFunction(nullptr) {
429    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
430    addInternalScalar(codeUnitPtrTy, "buffer");
431    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
432    addAttribute(MustExplicitlyTerminate());
433    setStride(codegen::SegmentSize);
434}
435
436ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const fd, StreamSet * const outputStream)
437: SegmentOrientedKernel(b, "read_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
438// input streams
439,{}
440// output streams
441,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
442// input scalars
443,{Binding{"fileDescriptor", fd}}
444// output scalars
445,{Binding{b->getSizeTy(), "fileItems"}}
446// internal scalars
447,{})
448, mCodeUnitWidth(outputStream->getFieldWidth()) {
449    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
450    addInternalScalar(codeUnitPtrTy, "buffer");
451    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
452    addAttribute(MustExplicitlyTerminate());
453    setStride(codegen::SegmentSize);
454}
455
456
457FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const useMMap, Scalar * const fd, StreamSet * const outputStream)
458: SegmentOrientedKernel(b, "FD_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
459// input streams
460,{}
461// output stream
462,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
463// input scalar
464,{Binding{"useMMap", useMMap}
465, Binding{"fileDescriptor", fd}}
466// output scalar
467,{Binding{b->getSizeTy(), "fileItems"}}
468// internal scalars
469,{})
470, mCodeUnitWidth(outputStream->getFieldWidth())
471, mFileSizeFunction(nullptr) {
472    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
473    addInternalScalar(codeUnitPtrTy, "buffer");
474    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
475    addAttribute(MustExplicitlyTerminate());
476    setStride(codegen::SegmentSize);
477}
478
479MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * fileSource, Scalar * fileItems, StreamSet * const outputStream)
480: SegmentOrientedKernel(b, "memory_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth()) + ":" + std::to_string(outputStream->getNumElements()),
481// input streams
482{},
483// output stream
484{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}},
485// input scalar
486{Binding{"fileSource", fileSource}, Binding{"fileItems", fileItems}},
487{},
488// internal scalar
489{Binding{fileSource->getType(), "buffer"}
490,Binding{fileSource->getType(), "ancillaryBuffer"}
491})
492, mStreamSetCount(outputStream->getNumElements())
493, mCodeUnitWidth(outputStream->getFieldWidth()) {
494    addAttribute(MustExplicitlyTerminate());
495    setStride(codegen::SegmentSize);
496}
497
498}
Note: See TracBrowser for help on using the repository browser.