source: icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp @ 6230

Last change on this file since 6230 was 6230, checked in by cameron, 7 months ago

Resolve conflict with Mac OS system macro PAGE_SIZE

File size: 25.2 KB
Line 
1/*
2 *  Copyright (c) 2018 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5#include "source_kernel.h"
6#include <kernels/kernel_builder.h>
7#include <kernels/streamset.h>
8#include <llvm/IR/Module.h>
9#include <sys/stat.h>
10#include <fcntl.h>
11#include <toolchain/toolchain.h>
12#include <boost/interprocess/mapped_region.hpp>
13
14using namespace llvm;
15
16inline unsigned getPageSize() {
17    return boost::interprocess::mapped_region::get_page_size();
18}
19
20extern "C" uint64_t file_size(const uint32_t fd) {
21    struct stat st;
22    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
23        st.st_size = 0;
24    }
25    return st.st_size;
26}
27
28namespace kernel {
29
30/// MMAP SOURCE KERNEL
31
32Function * MMapSourceKernel::linkFileSizeMethod(const std::unique_ptr<kernel::KernelBuilder> & b) {
33    return b->LinkFunction("file_size", file_size);
34}
35
36void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
37
38    BasicBlock * const emptyFile = b->CreateBasicBlock("emptyFile");
39    BasicBlock * const nonEmptyFile = b->CreateBasicBlock("NonEmptyFile");
40    BasicBlock * const exit = b->CreateBasicBlock("Exit");
41    IntegerType * const sizeTy = b->getSizeTy();
42    ConstantInt * const STRIDE_SIZE = b->getSize(stride);
43    Constant * const PAGE_ITEMS = b->getSize(stride /(codeUnitWidth/8));
44    Value * const fd = b->getScalarField("fileDescriptor");
45
46    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
47    b->setScalarField("ancillaryBuffer", ConstantPointerNull::get(codeUnitPtrTy));
48    assert (fileSizeMethod);
49    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(fileSizeMethod, fd), sizeTy);
50    b->CreateLikelyCondBr(b->CreateIsNotNull(fileSize), nonEmptyFile, emptyFile);
51
52    b->SetInsertPoint(nonEmptyFile);
53    Value * const fileBuffer = b->CreatePointerCast(b->CreateFileSourceMMap(fd, fileSize), codeUnitPtrTy);
54    b->setScalarField("buffer", fileBuffer);
55    b->setBaseAddress("sourceBuffer", fileBuffer);
56    b->CreateMAdvise(fileBuffer, fileSize, CBuilder::ADVICE_WILLNEED);
57    Value * fileItems = fileSize;
58    if (LLVM_UNLIKELY(codeUnitWidth > 8)) {
59        fileItems = b->CreateUDiv(fileSize, b->getSize(codeUnitWidth / 8));
60    }
61    b->setScalarField("fileItems", fileItems);
62    b->setCapacity("sourceBuffer", fileItems);
63    b->CreateBr(exit);
64
65    b->SetInsertPoint(emptyFile);
66    Value * const emptyFilePtr = b->CreatePointerCast(b->CreateAnonymousMMap(STRIDE_SIZE), codeUnitPtrTy);
67    b->setScalarField("buffer", emptyFilePtr);
68    b->setBaseAddress("sourceBuffer", emptyFilePtr);
69    b->setScalarField("fileItems", PAGE_ITEMS);
70    b->setTerminationSignal();
71    b->CreateBr(exit);
72
73    b->SetInsertPoint(exit);
74}
75
76
77void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
78
79    BasicBlock * const dropPages = b->CreateBasicBlock("dropPages");
80    BasicBlock * const checkRemaining = b->CreateBasicBlock("checkRemaining");
81    BasicBlock * const setTermination = b->CreateBasicBlock("setTermination");
82    BasicBlock * const exit = b->CreateBasicBlock("mmapSourceExit");
83
84    Constant * const MMAP_PAGE_SIZE = b->getSize(getPageSize());
85
86    Constant * const STRIDE_SIZE = b->getSize(stride);
87
88    Constant * const PAGE_ITEMS = b->getSize((8 * stride) / codeUnitWidth);
89    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
90    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
91    Constant * const PADDING_SIZE = b->getSize(b->getBitBlockWidth() * codeUnitWidth / 8);
92
93    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
94    Value * const consumedBytes = b->CreateMul(consumedItems, CODE_UNIT_BYTES);
95    Value * const consumedPageOffset = b->CreateAnd(consumedBytes, ConstantExpr::getNeg(MMAP_PAGE_SIZE));
96    Value * const consumedBuffer = b->getRawOutputPointer("sourceBuffer", consumedPageOffset);
97    Value * const readableBuffer = b->getScalarField("buffer");
98    Value * const unnecessaryBytes = b->CreatePtrDiff(consumedBuffer, readableBuffer);
99
100    // avoid calling madvise unless an actual page table change could occur
101    b->CreateLikelyCondBr(b->CreateIsNotNull(unnecessaryBytes), dropPages, checkRemaining);
102
103    b->SetInsertPoint(dropPages);
104    // instruct the OS that it can safely drop any fully consumed pages
105    b->CreateMAdvise(readableBuffer, unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
106    //b->setScalarField("buffer", b->CreateGEP(readableBuffer, unnecessaryBytes));
107    b->CreateBr(checkRemaining);
108
109    // determine whether or not we've exhausted the file buffer
110    b->SetInsertPoint(checkRemaining);
111    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
112    Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_ITEMS);
113    Value * const fileItems = b->getScalarField("fileItems");
114    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
115    b->CreateUnlikelyCondBr(lastPage, setTermination, exit);
116
117    // If this is the last page, create a temporary buffer of up to two pages size, copy the unconsumed data
118    // and zero any bytes that are not used.
119    b->SetInsertPoint(setTermination);
120    Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
121    Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
122    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
123
124    DataLayout DL(b->getModule());
125    Type * const intPtrTy = DL.getIntPtrType(readEnd->getType());
126    Value * const readEndInt = b->CreatePtrToInt(readEnd, intPtrTy);
127    Value * const readStartInt = b->CreatePtrToInt(readStart, intPtrTy);
128    Value * unconsumedBytes = b->CreateSub(readEndInt, readStartInt);
129    unconsumedBytes = b->CreateTrunc(unconsumedBytes, b->getSizeTy());
130
131    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, PADDING_SIZE), STRIDE_SIZE);
132    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
133
134    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
135    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
136    // get the difference between our base and from position then compute an offsetted temporary buffer address
137    Value * const base = b->getBaseAddress("sourceBuffer");
138    Value * const baseInt = b->CreatePtrToInt(base, intPtrTy);
139    Value * const diff = b->CreateSub(baseInt, readStartInt);
140    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
141    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
142    // set the original base address as the buffer address.
143    //b->setScalarField("buffer", b->CreatePointerCast(base, codeUnitPtrTy));
144    b->setScalarField("ancillaryBuffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
145    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
146    b->setTerminationSignal();
147    BasicBlock * const terminationExit = b->GetInsertBlock();
148    b->CreateBr(exit);
149
150    // finally, set the "produced" count to reflect current position in the file
151    b->SetInsertPoint(exit);
152    PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
153    newProducedItems->addIncoming(nextProducedItems, checkRemaining);
154    newProducedItems->addIncoming(fileItems, terminationExit);
155    b->setProducedItemCount("sourceBuffer", newProducedItems);
156}
157void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b, const unsigned codeUnitWidth) {
158    b->CreateFree(b->getScalarField("ancillaryBuffer"));
159    Value * const fileItems = b->getScalarField("fileItems");
160    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
161    Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES);
162    b->CreateMUnmap(b->getScalarField("buffer"), fileSize);
163}
164
165/// READ SOURCE KERNEL
166
167void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
168    ConstantInt * const bufferItems = b->getSize(stride * 4);
169    const auto codeUnitSize = codeUnitWidth / 8;
170    ConstantInt * const bufferBytes = b->getSize(stride * 4 * codeUnitSize);
171    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
172    Value * const buffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy);
173    b->setBaseAddress("sourceBuffer", buffer);
174    b->setScalarField("ancillaryBuffer", ConstantPointerNull::get(codeUnitPtrTy));
175    b->setScalarField("buffer", buffer);
176    b->setCapacity("sourceBuffer", bufferItems);
177}
178
179void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
180
181    ConstantInt * const itemsToRead = b->getSize(stride);
182    ConstantInt * const codeUnitBytes = b->getSize(codeUnitWidth / 8);
183
184    BasicBlock * const moveData = b->CreateBasicBlock("MoveData");
185    BasicBlock * const prepareBuffer = b->CreateBasicBlock("PrepareBuffer");
186    BasicBlock * const readData = b->CreateBasicBlock("ReadData");
187    BasicBlock * const setTermination = b->CreateBasicBlock("SetTermination");
188    BasicBlock * const readExit = b->CreateBasicBlock("ReadExit");
189
190    // Can we append to our existing buffer without impacting any subsequent kernel?
191    Value * const produced = b->getProducedItemCount("sourceBuffer");
192    Value * const itemsPending = b->CreateAdd(produced, itemsToRead);
193    Value * const capacity = b->getCapacity("sourceBuffer");
194    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", itemsPending);
195    Value * const baseBuffer = b->getScalarField("buffer");
196    Value * const bufferLimit = b->CreateGEP(baseBuffer, capacity);
197    b->CreateLikelyCondBr(b->CreateICmpULE(readEnd, bufferLimit), readData, moveData);
198
199    // No. If we can copy the unconsumed data back to the start of the buffer *and* write a full
200    // segment of data without overwriting the currently unconsumed data, do so since it won't
201    // affect any potential consumer that could be using the "stale" output base pointer.
202    b->SetInsertPoint(moveData);
203
204    // Determine how much data has been consumed and how much needs to be copied back, noting
205    // that our "unproduced" data must be block aligned.
206    BasicBlock * const copyBack = b->CreateBasicBlock("CopyBack");
207    BasicBlock * const expandAndCopyBack = b->CreateBasicBlock("ExpandAndCopyBack");
208    const auto blockSize = b->getBitBlockWidth() / 8;
209    Constant * const blockSizeAlignmentMask = ConstantExpr::getNeg(b->getSize(blockSize));
210    Value * const consumed = b->getConsumedItemCount("sourceBuffer");
211    Value * const offset = b->CreateAnd(consumed, blockSizeAlignmentMask);
212    Value * const unreadData = b->getRawOutputPointer("sourceBuffer", offset);
213    Value * const remainingItems = b->CreateSub(produced, offset);
214    Value * const potentialItems = b->CreateAdd(remainingItems, itemsToRead);
215    Value * const remainingBytes = b->CreateMul(remainingItems, codeUnitBytes);
216    // Have we consumed enough data that we can safely copy back the unconsumed data and still leave enough space
217    // for one segment without needing a temporary buffer?
218    Value * const canCopy = b->CreateICmpULT(b->CreateGEP(baseBuffer, potentialItems), unreadData);
219    b->CreateLikelyCondBr(canCopy, copyBack, expandAndCopyBack);
220
221    // If so, just copy the data ...
222    b->SetInsertPoint(copyBack);
223    b->CreateMemCpy(baseBuffer, unreadData, remainingBytes, blockSize);
224    b->CreateBr(prepareBuffer);
225
226    // Otherwise, allocate a buffer with twice the capacity and copy the unconsumed data back into it
227    b->SetInsertPoint(expandAndCopyBack);
228    Value * const expandedCapacity = b->CreateShl(capacity, 1);
229    Value * const expandedBytes = b->CreateMul(expandedCapacity, codeUnitBytes);
230    Value * const expandedBuffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(expandedBytes), unreadData->getType());
231    b->CreateMemCpy(expandedBuffer, unreadData, remainingBytes, blockSize);
232    // Free the prior buffer if it exists
233    Value * const ancillaryBuffer = b->getScalarField("ancillaryBuffer");
234    b->CreateFree(ancillaryBuffer);
235    b->setScalarField("ancillaryBuffer", baseBuffer);
236    b->setScalarField("buffer", expandedBuffer);
237    b->setCapacity("sourceBuffer", expandedCapacity);
238    b->CreateBr(prepareBuffer);
239
240    b->SetInsertPoint(prepareBuffer);
241    PHINode * const newBaseBuffer = b->CreatePHI(baseBuffer->getType(), 2);
242    newBaseBuffer->addIncoming(baseBuffer, copyBack);
243    newBaseBuffer->addIncoming(expandedBuffer, expandAndCopyBack);
244    Value * const newBaseAddress = b->CreateGEP(newBaseBuffer, b->CreateNeg(offset));
245    b->setBaseAddress("sourceBuffer", newBaseAddress);
246    b->CreateBr(readData);
247
248    // Regardless of whether we're simply appending data or had to allocate a new buffer, read a new page
249    // of data into the input source buffer. If we fail to read a full page ...
250    b->SetInsertPoint(readData);
251    Value * const sourceBuffer = b->getRawOutputPointer("sourceBuffer", produced);
252    Value * const fd = b->getScalarField("fileDescriptor");
253    Constant * const bytesToRead = ConstantExpr::getMul(itemsToRead, codeUnitBytes);
254    Value * const bytesRead = b->CreateReadCall(fd, sourceBuffer, bytesToRead);
255    Value * const itemsRead = b->CreateUDiv(bytesRead, codeUnitBytes);
256    Value * const itemsBuffered = b->CreateAdd(produced, itemsRead);
257    b->CreateUnlikelyCondBr(b->CreateICmpULT(itemsBuffered, itemsPending), setTermination, readExit);
258
259    // ... set the termination signal.
260    b->SetInsertPoint(setTermination);
261    Value * const bytesToZero = b->CreateMul(b->CreateSub(itemsPending, itemsBuffered), codeUnitBytes);
262    b->CreateMemZero(b->getRawOutputPointer("sourceBuffer", itemsBuffered), bytesToZero);
263    b->setScalarField("fileItems", itemsBuffered);
264    b->setTerminationSignal();
265    b->CreateBr(readExit);
266
267    b->SetInsertPoint(readExit);
268    PHINode * const itemsProduced = b->CreatePHI(itemsPending->getType(), 2);
269    itemsProduced->addIncoming(itemsPending, readData);
270    itemsProduced->addIncoming(itemsBuffered, setTermination);
271    b->setProducedItemCount("sourceBuffer", itemsProduced);
272}
273
274void ReadSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
275    b->CreateFree(b->getScalarField("ancillaryBuffer"));
276    b->CreateFree(b->getScalarField("buffer"));
277}
278
279/// Hybrid MMap/Read source kernel
280
281void FDSourceKernel::linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & b) {
282    mFileSizeFunction = MMapSourceKernel::linkFileSizeMethod(b);
283}
284
285void FDSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
286    BasicBlock * finalizeRead = b->CreateBasicBlock("finalizeRead");
287    BasicBlock * finalizeMMap = b->CreateBasicBlock("finalizeMMap");
288    BasicBlock * finalizeDone = b->CreateBasicBlock("finalizeDone");
289    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), finalizeMMap, finalizeRead);
290    b->SetInsertPoint(finalizeMMap);
291    MMapSourceKernel::freeBuffer(b, mCodeUnitWidth);
292    b->CreateBr(finalizeDone);
293    b->SetInsertPoint(finalizeRead);
294    ReadSourceKernel::freeBuffer(b);
295    b->CreateBr(finalizeDone);
296    b->SetInsertPoint(finalizeDone);
297}
298
299void FDSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
300    BasicBlock * initializeRead = b->CreateBasicBlock("initializeRead");
301    BasicBlock * tryMMap = b->CreateBasicBlock("tryMMap");
302    BasicBlock * initializeMMap = b->CreateBasicBlock("initializeMMap");
303    BasicBlock * initializeDone = b->CreateBasicBlock("initializeDone");
304
305    // The source will use MMapSource or readSoure kernel logic depending on the useMMap
306    // parameter, possibly overridden.
307    Value * useMMap = b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty());
308    // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic.
309    Value * fd = b->getScalarField("fileDescriptor");
310    useMMap = b->CreateAnd(useMMap, b->CreateICmpNE(fd, b->getInt32(STDIN_FILENO)));
311    b->CreateCondBr(useMMap, tryMMap, initializeRead);
312
313    b->SetInsertPoint(tryMMap);
314    // If the fileSize is 0, we may have a virtual file such as /proc/cpuinfo
315    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(mFileSizeFunction, fd), b->getSizeTy());
316    useMMap = b->CreateICmpNE(fileSize, b->getSize(0));
317    b->CreateCondBr(useMMap, initializeMMap, initializeRead);
318
319    b->SetInsertPoint(initializeMMap);
320    MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, mStride, b);
321    b->CreateBr(initializeDone);
322
323    b->SetInsertPoint(initializeRead);
324    // Ensure that readSource logic is used throughout.
325    b->setScalarField("useMMap", b->getInt8(0));
326    ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, mStride,b);
327    b->CreateBr(initializeDone);
328    b->SetInsertPoint(initializeDone);
329}
330
331void FDSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
332    BasicBlock * DoSegmentRead = b->CreateBasicBlock("DoSegmentRead");
333    BasicBlock * DoSegmentMMap = b->CreateBasicBlock("DoSegmentMMap");
334    BasicBlock * DoSegmentDone = b->CreateBasicBlock("DoSegmentDone");
335    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), DoSegmentMMap, DoSegmentRead);
336    b->SetInsertPoint(DoSegmentMMap);
337    MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride, b);
338    b->CreateBr(DoSegmentDone);
339    b->SetInsertPoint(DoSegmentRead);
340    ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, mStride, b);
341    b->CreateBr(DoSegmentDone);
342    b->SetInsertPoint(DoSegmentDone);
343}
344
345/// MEMORY SOURCE KERNEL
346
347void MemorySourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
348    Value * const fileSource = b->getScalarField("fileSource");
349    b->setBaseAddress("sourceBuffer", fileSource);
350    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::EnableAsserts))) {
351        b->CreateAssert(fileSource, getName() + " fileSource cannot be null");
352    }
353    Value * const fileItems = b->getScalarField("fileItems");
354    b->setCapacity("sourceBuffer", fileItems);
355}
356
357void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
358
359    Constant * const STRIDE_ITEMS = b->getSize(getStride());
360    Constant * const STRIDE_SIZE = b->getSize(getStride() * mCodeUnitWidth);
361    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
362
363    BasicBlock * const entry = b->GetInsertBlock();
364    BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary");
365    BasicBlock * const exit = b->CreateBasicBlock("exit");
366
367    Value * const fileItems = b->getScalarField("fileItems");
368    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
369    Value * const nextProducedItems = b->CreateAdd(producedItems, STRIDE_ITEMS);
370    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
371    b->CreateUnlikelyCondBr(lastPage, createTemporary, exit);
372
373    b->SetInsertPoint(createTemporary);
374    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
375    Value * readStart = nullptr;
376    Value * readEnd = nullptr;
377
378    // compute the range of our unconsumed buffer slice
379    if (LLVM_UNLIKELY(mStreamSetCount > 1)) {
380        Constant * const ZERO = b->getSize(0);
381        const StreamSetBuffer * const sourceBuffer = getOutputStreamSetBuffer("sourceBuffer");
382        Value * const fromIndex = b->CreateUDiv(consumedItems, BLOCK_WIDTH);
383        readStart = sourceBuffer->getStreamBlockPtr(b.get(), ZERO, fromIndex);
384        Value * const toIndex = b->CreateUDivCeil(fileItems, BLOCK_WIDTH);
385        // since we know this is an ExternalBuffer, we don't need to consider any potential modulus calculations.
386        readEnd = sourceBuffer->getStreamBlockPtr(b.get(), ZERO, toIndex);
387    } else {
388        // make sure our copy is block-aligned
389        Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
390        readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
391        readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
392    }
393
394    DataLayout DL(b->getModule());
395    Type * const intPtrTy = DL.getIntPtrType(readEnd->getType());
396    Value * const readEndInt = b->CreatePtrToInt(readEnd, intPtrTy);
397    Value * const readStartInt = b->CreatePtrToInt(readStart, intPtrTy);
398    Value * const unconsumedBytes = b->CreateTrunc(b->CreateSub(readEndInt, readStartInt), b->getSizeTy());
399    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), STRIDE_SIZE);
400    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
401    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
402    b->setScalarField("ancillaryBuffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
403    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
404    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
405
406    // get the difference between our base and from position then compute an offsetted temporary buffer address
407    Value * const base = b->getBaseAddress("sourceBuffer");
408    Value * const baseInt = b->CreatePtrToInt(base, intPtrTy);
409    Value * const diff = b->CreateSub(baseInt, readStartInt);
410    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
411    // set the temporary buffer as the new source buffer
412    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
413    b->setTerminationSignal();
414    BasicBlock * const terminationExit = b->GetInsertBlock();
415    b->CreateBr(exit);
416
417    b->SetInsertPoint(exit);
418    PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
419    newProducedItems->addIncoming(nextProducedItems, entry);
420    newProducedItems->addIncoming(fileItems, terminationExit);
421    b->setProducedItemCount("sourceBuffer", newProducedItems);   
422}
423
424void MemorySourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
425    b->CreateFree(b->getScalarField("ancillaryBuffer"));
426}
427
428MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const fd, StreamSet * const outputStream)
429: SegmentOrientedKernel("mmap_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
430// input streams
431,{}
432// output streams
433,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
434// input scalars
435,{Binding{"fileDescriptor", fd}}
436// output scalars
437,{Binding{b->getSizeTy(), "fileItems"}}
438// internal scalars
439,{})
440, mCodeUnitWidth(outputStream->getFieldWidth())
441, mFileSizeFunction(nullptr) {
442    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
443    addInternalScalar(codeUnitPtrTy, "buffer");
444    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
445    addAttribute(MustExplicitlyTerminate());
446    setStride(codegen::SegmentSize);
447}
448
449ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const fd, StreamSet * const outputStream)
450: SegmentOrientedKernel("read_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
451// input streams
452,{}
453// output streams
454,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
455// input scalars
456,{Binding{"fileDescriptor", fd}}
457// output scalars
458,{Binding{b->getSizeTy(), "fileItems"}}
459// internal scalars
460,{})
461, mCodeUnitWidth(outputStream->getFieldWidth()) {
462    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
463    addInternalScalar(codeUnitPtrTy, "buffer");
464    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
465    addAttribute(MustExplicitlyTerminate());
466    setStride(codegen::SegmentSize);
467}
468
469
470FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, Scalar * const useMMap, Scalar * const fd, StreamSet * const outputStream)
471: SegmentOrientedKernel("FD_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth())
472// input streams
473,{}
474// output stream
475,{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}}
476// input scalar
477,{Binding{"useMMap", useMMap}
478, Binding{"fileDescriptor", fd}}
479// output scalar
480,{Binding{b->getSizeTy(), "fileItems"}}
481// internal scalars
482,{})
483, mCodeUnitWidth(outputStream->getFieldWidth())
484, mFileSizeFunction(nullptr) {
485    PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
486    addInternalScalar(codeUnitPtrTy, "buffer");
487    addInternalScalar(codeUnitPtrTy, "ancillaryBuffer");
488    addAttribute(MustExplicitlyTerminate());
489    setStride(codegen::SegmentSize);
490}
491
492MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> &, Scalar * fileSource, Scalar * fileItems, StreamSet * const outputStream)
493: SegmentOrientedKernel("memory_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(outputStream->getFieldWidth()) + ":" + std::to_string(outputStream->getNumElements()),
494// input streams
495{},
496// output stream
497{Binding{"sourceBuffer", outputStream, FixedRate(), ManagedBuffer()}},
498// input scalar
499{Binding{"fileSource", fileSource}, Binding{"fileItems", fileItems}},
500{},
501// internal scalar
502{Binding{fileSource->getType(), "buffer"}
503,Binding{fileSource->getType(), "ancillaryBuffer"}
504})
505, mStreamSetCount(outputStream->getNumElements())
506, mCodeUnitWidth(outputStream->getFieldWidth()) {
507    addAttribute(MustExplicitlyTerminate());
508    setStride(codegen::SegmentSize);
509}
510
511}
Note: See TracBrowser for help on using the repository browser.