source: icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp @ 6008

Last change on this file since 6008 was 6008, checked in by nmedfort, 15 months ago

Removed temporary buffers from pipeline and placed them in the source kernels.

File size: 22.6 KB
Line 
1/*
2 *  Copyright (c) 2018 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5#include "source_kernel.h"
6#include <kernels/kernel_builder.h>
7#include <kernels/streamset.h>
8#include <llvm/IR/Module.h>
9#include <sys/stat.h>
10#include <fcntl.h>
11#include <toolchain/toolchain.h>
12
13using namespace llvm;
14
15extern "C" uint64_t file_size(const uint32_t fd) {
16    struct stat st;
17    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
18        st.st_size = 0;
19    }
20    return st.st_size;
21}
22
23namespace kernel {
24
25/// MMAP SOURCE KERNEL
26
27Function * MMapSourceKernel::linkFileSizeMethod(const std::unique_ptr<kernel::KernelBuilder> & b) {
28    return b->LinkFunction("file_size", &file_size);
29}
30
31void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
32
33    BasicBlock * const emptyFile = b->CreateBasicBlock("emptyFile");
34    BasicBlock * const nonEmptyFile = b->CreateBasicBlock("NonEmptyFile");
35    BasicBlock * const exit = b->CreateBasicBlock("Exit");
36    IntegerType * const sizeTy = b->getSizeTy();
37    Value * const fd = b->getScalarField("fileDescriptor");
38    assert (fileSizeMethod);
39    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(fileSizeMethod, fd), sizeTy);
40    b->CreateLikelyCondBr(b->CreateIsNotNull(fileSize), nonEmptyFile, emptyFile);
41
42    b->SetInsertPoint(nonEmptyFile);
43    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
44    Value * const fileBuffer = b->CreatePointerCast(b->CreateFileSourceMMap(fd, fileSize), codeUnitPtrTy);
45    b->setScalarField("buffer", fileBuffer);
46    b->setBaseAddress("sourceBuffer", fileBuffer);
47    b->CreateMAdvise(fileBuffer, fileSize, CBuilder::ADVICE_WILLNEED);
48    if (LLVM_UNLIKELY(codeUnitWidth > 8)) {
49        fileSize = b->CreateUDiv(fileSize, b->getSize(codeUnitWidth / 8));
50    }
51    b->setBufferedSize("sourceBuffer", fileSize);
52    b->setScalarField("fileSize", fileSize);
53    b->setProducedItemCount("sourceBuffer", fileSize);
54    b->setCapacity("sourceBuffer", fileSize);
55    b->CreateBr(exit);
56
57    b->SetInsertPoint(emptyFile);
58    ConstantInt * const PAGE_SIZE = b->getSize(getpagesize());
59    ConstantInt * const ZERO = b->getSize(0);
60    Value * const emptyFilePtr = b->CreatePointerCast(b->CreateAnonymousMMap(PAGE_SIZE), codeUnitPtrTy);
61    b->setScalarField("buffer", emptyFilePtr);
62    b->setBaseAddress("sourceBuffer", emptyFilePtr);
63    b->setScalarField("fileSize", ZERO);
64    b->setBufferedSize("sourceBuffer", PAGE_SIZE);
65    b->setCapacity("sourceBuffer", PAGE_SIZE);
66    b->setTerminationSignal();
67    b->CreateBr(exit);
68
69    b->SetInsertPoint(exit);
70}
71
72
73void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) {
74
75    BasicBlock * const dropPages = b->CreateBasicBlock("dropPages");
76    BasicBlock * const checkRemaining = b->CreateBasicBlock("checkRemaining");
77    BasicBlock * const setTermination = b->CreateBasicBlock("setTermination");
78    BasicBlock * const exit = b->CreateBasicBlock("mmapSourceExit");
79
80    Constant * const PAGE_SIZE = b->getSize(getpagesize());
81    Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
82    Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8);
83
84    Value * const fileItems = b->getScalarField("fileSize");
85    Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
86    Value * const consumedBytes = b->CreateMul(consumedItems, CODE_UNIT_BYTES);
87    Value * const consumedPageOffset = b->CreateAnd(consumedBytes, ConstantExpr::getNeg(PAGE_SIZE));
88    Value * const consumedBuffer = b->getRawOutputPointer("sourceBuffer", consumedPageOffset);
89    Value * const readableBuffer = b->getScalarField("buffer");
90    Value * const unnecessaryBytes = b->CreatePtrDiff(consumedBuffer, readableBuffer);
91
92    // avoid calling madvise unless an actual page table change could occur
93    b->CreateLikelyCondBr(b->CreateIsNotNull(unnecessaryBytes), dropPages, checkRemaining);
94
95    b->SetInsertPoint(dropPages);
96    // instruct the OS that it can safely drop any fully consumed pages
97    b->CreateMAdvise(readableBuffer, unnecessaryBytes, CBuilder::ADVICE_DONTNEED);
98    b->setScalarField("buffer", b->CreateGEP(readableBuffer, unnecessaryBytes));
99    b->CreateBr(checkRemaining);
100
101    // determine whether or not we've exhausted the file buffer
102    b->SetInsertPoint(checkRemaining);
103    Value * const producedItems = b->getProducedItemCount("sourceBuffer");
104    Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE);
105    Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
106    b->CreateUnlikelyCondBr(lastPage, setTermination, exit);
107
108    // If this is the last page, create a temporary buffer of up to two pages size, copy the unconsumed data
109    // and zero any bytes that are not used.
110    b->SetInsertPoint(setTermination);
111    Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
112    Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
113    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
114    Value * const unconsumedBytes = b->CreatePtrDiff(readEnd, readStart);
115    Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
116    Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
117    b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
118    b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
119    // get the difference between our base and from position then compute an offsetted temporary buffer address
120    Value * const base = b->getBaseAddress("sourceBuffer");
121    Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
122    Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
123    b->CreateConsumerWait();
124    // Unmap the file buffer and set the temporary buffer as the new source buffer
125    Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES);
126    b->CreateMUnmap(base, fileSize);
127    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
128    b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
129    b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
130    b->setTerminationSignal();
131    BasicBlock * const terminationExit = b->GetInsertBlock();
132    b->CreateBr(exit);
133
134    // finally, set the "produced" count to reflect current position in the file
135    b->SetInsertPoint(exit);
136    PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
137    newProducedItems->addIncoming(nextProducedItems, checkRemaining);
138    newProducedItems->addIncoming(fileItems, terminationExit);
139    b->setProducedItemCount("sourceBuffer", newProducedItems);
140}
141
142void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
143    b->CreateFree(b->getScalarField("buffer"));
144}
145
146/// READ SOURCE KERNEL
147
148void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
149    const unsigned pageSize = getpagesize();
150    const auto bufferSize = std::max(pageSize * 8, codegen::SegmentSize * stride  * 4);
151    ConstantInt * const bufferItems = b->getSize(bufferSize);
152    const auto codeUnitSize = codeUnitWidth / 8;
153    ConstantInt * const bufferBytes = b->getSize(bufferSize * codeUnitSize);
154    PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo();
155    Value * const buffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy);
156    b->setBaseAddress("sourceBuffer", buffer);
157    b->setScalarField("buffer", buffer);
158    b->setCapacity("sourceBuffer", bufferItems);
159}
160
161void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const unsigned stride, const std::unique_ptr<KernelBuilder> & b) {
162
163    const unsigned pageSize = getpagesize();
164    ConstantInt * const itemsToRead = b->getSize(std::max(pageSize, codegen::SegmentSize * stride * 2));
165    ConstantInt * const codeUnitBytes = b->getSize(codeUnitWidth / 8);
166    ConstantInt * const itemsPerSegment = b->getSize(codegen::SegmentSize * stride);
167
168    BasicBlock * const entry = b->GetInsertBlock();
169    BasicBlock * const checkData = b->CreateBasicBlock("CheckData");
170    BasicBlock * const moveData = b->CreateBasicBlock("MoveData");
171    BasicBlock * const prepareBuffer = b->CreateBasicBlock("PrepareBuffer");
172    BasicBlock * const readData = b->CreateBasicBlock("ReadData");
173    BasicBlock * const setTermination = b->CreateBasicBlock("SetTermination");
174    BasicBlock * const readExit = b->CreateBasicBlock("ReadExit");
175
176    // Do we have enough unread data to support one segment?
177    Value * const produced = b->getProducedItemCount("sourceBuffer");
178    Value * const buffered = b->getBufferedSize("sourceBuffer");
179    Value * const itemsPending = b->CreateAdd(produced, itemsPerSegment);
180
181    b->CreateLikelyCondBr(b->CreateICmpULT(itemsPending, buffered), readExit, checkData);
182
183    // Can we append to our existing buffer without impacting any subsequent kernel?
184    b->SetInsertPoint(checkData);
185    Value * const capacity = b->getCapacity("sourceBuffer");
186    Value * const readEnd = b->getRawOutputPointer("sourceBuffer", b->CreateAdd(buffered, itemsToRead));
187    Value * const baseBuffer = b->getScalarField("buffer");
188    Value * const bufferLimit = b->CreateGEP(baseBuffer, capacity);
189    b->CreateLikelyCondBr(b->CreateICmpULE(readEnd, bufferLimit), readData, moveData);
190
191    // First wait on any consumers to finish processing then check how much data has been consumed.
192    b->SetInsertPoint(moveData);
193    b->CreateConsumerWait();
194
195    // Then determine how much data has been consumed and how much needs to be copied back, noting
196    // that our "unproduced" data must be block aligned.
197    BasicBlock * const copyBack = b->CreateBasicBlock("CopyBack");
198    BasicBlock * const expandAndCopyBack = b->CreateBasicBlock("ExpandAndCopyBack");
199
200    const auto blockSize = b->getBitBlockWidth() / 8;
201    Constant * const blockSizeAlignmentMask = ConstantExpr::getNeg(b->getSize(blockSize));
202    Value * const consumed = b->getConsumedItemCount("sourceBuffer");
203    Value * const offset = b->CreateAnd(consumed, blockSizeAlignmentMask);
204    Value * const unreadData = b->getRawOutputPointer("sourceBuffer", offset);
205    Value * const remainingItems = b->CreateSub(buffered, offset);
206    Value * const remainingBytes = b->CreateMul(remainingItems, codeUnitBytes);
207
208    // Have we consumed enough data that we can safely copy back the unconsumed data without needing a temporary buffer?
209    Value * const canCopy = b->CreateICmpULT(b->CreateGEP(baseBuffer, remainingItems), b->getRawOutputPointer("sourceBuffer", offset));
210    b->CreateLikelyCondBr(canCopy, copyBack, expandAndCopyBack);
211
212    // If so, just copy the data ...
213    b->SetInsertPoint(copyBack);
214    b->CreateMemCpy(baseBuffer, unreadData, remainingBytes, blockSize);
215    b->CreateBr(prepareBuffer);
216
217    // Otherwise, allocate a buffer with twice the capacity and copy the unconsumed data back into it
218    b->SetInsertPoint(expandAndCopyBack);
219    Value * const expandedCapacity = b->CreateShl(capacity, 1);
220    Value * const expandedBytes = b->CreateMul(expandedCapacity, codeUnitBytes);
221    Value * const expandedBuffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(expandedBytes), unreadData->getType());
222    b->CreateMemCpy(expandedBuffer, unreadData, remainingBytes, blockSize);
223    b->setScalarField("buffer", expandedBuffer);
224    b->setCapacity("sourceBuffer", expandedCapacity);
225    b->CreateFree(baseBuffer);
226    b->CreateBr(prepareBuffer);
227
228    b->SetInsertPoint(prepareBuffer);
229    PHINode * newBaseBuffer = b->CreatePHI(baseBuffer->getType(), 2);
230    newBaseBuffer->addIncoming(baseBuffer, copyBack);
231    newBaseBuffer->addIncoming(expandedBuffer, expandAndCopyBack);
232    b->setBaseAddress("sourceBuffer", b->CreateGEP(newBaseBuffer, b->CreateNeg(offset)));
233    b->CreateBr(readData);
234
235    // Regardless of whether we're simply appending data or had to allocate a new buffer, read a new page
236    // of data into the input source buffer. If we fail to read a full page ...
237    b->SetInsertPoint(readData);
238    Value * const sourceBuffer = b->getRawOutputPointer("sourceBuffer", buffered);
239    Value * const fd = b->getScalarField("fileDescriptor");
240    Constant * const bytesToRead = ConstantExpr::getMul(itemsToRead, codeUnitBytes);
241    Value * const bytesRead = b->CreateReadCall(fd, sourceBuffer, bytesToRead);
242    Value * const itemsRead = b->CreateUDiv(bytesRead, codeUnitBytes);
243    Value * const itemsBuffered = b->CreateAdd(buffered, itemsRead);
244    b->setBufferedSize("sourceBuffer", itemsBuffered);
245    b->CreateUnlikelyCondBr(b->CreateICmpULT(itemsBuffered, itemsPending), setTermination, readExit);
246
247    // ... set the termination signal.
248    b->SetInsertPoint(setTermination);
249    Value * const bytesToZero = b->CreateMul(b->CreateSub(itemsPending, itemsBuffered), codeUnitBytes);
250    b->CreateMemZero(b->getRawOutputPointer("sourceBuffer", itemsBuffered), bytesToZero);
251    b->setTerminationSignal();
252    b->CreateBr(readExit);
253
254    readExit->moveAfter(setTermination);
255    b->SetInsertPoint(readExit);
256    PHINode * const itemsProduced = b->CreatePHI(itemsPending->getType(), 3);
257    itemsProduced->addIncoming(itemsPending, entry);
258    itemsProduced->addIncoming(itemsPending, readData);
259    itemsProduced->addIncoming(itemsBuffered, setTermination);
260    b->setProducedItemCount("sourceBuffer", itemsProduced);
261}
262
263void ReadSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) {
264    b->CreateFree(b->getScalarField("buffer"));
265}
266
267/// Hybrid MMap/Read source kernel
268
269void FDSourceKernel::linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & b) {
270    mFileSizeFunction = MMapSourceKernel::linkFileSizeMethod(b);
271}
272
273void FDSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
274    BasicBlock * finalizeRead = b->CreateBasicBlock("finalizeRead");
275    BasicBlock * finalizeMMap = b->CreateBasicBlock("finalizeMMap");
276    BasicBlock * finalizeDone = b->CreateBasicBlock("finalizeDone");
277    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), finalizeMMap, finalizeRead);
278    b->SetInsertPoint(finalizeMMap);
279    MMapSourceKernel::freeBuffer(b);
280    b->CreateBr(finalizeDone);
281    b->SetInsertPoint(finalizeRead);
282    ReadSourceKernel::freeBuffer(b);
283    b->CreateBr(finalizeDone);
284    b->SetInsertPoint(finalizeDone);
285}
286
287void FDSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
288    BasicBlock * initializeRead = b->CreateBasicBlock("initializeRead");
289    BasicBlock * tryMMap = b->CreateBasicBlock("tryMMap");
290    BasicBlock * initializeMMap = b->CreateBasicBlock("initializeMMap");
291    BasicBlock * initializeDone = b->CreateBasicBlock("initializeDone");
292
293    // The source will use MMapSource or readSoure kernel logic depending on the useMMap
294    // parameter, possibly overridden.
295    Value * useMMap = b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty());
296    // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic.
297    Value * fd = b->getScalarField("fileDescriptor");
298    useMMap = b->CreateAnd(useMMap, b->CreateICmpNE(fd, b->getInt32(STDIN_FILENO)));
299    b->CreateCondBr(useMMap, tryMMap, initializeRead);
300
301    b->SetInsertPoint(tryMMap);
302    // If the fileSize is 0, we may have a virtual file such as /proc/cpuinfo
303    Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(mFileSizeFunction, fd), b->getSizeTy());
304    useMMap = b->CreateICmpNE(fileSize, b->getSize(0));
305    b->CreateCondBr(useMMap, initializeMMap, initializeRead);
306    b->SetInsertPoint(initializeMMap);
307    MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, b);
308    b->CreateBr(initializeDone);
309
310    b->SetInsertPoint(initializeRead);
311    // Ensure that readSource logic is used throughout.
312    b->setScalarField("useMMap", b->getInt8(0));
313    ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, getStride(), b);
314    b->CreateBr(initializeDone);
315    b->SetInsertPoint(initializeDone);
316}
317
318void FDSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
319    BasicBlock * DoSegmentRead = b->CreateBasicBlock("DoSegmentRead");
320    BasicBlock * DoSegmentMMap = b->CreateBasicBlock("DoSegmentMMap");
321    BasicBlock * DoSegmentDone = b->CreateBasicBlock("DoSegmentDone");
322    b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), DoSegmentMMap, DoSegmentRead);
323    b->SetInsertPoint(DoSegmentMMap);
324    MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, b);
325    b->CreateBr(DoSegmentDone);
326    b->SetInsertPoint(DoSegmentRead);
327    ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, getStride(), b);
328    b->CreateBr(DoSegmentDone);
329    b->SetInsertPoint(DoSegmentDone);
330}
331
332/// MEMORY SOURCE KERNEL
333
334void MemorySourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) {
335    Value * const fileSource = b->getScalarField("fileSource");
336    b->setBaseAddress("sourceBuffer", fileSource);
337    Value * const fileSize = b->getScalarField("fileSize");
338    b->setBufferedSize("sourceBuffer", fileSize);
339    b->setCapacity("sourceBuffer", fileSize);
340    if (mStreamSetCount > 1) {
341        b->setProducedItemCount("sourceBuffer", fileSize);
342        b->setTerminationSignal();
343    }
344}
345
346void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) {
347    if (mStreamSetCount == 1) {
348        Constant * const PAGE_SIZE = b->getSize(getStride());
349        Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth());
350
351        BasicBlock * const entry = b->GetInsertBlock();
352        BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary");
353        BasicBlock * const exit = b->CreateBasicBlock("exit");
354
355        Value * const fileItems = b->getScalarField("fileSize");
356        Value * const producedItems = b->getProducedItemCount("sourceBuffer");
357        Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE);
358        Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems);
359        b->CreateUnlikelyCondBr(lastPage, createTemporary, exit);
360
361        b->SetInsertPoint(createTemporary);
362        Value * const consumedItems = b->getConsumedItemCount("sourceBuffer");
363        Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH));
364        Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset);
365        Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems);
366        Value * const unconsumedBytes = b->CreatePtrDiff(readEnd, readStart);
367        Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE);
368        Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment());
369        b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1);
370        b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1);
371
372        // get the difference between our base and from position then compute an offsetted temporary buffer address
373        Value * const base = b->getBaseAddress("sourceBuffer");
374        Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart);
375        Value * const offsettedBuffer = b->CreateGEP(buffer, diff);
376        b->CreateConsumerWait();
377
378        // set the temporary buffer as the new source buffer
379        PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo();
380        b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy));
381        b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy));
382        b->setTerminationSignal();
383        BasicBlock * const terminationExit = b->GetInsertBlock();
384        b->CreateBr(exit);
385
386        b->SetInsertPoint(exit);
387        PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2);
388        newProducedItems->addIncoming(nextProducedItems, entry);
389        newProducedItems->addIncoming(fileItems, terminationExit);
390        b->setProducedItemCount("sourceBuffer", newProducedItems);
391    }
392}
393
394void MemorySourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) {
395    if (mStreamSetCount == 1) {
396        b->CreateFree(b->getScalarField("buffer"));
397    }
398}
399
400MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
401: SegmentOrientedKernel("mmap_source@" + std::to_string(codeUnitWidth)
402, {}
403, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
404, {Binding{b->getInt32Ty(), "fileDescriptor"}}
405, {Binding{b->getSizeTy(), "fileSize"}}
406, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
407, mCodeUnitWidth(codeUnitWidth)
408, mFileSizeFunction(nullptr) {
409    addAttribute(MustExplicitlyTerminate());
410    setStride(getpagesize());
411}
412
413ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
414: SegmentOrientedKernel("read_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(codeUnitWidth)
415, {}
416, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
417, {Binding{b->getInt32Ty(), "fileDescriptor"}}
418, {}
419, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
420, mCodeUnitWidth(codeUnitWidth) {
421    addAttribute(MustExplicitlyTerminate());
422    setStride(getpagesize());
423}
424
425
426FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth)
427: SegmentOrientedKernel("FD_source@" + std::to_string(codeUnitWidth)
428, {}
429// output stream
430, {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
431// input scalar
432, {Binding{b->getInt8Ty(), "useMMap"}, Binding{b->getInt32Ty(), "fileDescriptor"}}
433, {}
434// internal scalar
435, {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}, Binding{b->getSizeTy(), "fileSize"}})
436, mCodeUnitWidth(codeUnitWidth)
437, mFileSizeFunction(nullptr) {
438    addAttribute(MustExplicitlyTerminate());
439    setStride(getpagesize());
440}
441
442MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned streamSetCount, const unsigned codeUnitWidth)
443: SegmentOrientedKernel("memory_source@" + std::to_string(streamSetCount) + ":" + std::to_string(codeUnitWidth),
444{},
445// output stream
446{Binding{b->getStreamSetTy(streamSetCount, codeUnitWidth), "sourceBuffer"}},
447// input scalar
448{Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "fileSource"}, Binding{b->getSizeTy(), "fileSize"}},
449{},
450// internal scalar
451{Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}})
452, mStreamSetCount(streamSetCount)
453, mCodeUnitWidth(codeUnitWidth) {
454    addAttribute(MustExplicitlyTerminate());
455    setStride(getpagesize());
456}
457
458}
Note: See TracBrowser for help on using the repository browser.