1 | /* |
---|
2 | * Copyright (c) 2018 International Characters. |
---|
3 | * This software is licensed to the public under the Open Software License 3.0. |
---|
4 | */ |
---|
5 | #include "source_kernel.h" |
---|
6 | #include <kernels/kernel_builder.h> |
---|
7 | #include <kernels/streamset.h> |
---|
8 | #include <llvm/IR/Module.h> |
---|
9 | #include <sys/stat.h> |
---|
10 | #include <fcntl.h> |
---|
11 | #include <toolchain/toolchain.h> |
---|
12 | |
---|
13 | using namespace llvm; |
---|
14 | |
---|
15 | extern "C" uint64_t file_size(const uint32_t fd) { |
---|
16 | struct stat st; |
---|
17 | if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) { |
---|
18 | st.st_size = 0; |
---|
19 | } |
---|
20 | return st.st_size; |
---|
21 | } |
---|
22 | |
---|
23 | namespace kernel { |
---|
24 | |
---|
25 | /// MMAP SOURCE KERNEL |
---|
26 | |
---|
27 | Function * MMapSourceKernel::linkFileSizeMethod(const std::unique_ptr<kernel::KernelBuilder> & b) { |
---|
28 | return b->LinkFunction("file_size", &file_size); |
---|
29 | } |
---|
30 | |
---|
31 | void MMapSourceKernel::generateInitializeMethod(Function * const fileSizeMethod, const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) { |
---|
32 | |
---|
33 | BasicBlock * const emptyFile = b->CreateBasicBlock("emptyFile"); |
---|
34 | BasicBlock * const nonEmptyFile = b->CreateBasicBlock("NonEmptyFile"); |
---|
35 | BasicBlock * const exit = b->CreateBasicBlock("Exit"); |
---|
36 | IntegerType * const sizeTy = b->getSizeTy(); |
---|
37 | ConstantInt * const PAGE_SIZE = b->getSize(getpagesize()); |
---|
38 | Value * const fd = b->getScalarField("fileDescriptor"); |
---|
39 | assert (fileSizeMethod); |
---|
40 | Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(fileSizeMethod, fd), sizeTy); |
---|
41 | b->CreateLikelyCondBr(b->CreateIsNotNull(fileSize), nonEmptyFile, emptyFile); |
---|
42 | |
---|
43 | b->SetInsertPoint(nonEmptyFile); |
---|
44 | PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo(); |
---|
45 | Value * const fileBuffer = b->CreatePointerCast(b->CreateFileSourceMMap(fd, fileSize), codeUnitPtrTy); |
---|
46 | b->setScalarField("buffer", fileBuffer); |
---|
47 | b->setBaseAddress("sourceBuffer", fileBuffer); |
---|
48 | b->CreateMAdvise(fileBuffer, fileSize, CBuilder::ADVICE_WILLNEED); |
---|
49 | if (LLVM_UNLIKELY(codeUnitWidth > 8)) { |
---|
50 | fileSize = b->CreateUDiv(fileSize, b->getSize(codeUnitWidth / 8)); |
---|
51 | } |
---|
52 | b->setScalarField("fileSize", fileSize); |
---|
53 | b->CreateBr(exit); |
---|
54 | |
---|
55 | b->SetInsertPoint(emptyFile); |
---|
56 | Value * const emptyFilePtr = b->CreatePointerCast(b->CreateAnonymousMMap(PAGE_SIZE), codeUnitPtrTy); |
---|
57 | b->setScalarField("buffer", emptyFilePtr); |
---|
58 | b->setBaseAddress("sourceBuffer", emptyFilePtr); |
---|
59 | b->setScalarField("fileSize", PAGE_SIZE); |
---|
60 | b->setTerminationSignal(); |
---|
61 | b->CreateBr(exit); |
---|
62 | |
---|
63 | b->SetInsertPoint(exit); |
---|
64 | } |
---|
65 | |
---|
66 | |
---|
67 | void MMapSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) { |
---|
68 | |
---|
69 | BasicBlock * const dropPages = b->CreateBasicBlock("dropPages"); |
---|
70 | BasicBlock * const checkRemaining = b->CreateBasicBlock("checkRemaining"); |
---|
71 | BasicBlock * const setTermination = b->CreateBasicBlock("setTermination"); |
---|
72 | BasicBlock * const exit = b->CreateBasicBlock("mmapSourceExit"); |
---|
73 | |
---|
74 | Constant * const PAGE_SIZE = b->getSize(getpagesize()); |
---|
75 | Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth()); |
---|
76 | Constant * const CODE_UNIT_BYTES = b->getSize(codeUnitWidth / 8); |
---|
77 | |
---|
78 | Value * const fileItems = b->getScalarField("fileSize"); |
---|
79 | Value * const consumedItems = b->getConsumedItemCount("sourceBuffer"); |
---|
80 | Value * const consumedBytes = b->CreateMul(consumedItems, CODE_UNIT_BYTES); |
---|
81 | Value * const consumedPageOffset = b->CreateAnd(consumedBytes, ConstantExpr::getNeg(PAGE_SIZE)); |
---|
82 | Value * const consumedBuffer = b->getRawOutputPointer("sourceBuffer", consumedPageOffset); |
---|
83 | Value * const readableBuffer = b->getScalarField("buffer"); |
---|
84 | Value * const unnecessaryBytes = b->CreatePtrDiff(consumedBuffer, readableBuffer); |
---|
85 | |
---|
86 | // avoid calling madvise unless an actual page table change could occur |
---|
87 | b->CreateLikelyCondBr(b->CreateIsNotNull(unnecessaryBytes), dropPages, checkRemaining); |
---|
88 | |
---|
89 | b->SetInsertPoint(dropPages); |
---|
90 | // instruct the OS that it can safely drop any fully consumed pages |
---|
91 | b->CreateMAdvise(readableBuffer, unnecessaryBytes, CBuilder::ADVICE_DONTNEED); |
---|
92 | b->setScalarField("buffer", b->CreateGEP(readableBuffer, unnecessaryBytes)); |
---|
93 | b->CreateBr(checkRemaining); |
---|
94 | |
---|
95 | // determine whether or not we've exhausted the file buffer |
---|
96 | b->SetInsertPoint(checkRemaining); |
---|
97 | Value * const producedItems = b->getProducedItemCount("sourceBuffer"); |
---|
98 | Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE); |
---|
99 | Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems); |
---|
100 | b->CreateUnlikelyCondBr(lastPage, setTermination, exit); |
---|
101 | |
---|
102 | // If this is the last page, create a temporary buffer of up to two pages size, copy the unconsumed data |
---|
103 | // and zero any bytes that are not used. |
---|
104 | b->SetInsertPoint(setTermination); |
---|
105 | Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH)); |
---|
106 | Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset); |
---|
107 | Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems); |
---|
108 | Value * const unconsumedBytes = b->CreateTrunc(b->CreatePtrDiff(readEnd, readStart), b->getSizeTy()); |
---|
109 | Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE); |
---|
110 | Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment()); |
---|
111 | b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1); |
---|
112 | b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1); |
---|
113 | // get the difference between our base and from position then compute an offsetted temporary buffer address |
---|
114 | Value * const base = b->getBaseAddress("sourceBuffer"); |
---|
115 | Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart); |
---|
116 | Value * const offsettedBuffer = b->CreateGEP(buffer, diff); |
---|
117 | b->CreateConsumerWait(); |
---|
118 | // Unmap the file buffer and set the temporary buffer as the new source buffer |
---|
119 | Value * const fileSize = b->CreateMul(fileItems, CODE_UNIT_BYTES); |
---|
120 | b->CreateMUnmap(base, fileSize); |
---|
121 | PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo(); |
---|
122 | b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy)); |
---|
123 | b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy)); |
---|
124 | b->setTerminationSignal(); |
---|
125 | BasicBlock * const terminationExit = b->GetInsertBlock(); |
---|
126 | b->CreateBr(exit); |
---|
127 | |
---|
128 | // finally, set the "produced" count to reflect current position in the file |
---|
129 | b->SetInsertPoint(exit); |
---|
130 | PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2); |
---|
131 | newProducedItems->addIncoming(nextProducedItems, checkRemaining); |
---|
132 | newProducedItems->addIncoming(fileItems, terminationExit); |
---|
133 | b->setProducedItemCount("sourceBuffer", newProducedItems); |
---|
134 | } |
---|
135 | |
---|
136 | void MMapSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) { |
---|
137 | b->CreateFree(b->getScalarField("buffer")); |
---|
138 | } |
---|
139 | |
---|
140 | /// READ SOURCE KERNEL |
---|
141 | |
---|
142 | void ReadSourceKernel::generateInitializeMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) { |
---|
143 | const unsigned pageSize = getpagesize(); |
---|
144 | ConstantInt * const bufferItems = b->getSize(pageSize * 4); |
---|
145 | const auto codeUnitSize = codeUnitWidth / 8; |
---|
146 | ConstantInt * const bufferBytes = b->getSize(pageSize * 4 * codeUnitSize); |
---|
147 | PointerType * const codeUnitPtrTy = b->getIntNTy(codeUnitWidth)->getPointerTo(); |
---|
148 | Value * const buffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufferBytes), codeUnitPtrTy); |
---|
149 | b->setBaseAddress("sourceBuffer", buffer); |
---|
150 | b->setScalarField("buffer", buffer); |
---|
151 | b->setCapacity("sourceBuffer", bufferItems); |
---|
152 | } |
---|
153 | |
---|
154 | void ReadSourceKernel::generateDoSegmentMethod(const unsigned codeUnitWidth, const std::unique_ptr<KernelBuilder> & b) { |
---|
155 | |
---|
156 | const unsigned pageSize = getpagesize(); |
---|
157 | ConstantInt * const itemsToRead = b->getSize(pageSize); |
---|
158 | ConstantInt * const codeUnitBytes = b->getSize(codeUnitWidth / 8); |
---|
159 | |
---|
160 | BasicBlock * const moveData = b->CreateBasicBlock("MoveData"); |
---|
161 | BasicBlock * const prepareBuffer = b->CreateBasicBlock("PrepareBuffer"); |
---|
162 | BasicBlock * const readData = b->CreateBasicBlock("ReadData"); |
---|
163 | BasicBlock * const setTermination = b->CreateBasicBlock("SetTermination"); |
---|
164 | BasicBlock * const readExit = b->CreateBasicBlock("ReadExit"); |
---|
165 | |
---|
166 | // Do we have enough unread data to support one segment? |
---|
167 | Value * const produced = b->getProducedItemCount("sourceBuffer"); |
---|
168 | Value * const itemsPending = b->CreateAdd(produced, itemsToRead); |
---|
169 | |
---|
170 | // Can we append to our existing buffer without impacting any subsequent kernel? |
---|
171 | Value * const capacity = b->getCapacity("sourceBuffer"); |
---|
172 | Value * const readEnd = b->getRawOutputPointer("sourceBuffer", itemsPending); |
---|
173 | Value * const baseBuffer = b->getScalarField("buffer"); |
---|
174 | Value * const bufferLimit = b->CreateGEP(baseBuffer, capacity); |
---|
175 | b->CreateLikelyCondBr(b->CreateICmpULE(readEnd, bufferLimit), readData, moveData); |
---|
176 | |
---|
177 | // First wait on any consumers to finish processing then check how much data has been consumed. |
---|
178 | b->SetInsertPoint(moveData); |
---|
179 | b->CreateConsumerWait(); |
---|
180 | |
---|
181 | // Then determine how much data has been consumed and how much needs to be copied back, noting |
---|
182 | // that our "unproduced" data must be block aligned. |
---|
183 | BasicBlock * const copyBack = b->CreateBasicBlock("CopyBack"); |
---|
184 | BasicBlock * const expandAndCopyBack = b->CreateBasicBlock("ExpandAndCopyBack"); |
---|
185 | |
---|
186 | const auto blockSize = b->getBitBlockWidth() / 8; |
---|
187 | Constant * const blockSizeAlignmentMask = ConstantExpr::getNeg(b->getSize(blockSize)); |
---|
188 | Value * const consumed = b->getConsumedItemCount("sourceBuffer"); |
---|
189 | Value * const offset = b->CreateAnd(consumed, blockSizeAlignmentMask); |
---|
190 | Value * const unreadData = b->getRawOutputPointer("sourceBuffer", offset); |
---|
191 | Value * const remainingItems = b->CreateSub(produced, offset); |
---|
192 | Value * const remainingBytes = b->CreateMul(remainingItems, codeUnitBytes); |
---|
193 | |
---|
194 | // Have we consumed enough data that we can safely copy back the unconsumed data without needing a temporary buffer? |
---|
195 | Value * const canCopy = b->CreateICmpULT(b->CreateGEP(baseBuffer, remainingItems), b->getRawOutputPointer("sourceBuffer", offset)); |
---|
196 | b->CreateLikelyCondBr(canCopy, copyBack, expandAndCopyBack); |
---|
197 | |
---|
198 | // If so, just copy the data ... |
---|
199 | b->SetInsertPoint(copyBack); |
---|
200 | b->CreateMemCpy(baseBuffer, unreadData, remainingBytes, blockSize); |
---|
201 | b->CreateBr(prepareBuffer); |
---|
202 | |
---|
203 | // Otherwise, allocate a buffer with twice the capacity and copy the unconsumed data back into it |
---|
204 | b->SetInsertPoint(expandAndCopyBack); |
---|
205 | Value * const expandedCapacity = b->CreateShl(capacity, 1); |
---|
206 | Value * const expandedBytes = b->CreateMul(expandedCapacity, codeUnitBytes); |
---|
207 | Value * const expandedBuffer = b->CreatePointerCast(b->CreateCacheAlignedMalloc(expandedBytes), unreadData->getType()); |
---|
208 | b->CreateMemCpy(expandedBuffer, unreadData, remainingBytes, blockSize); |
---|
209 | b->setScalarField("buffer", expandedBuffer); |
---|
210 | b->setCapacity("sourceBuffer", expandedCapacity); |
---|
211 | b->CreateFree(baseBuffer); |
---|
212 | b->CreateBr(prepareBuffer); |
---|
213 | |
---|
214 | b->SetInsertPoint(prepareBuffer); |
---|
215 | PHINode * const newBaseBuffer = b->CreatePHI(baseBuffer->getType(), 2); |
---|
216 | newBaseBuffer->addIncoming(baseBuffer, copyBack); |
---|
217 | newBaseBuffer->addIncoming(expandedBuffer, expandAndCopyBack); |
---|
218 | b->setBaseAddress("sourceBuffer", b->CreateGEP(newBaseBuffer, b->CreateNeg(offset))); |
---|
219 | b->CreateBr(readData); |
---|
220 | |
---|
221 | // Regardless of whether we're simply appending data or had to allocate a new buffer, read a new page |
---|
222 | // of data into the input source buffer. If we fail to read a full page ... |
---|
223 | b->SetInsertPoint(readData); |
---|
224 | Value * const sourceBuffer = b->getRawOutputPointer("sourceBuffer", produced); |
---|
225 | Value * const fd = b->getScalarField("fileDescriptor"); |
---|
226 | Constant * const bytesToRead = ConstantExpr::getMul(itemsToRead, codeUnitBytes); |
---|
227 | Value * const bytesRead = b->CreateReadCall(fd, sourceBuffer, bytesToRead); |
---|
228 | Value * const itemsRead = b->CreateUDiv(bytesRead, codeUnitBytes); |
---|
229 | Value * const itemsBuffered = b->CreateAdd(produced, itemsRead); |
---|
230 | b->CreateUnlikelyCondBr(b->CreateICmpULT(itemsBuffered, itemsPending), setTermination, readExit); |
---|
231 | |
---|
232 | // ... set the termination signal. |
---|
233 | b->SetInsertPoint(setTermination); |
---|
234 | Value * const bytesToZero = b->CreateMul(b->CreateSub(itemsPending, itemsBuffered), codeUnitBytes); |
---|
235 | b->CreateMemZero(b->getRawOutputPointer("sourceBuffer", itemsBuffered), bytesToZero); |
---|
236 | b->setTerminationSignal(); |
---|
237 | b->CreateBr(readExit); |
---|
238 | |
---|
239 | b->SetInsertPoint(readExit); |
---|
240 | PHINode * const itemsProduced = b->CreatePHI(itemsPending->getType(), 2); |
---|
241 | itemsProduced->addIncoming(itemsPending, readData); |
---|
242 | itemsProduced->addIncoming(itemsBuffered, setTermination); |
---|
243 | b->setProducedItemCount("sourceBuffer", itemsProduced); |
---|
244 | } |
---|
245 | |
---|
246 | void ReadSourceKernel::freeBuffer(const std::unique_ptr<KernelBuilder> & b) { |
---|
247 | b->CreateFree(b->getScalarField("buffer")); |
---|
248 | } |
---|
249 | |
---|
250 | /// Hybrid MMap/Read source kernel |
---|
251 | |
---|
252 | void FDSourceKernel::linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & b) { |
---|
253 | mFileSizeFunction = MMapSourceKernel::linkFileSizeMethod(b); |
---|
254 | } |
---|
255 | |
---|
256 | void FDSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) { |
---|
257 | BasicBlock * finalizeRead = b->CreateBasicBlock("finalizeRead"); |
---|
258 | BasicBlock * finalizeMMap = b->CreateBasicBlock("finalizeMMap"); |
---|
259 | BasicBlock * finalizeDone = b->CreateBasicBlock("finalizeDone"); |
---|
260 | b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), finalizeMMap, finalizeRead); |
---|
261 | b->SetInsertPoint(finalizeMMap); |
---|
262 | MMapSourceKernel::freeBuffer(b); |
---|
263 | b->CreateBr(finalizeDone); |
---|
264 | b->SetInsertPoint(finalizeRead); |
---|
265 | ReadSourceKernel::freeBuffer(b); |
---|
266 | b->CreateBr(finalizeDone); |
---|
267 | b->SetInsertPoint(finalizeDone); |
---|
268 | } |
---|
269 | |
---|
270 | void FDSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) { |
---|
271 | BasicBlock * initializeRead = b->CreateBasicBlock("initializeRead"); |
---|
272 | BasicBlock * tryMMap = b->CreateBasicBlock("tryMMap"); |
---|
273 | BasicBlock * initializeMMap = b->CreateBasicBlock("initializeMMap"); |
---|
274 | BasicBlock * initializeDone = b->CreateBasicBlock("initializeDone"); |
---|
275 | |
---|
276 | // The source will use MMapSource or readSoure kernel logic depending on the useMMap |
---|
277 | // parameter, possibly overridden. |
---|
278 | Value * useMMap = b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()); |
---|
279 | // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic. |
---|
280 | Value * fd = b->getScalarField("fileDescriptor"); |
---|
281 | useMMap = b->CreateAnd(useMMap, b->CreateICmpNE(fd, b->getInt32(STDIN_FILENO))); |
---|
282 | b->CreateCondBr(useMMap, tryMMap, initializeRead); |
---|
283 | |
---|
284 | b->SetInsertPoint(tryMMap); |
---|
285 | // If the fileSize is 0, we may have a virtual file such as /proc/cpuinfo |
---|
286 | Value * fileSize = b->CreateZExtOrTrunc(b->CreateCall(mFileSizeFunction, fd), b->getSizeTy()); |
---|
287 | useMMap = b->CreateICmpNE(fileSize, b->getSize(0)); |
---|
288 | b->CreateCondBr(useMMap, initializeMMap, initializeRead); |
---|
289 | b->SetInsertPoint(initializeMMap); |
---|
290 | MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, b); |
---|
291 | b->CreateBr(initializeDone); |
---|
292 | |
---|
293 | b->SetInsertPoint(initializeRead); |
---|
294 | // Ensure that readSource logic is used throughout. |
---|
295 | b->setScalarField("useMMap", b->getInt8(0)); |
---|
296 | ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth,b); |
---|
297 | b->CreateBr(initializeDone); |
---|
298 | b->SetInsertPoint(initializeDone); |
---|
299 | } |
---|
300 | |
---|
301 | void FDSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) { |
---|
302 | BasicBlock * DoSegmentRead = b->CreateBasicBlock("DoSegmentRead"); |
---|
303 | BasicBlock * DoSegmentMMap = b->CreateBasicBlock("DoSegmentMMap"); |
---|
304 | BasicBlock * DoSegmentDone = b->CreateBasicBlock("DoSegmentDone"); |
---|
305 | b->CreateCondBr(b->CreateTrunc(b->getScalarField("useMMap"), b->getInt1Ty()), DoSegmentMMap, DoSegmentRead); |
---|
306 | b->SetInsertPoint(DoSegmentMMap); |
---|
307 | MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, b); |
---|
308 | b->CreateBr(DoSegmentDone); |
---|
309 | b->SetInsertPoint(DoSegmentRead); |
---|
310 | ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, b); |
---|
311 | b->CreateBr(DoSegmentDone); |
---|
312 | b->SetInsertPoint(DoSegmentDone); |
---|
313 | } |
---|
314 | |
---|
315 | /// MEMORY SOURCE KERNEL |
---|
316 | |
---|
317 | void MemorySourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & b) { |
---|
318 | Value * const fileSource = b->getScalarField("fileSource"); |
---|
319 | b->setBaseAddress("sourceBuffer", fileSource); |
---|
320 | Value * const fileSize = b->getScalarField("fileSize"); |
---|
321 | b->setCapacity("sourceBuffer", fileSize); |
---|
322 | if (mStreamSetCount > 1) { |
---|
323 | b->setProducedItemCount("sourceBuffer", fileSize); |
---|
324 | b->setTerminationSignal(); |
---|
325 | } |
---|
326 | } |
---|
327 | |
---|
328 | void MemorySourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & b) { |
---|
329 | if (mStreamSetCount == 1) { |
---|
330 | Constant * const PAGE_SIZE = b->getSize(getStride()); |
---|
331 | Constant * const BLOCK_WIDTH = b->getSize(b->getBitBlockWidth()); |
---|
332 | |
---|
333 | BasicBlock * const entry = b->GetInsertBlock(); |
---|
334 | BasicBlock * const createTemporary = b->CreateBasicBlock("createTemporary"); |
---|
335 | BasicBlock * const exit = b->CreateBasicBlock("exit"); |
---|
336 | |
---|
337 | Value * const fileItems = b->getScalarField("fileSize"); |
---|
338 | Value * const producedItems = b->getProducedItemCount("sourceBuffer"); |
---|
339 | Value * const nextProducedItems = b->CreateAdd(producedItems, PAGE_SIZE); |
---|
340 | Value * const lastPage = b->CreateICmpULE(fileItems, nextProducedItems); |
---|
341 | b->CreateUnlikelyCondBr(lastPage, createTemporary, exit); |
---|
342 | |
---|
343 | b->SetInsertPoint(createTemporary); |
---|
344 | Value * const consumedItems = b->getConsumedItemCount("sourceBuffer"); |
---|
345 | Value * const consumedOffset = b->CreateAnd(consumedItems, ConstantExpr::getNeg(BLOCK_WIDTH)); |
---|
346 | Value * const readStart = b->getRawOutputPointer("sourceBuffer", consumedOffset); |
---|
347 | Value * const readEnd = b->getRawOutputPointer("sourceBuffer", fileItems); |
---|
348 | Value * const unconsumedBytes = b->CreateTrunc(b->CreatePtrDiff(readEnd, readStart), b->getSizeTy()); |
---|
349 | Value * const bufferSize = b->CreateRoundUp(b->CreateAdd(unconsumedBytes, BLOCK_WIDTH), PAGE_SIZE); |
---|
350 | Value * const buffer = b->CreateAlignedMalloc(bufferSize, b->getCacheAlignment()); |
---|
351 | b->CreateMemCpy(buffer, readStart, unconsumedBytes, 1); |
---|
352 | b->CreateMemZero(b->CreateGEP(buffer, unconsumedBytes), b->CreateSub(bufferSize, unconsumedBytes), 1); |
---|
353 | |
---|
354 | // get the difference between our base and from position then compute an offsetted temporary buffer address |
---|
355 | Value * const base = b->getBaseAddress("sourceBuffer"); |
---|
356 | Value * const diff = b->CreatePtrDiff(b->CreatePointerCast(base, readStart->getType()), readStart); |
---|
357 | Value * const offsettedBuffer = b->CreateGEP(buffer, diff); |
---|
358 | b->CreateConsumerWait(); |
---|
359 | |
---|
360 | // set the temporary buffer as the new source buffer |
---|
361 | PointerType * const codeUnitPtrTy = b->getIntNTy(mCodeUnitWidth)->getPointerTo(); |
---|
362 | b->setScalarField("buffer", b->CreatePointerCast(buffer, codeUnitPtrTy)); |
---|
363 | b->setBaseAddress("sourceBuffer", b->CreatePointerCast(offsettedBuffer, codeUnitPtrTy)); |
---|
364 | b->setTerminationSignal(); |
---|
365 | BasicBlock * const terminationExit = b->GetInsertBlock(); |
---|
366 | b->CreateBr(exit); |
---|
367 | |
---|
368 | b->SetInsertPoint(exit); |
---|
369 | PHINode * const newProducedItems = b->CreatePHI(b->getSizeTy(), 2); |
---|
370 | newProducedItems->addIncoming(nextProducedItems, entry); |
---|
371 | newProducedItems->addIncoming(fileItems, terminationExit); |
---|
372 | b->setProducedItemCount("sourceBuffer", newProducedItems); |
---|
373 | } |
---|
374 | } |
---|
375 | |
---|
376 | void MemorySourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & b) { |
---|
377 | if (mStreamSetCount == 1) { |
---|
378 | b->CreateFree(b->getScalarField("buffer")); |
---|
379 | } |
---|
380 | } |
---|
381 | |
---|
382 | MMapSourceKernel::MMapSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth) |
---|
383 | : SegmentOrientedKernel("mmap_source@" + std::to_string(codeUnitWidth) |
---|
384 | // input streams |
---|
385 | , {} |
---|
386 | // output streams |
---|
387 | , {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}} |
---|
388 | // input scalars |
---|
389 | , {Binding{b->getInt32Ty(), "fileDescriptor"}} |
---|
390 | // output scalars |
---|
391 | , {} |
---|
392 | // internal scalars |
---|
393 | , {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"} |
---|
394 | , Binding{b->getSizeTy(), "fileSize"}}) |
---|
395 | , mCodeUnitWidth(codeUnitWidth) |
---|
396 | , mFileSizeFunction(nullptr) { |
---|
397 | addAttribute(MustExplicitlyTerminate()); |
---|
398 | setStride(getpagesize()); |
---|
399 | } |
---|
400 | |
---|
401 | ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth) |
---|
402 | : SegmentOrientedKernel("read_source" + std::to_string(codegen::SegmentSize) + "@" + std::to_string(codeUnitWidth) |
---|
403 | // input streams |
---|
404 | , {} |
---|
405 | // output streams |
---|
406 | , {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}} |
---|
407 | // input scalars |
---|
408 | , {Binding{b->getInt32Ty(), "fileDescriptor"}} |
---|
409 | // output scalars |
---|
410 | , {} |
---|
411 | // internal scalars |
---|
412 | , {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}}) |
---|
413 | , mCodeUnitWidth(codeUnitWidth) { |
---|
414 | addAttribute(MustExplicitlyTerminate()); |
---|
415 | setStride(getpagesize()); |
---|
416 | } |
---|
417 | |
---|
418 | |
---|
419 | FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned codeUnitWidth) |
---|
420 | : SegmentOrientedKernel("FD_source@" + std::to_string(codeUnitWidth) |
---|
421 | // input streams |
---|
422 | , {} |
---|
423 | // output stream |
---|
424 | , {Binding{b->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}} |
---|
425 | // input scalar |
---|
426 | , {Binding{b->getInt8Ty(), "useMMap"}, Binding{b->getInt32Ty(), "fileDescriptor"}} |
---|
427 | , {} |
---|
428 | // internal scalars |
---|
429 | , {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}, |
---|
430 | Binding{b->getSizeTy(), "fileSize"}}) |
---|
431 | , mCodeUnitWidth(codeUnitWidth) |
---|
432 | , mFileSizeFunction(nullptr) { |
---|
433 | addAttribute(MustExplicitlyTerminate()); |
---|
434 | setStride(getpagesize()); |
---|
435 | } |
---|
436 | |
---|
437 | MemorySourceKernel::MemorySourceKernel(const std::unique_ptr<kernel::KernelBuilder> & b, const unsigned streamSetCount, const unsigned codeUnitWidth) |
---|
438 | : SegmentOrientedKernel("memory_source@" + std::to_string(streamSetCount) + ":" + std::to_string(codeUnitWidth), |
---|
439 | // input streams |
---|
440 | {}, |
---|
441 | // output stream |
---|
442 | {Binding{b->getStreamSetTy(streamSetCount, codeUnitWidth), "sourceBuffer"}}, |
---|
443 | // input scalar |
---|
444 | {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "fileSource"}, Binding{b->getSizeTy(), "fileSize"}}, |
---|
445 | {}, |
---|
446 | // internal scalar |
---|
447 | {Binding{b->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}}) |
---|
448 | , mStreamSetCount(streamSetCount) |
---|
449 | , mCodeUnitWidth(codeUnitWidth) { |
---|
450 | addAttribute(MustExplicitlyTerminate()); |
---|
451 | setStride(getpagesize()); |
---|
452 | } |
---|
453 | |
---|
454 | } |
---|