source: icGREP/icgrep-devel/icgrep/kernels/streamset.cpp @ 5544

Last change on this file since 5544 was 5544, checked in by cameron, 23 months ago

Deallocating dynamic buffers

File size: 39.0 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 */
5
6#include "streamset.h"
7#include <llvm/IR/Module.h>
8#include <llvm/Support/raw_ostream.h>
9#include <kernels/kernel.h>
10#include <kernels/kernel_builder.h>
11#include <toolchain/toolchain.h>
12#include <llvm/Support/Debug.h>
13#include <llvm/Support/Format.h>
14
15namespace llvm { class Constant; }
16namespace llvm { class Function; }
17
18using namespace parabix;
19using namespace llvm;
20using namespace IDISA;
21
22
23Type * StreamSetBuffer::getStreamSetBlockType() const { return mType;}
24
25ArrayType * resolveStreamSetType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type);
26
27StructType * resolveExpandableStreamSetType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type);
28
29void StreamSetBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
30    if (LLVM_LIKELY(mStreamSetBufferPtr == nullptr)) {
31        Type * const ty = getType();
32        mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(ty, iBuilder->getSize(mBufferBlocks));
33        iBuilder->CreateAlignedStore(Constant::getNullValue(ty), mStreamSetBufferPtr, iBuilder->getCacheAlignment());
34    } else {
35        report_fatal_error("StreamSetBuffer::allocateBuffer() was called twice on the same stream set");
36    }
37}
38
39Value * StreamSetBuffer::getStreamBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, const bool /* readOnly */) const {
40    if (codegen::EnableAsserts) {
41        Value * const count = getStreamSetCount(iBuilder, self);
42        Value * const index = iBuilder->CreateZExtOrTrunc(streamIndex, count->getType());
43        Value * const cond = iBuilder->CreateICmpULT(index, count);
44        iBuilder->CreateAssert(cond, "StreamSetBuffer: out-of-bounds stream access");
45    }
46    return iBuilder->CreateGEP(getStreamSetBlockPtr(iBuilder, self, blockIndex), {iBuilder->getInt32(0), streamIndex});
47}
48
49Value * StreamSetBuffer::getStreamPackPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool /* readOnly */) const {
50    if (codegen::EnableAsserts) {
51        Value * const count = getStreamSetCount(iBuilder, self);
52        Value * const index = iBuilder->CreateZExtOrTrunc(streamIndex, count->getType());
53        Value * const cond = iBuilder->CreateICmpULT(index, count);
54        iBuilder->CreateAssert(cond, "StreamSetBuffer: out-of-bounds stream access");
55    }
56    return iBuilder->CreateGEP(getStreamSetBlockPtr(iBuilder, self, blockIndex), {iBuilder->getInt32(0), streamIndex, packIndex});
57}
58
59void StreamSetBuffer::setBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * /* self */, Value * /* addr */) const {
60    report_fatal_error("setBaseAddress is not supported by this buffer type");
61}
62
63Value * StreamSetBuffer::getBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * /* self */) const {
64    report_fatal_error("getBufferedSize is not supported by this buffer type");
65}
66
67void StreamSetBuffer::setBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * /* self */, llvm::Value * /* size */) const {
68    report_fatal_error("setBufferedSize is not supported by this buffer type");
69}
70
71Value * StreamSetBuffer::getCapacity(IDISA::IDISA_Builder * const iBuilder, Value * /* self */) const {
72    report_fatal_error("getCapacity is not supported by this buffer type");
73}
74
75void StreamSetBuffer::setCapacity(IDISA::IDISA_Builder * const iBuilder, Value * /* self */, llvm::Value * /* c */) const {
76    report_fatal_error("setCapacity is not supported by this buffer type");
77}
78
79inline bool StreamSetBuffer::isCapacityGuaranteed(const Value * const index, const size_t capacity) const {
80    if (LLVM_UNLIKELY(isa<ConstantInt>(index))) {
81        if (LLVM_LIKELY(cast<ConstantInt>(index)->getLimitedValue() < capacity)) {
82            return true;
83        }
84    }
85    return false;
86}
87
88Value * StreamSetBuffer::getStreamSetCount(IDISA::IDISA_Builder * const iBuilder, Value *) const {
89    size_t count = 1;
90    if (isa<ArrayType>(mBaseType)) {
91        count = mBaseType->getArrayNumElements();
92    }
93    return iBuilder->getSize(count);
94}
95
96inline Value * StreamSetBuffer::modByBufferBlocks(IDISA::IDISA_Builder * const iBuilder, Value * const offset) const {
97    assert (offset->getType()->isIntegerTy());
98    if (isCapacityGuaranteed(offset, mBufferBlocks)) {
99        return offset;
100    } else if (mBufferBlocks == 1) {
101        return ConstantInt::getNullValue(iBuilder->getSizeTy());
102    } else if ((mBufferBlocks & (mBufferBlocks - 1)) == 0) { // is power of 2
103        return iBuilder->CreateAnd(offset, ConstantInt::get(offset->getType(), mBufferBlocks - 1));
104    } else {
105        return iBuilder->CreateURem(offset, ConstantInt::get(offset->getType(), mBufferBlocks));
106    }
107}
108
109/**
110 * @brief getRawItemPointer
111 *
112 * get a raw pointer the iN field at position absoluteItemPosition of the stream number streamIndex of the stream set.
113 * In the case of a stream whose fields are less than one byte (8 bits) in size, the pointer is to the containing byte.
114 * The type of the pointer is i8* for fields of 8 bits or less, otherwise iN* for N-bit fields.
115 */
116Value * StreamSetBuffer::getRawItemPointer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * absolutePosition) const {
117    Value * ptr = iBuilder->CreateGEP(getBaseAddress(iBuilder, self), {iBuilder->getInt32(0), streamIndex});
118    Value * relativePosition = absolutePosition;
119    const auto bw = mBaseType->getArrayElementType()->getScalarSizeInBits();
120    if (bw < 8) {
121        assert (bw  == 1 || bw == 2 || bw == 4);
122        relativePosition = iBuilder->CreateUDiv(relativePosition, ConstantInt::get(relativePosition->getType(), 8 / bw));
123        ptr = iBuilder->CreatePointerCast(ptr, iBuilder->getInt8PtrTy());
124    } else {
125        ptr = iBuilder->CreatePointerCast(ptr, iBuilder->getIntNTy(bw)->getPointerTo());
126    }
127    return iBuilder->CreateGEP(ptr, relativePosition);
128}
129
130Value * StreamSetBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
131    if (isa<ArrayType>(mType) && dyn_cast<ArrayType>(mType)->getNumElements() > 1) {
132        Constant * stride = iBuilder->getSize(iBuilder->getStride());
133        return iBuilder->CreateSub(stride, iBuilder->CreateURem(fromPosition, stride));
134    } else {
135        Constant * bufSize = iBuilder->getSize(mBufferBlocks * iBuilder->getStride());
136        return iBuilder->CreateSub(bufSize, iBuilder->CreateURem(fromPosition, bufSize, "linearItems"));
137    }
138}
139
140Value * StreamSetBuffer::getLinearlyAccessibleBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
141    Constant * bufBlocks = iBuilder->getSize(mBufferBlocks);
142    return iBuilder->CreateSub(bufBlocks, iBuilder->CreateURem(fromBlock, bufBlocks), "linearBlocks");
143}
144
145Value * StreamSetBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
146    return getLinearlyAccessibleItems(iBuilder, self, fromPosition);
147}
148
149Value * StreamSetBuffer::getLinearlyWritableBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
150    return getLinearlyAccessibleBlocks(iBuilder, self, fromBlock);
151}
152
153Value * StreamSetBuffer::getBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
154    iBuilder->CreateAssert(self, "StreamSetBuffer base address cannot be 0");
155    return self;
156}
157
158void StreamSetBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> & /* kb */) const {
159    /* do nothing: memory is stack allocated */
160}
161
162void StreamSetBuffer::createBlockCopy(IDISA::IDISA_Builder * const iBuilder, Value * targetBlockPtr, Value * sourceBlockPtr, Value * blocksToCopy) const {
163    Type * i8ptr = iBuilder->getInt8PtrTy();
164    unsigned alignment = iBuilder->getBitBlockWidth() / 8;
165    size_t numStreams = 1;
166    if (isa<ArrayType>(mBaseType)) {
167        numStreams = mBaseType->getArrayNumElements();
168    }
169    const auto fieldWidth = mBaseType->getArrayElementType()->getScalarSizeInBits();
170    Value * blockCopyBytes = iBuilder->CreateMul(blocksToCopy, iBuilder->getSize(iBuilder->getBitBlockWidth() * numStreams * fieldWidth/8));
171    iBuilder->CreateMemMove(iBuilder->CreateBitCast(targetBlockPtr, i8ptr), iBuilder->CreateBitCast(sourceBlockPtr, i8ptr), blockCopyBytes, alignment);
172}
173
174void StreamSetBuffer::createBlockAlignedCopy(IDISA::IDISA_Builder * const iBuilder, Value * targetBlockPtr, Value * sourceBlockPtr, Value * itemsToCopy) const {
175    Type * const int8PtrTy = iBuilder->getInt8PtrTy();
176    const unsigned alignment = iBuilder->getBitBlockWidth() / 8;
177    Constant * const blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
178    size_t numStreams = 1;
179    if (isa<ArrayType>(mBaseType)) {
180        numStreams = mBaseType->getArrayNumElements();
181    }
182    const auto fieldWidth = mBaseType->getArrayElementType()->getScalarSizeInBits();
183    if (numStreams == 1) {
184        Value * copyBits = iBuilder->CreateMul(itemsToCopy, iBuilder->getSize(fieldWidth));
185        Value * copyBytes = iBuilder->CreateLShr(iBuilder->CreateAdd(copyBits, iBuilder->getSize(7)), iBuilder->getSize(3));
186        iBuilder->CreateMemMove(iBuilder->CreateBitCast(targetBlockPtr, int8PtrTy), iBuilder->CreateBitCast(sourceBlockPtr, int8PtrTy), copyBytes, alignment);
187    } else {
188        Value * blocksToCopy = iBuilder->CreateUDiv(itemsToCopy, blockSize);
189        Value * partialItems = iBuilder->CreateURem(itemsToCopy, blockSize);
190        Value * partialBlockTargetPtr = iBuilder->CreateGEP(targetBlockPtr, blocksToCopy);
191        Value * partialBlockSourcePtr = iBuilder->CreateGEP(sourceBlockPtr, blocksToCopy);
192        Value * blockCopyBytes = iBuilder->CreateMul(blocksToCopy, iBuilder->getSize(iBuilder->getBitBlockWidth() * numStreams * fieldWidth/8));
193        iBuilder->CreateMemMove(iBuilder->CreateBitCast(targetBlockPtr, int8PtrTy), iBuilder->CreateBitCast(sourceBlockPtr, int8PtrTy), blockCopyBytes, alignment);
194        Value * partialCopyBitsPerStream = iBuilder->CreateMul(partialItems, iBuilder->getSize(fieldWidth));
195        Value * partialCopyBytesPerStream = iBuilder->CreateLShr(iBuilder->CreateAdd(partialCopyBitsPerStream, iBuilder->getSize(7)), iBuilder->getSize(3));
196        for (unsigned strm = 0; strm < numStreams; strm++) {
197            Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
198            Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
199            strmTargetPtr = iBuilder->CreateBitCast(strmTargetPtr, int8PtrTy);
200            strmSourcePtr = iBuilder->CreateBitCast(strmSourcePtr, int8PtrTy);
201            iBuilder->CreateMemMove(strmTargetPtr, strmSourcePtr, partialCopyBytesPerStream, alignment);
202        }
203    }
204}
205
206// Source File Buffer
207
208Type * SourceBuffer::getStreamSetBlockType() const {
209    return cast<PointerType>(mType->getStructElementType(int(SourceBuffer::Field::BaseAddress)))->getElementType();
210}
211
212
213Value * SourceBuffer::getBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
214    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::BufferedSize))});
215    return iBuilder->CreateLoad(ptr);
216}
217
218void SourceBuffer::setBufferedSize(IDISA::IDISA_Builder * const iBuilder, Value * self, llvm::Value * size) const {
219    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::BufferedSize))});
220    iBuilder->CreateStore(size, ptr);
221}
222
223Value * SourceBuffer::getCapacity(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
224    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::Capacity))});
225    return iBuilder->CreateLoad(ptr);
226}
227
228void SourceBuffer::setCapacity(IDISA::IDISA_Builder * const iBuilder, Value * self, llvm::Value * c) const {
229    Value * ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::Capacity))});
230    iBuilder->CreateStore(c, ptr);
231}
232
233void SourceBuffer::setBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * addr) const {
234    Value * const ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::BaseAddress))});
235
236    iBuilder->CreateStore(iBuilder->CreatePointerCast(addr, ptr->getType()->getPointerElementType()), ptr);
237}
238
239Value * SourceBuffer::getBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * const self) const {
240    iBuilder->CreateAssert(self, "SourceBuffer: instance cannot be null");
241    Value * const ptr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(int(SourceBuffer::Field::BaseAddress))});
242    Value * const addr = iBuilder->CreateLoad(ptr);
243    iBuilder->CreateAssert(addr, "SourceBuffer: base address cannot be 0");
244    return addr;
245}
246
247Value * SourceBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
248    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), blockIndex);
249}
250
251Value * SourceBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
252    return iBuilder->CreateSub(getCapacity(iBuilder, self), fromPosition);
253}
254
255Value * SourceBuffer::getLinearlyAccessibleBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
256    return iBuilder->CreateSub(iBuilder->CreateUDiv(getCapacity(iBuilder, self), iBuilder->getSize(iBuilder->getBitBlockWidth())), fromBlock);
257}
258
259
260// External File Buffer
261void ExternalBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> &) {
262    report_fatal_error("External buffers cannot be allocated.");
263}
264
265Value * ExternalBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
266    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), blockIndex);
267}
268
269Value * ExternalBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value *) const {
270    report_fatal_error("External buffers: getLinearlyAccessibleItems is not supported.");
271}
272
273// Circular Buffer
274Value * CircularBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * const self, Value * const blockIndex) const {
275    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), modByBufferBlocks(iBuilder, blockIndex));
276}
277
278Value * CircularBuffer::getRawItemPointer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * absolutePosition) const {
279    Value * ptr = iBuilder->CreateGEP(getBaseAddress(iBuilder, self), {iBuilder->getInt32(0), streamIndex});
280    Value * relativePosition = iBuilder->CreateURem(absolutePosition, ConstantInt::get(absolutePosition->getType(), mBufferBlocks * iBuilder->getBitBlockWidth()));
281    const auto bw = mBaseType->getArrayElementType()->getScalarSizeInBits();
282    if (bw < 8) {
283        assert (bw  == 1 || bw == 2 || bw == 4);
284        relativePosition = iBuilder->CreateUDiv(relativePosition, ConstantInt::get(relativePosition->getType(), 8 / bw));
285        ptr = iBuilder->CreatePointerCast(ptr, iBuilder->getInt8PtrTy());
286    } else {
287        ptr = iBuilder->CreatePointerCast(ptr, iBuilder->getIntNTy(bw)->getPointerTo());
288    }
289    return iBuilder->CreateGEP(ptr, relativePosition);
290}
291
292// CircularCopybackBuffer Buffer
293void CircularCopybackBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
294    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType(), iBuilder->getSize(mBufferBlocks + mOverflowBlocks));
295}
296
297void CircularCopybackBuffer::createCopyBack(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * overFlowItems) const {
298    Value * overFlowAreaPtr = iBuilder->CreateGEP(self, iBuilder->getSize(mBufferBlocks));
299    createBlockAlignedCopy(iBuilder, self, overFlowAreaPtr, overFlowItems);
300}
301
302Value * CircularCopybackBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
303    return iBuilder->CreateAdd(getLinearlyAccessibleItems(iBuilder, self, fromPosition), iBuilder->getSize(mOverflowBlocks * iBuilder->getBitBlockWidth()));
304}
305
306Value * CircularCopybackBuffer::getLinearlyWritableBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
307    return iBuilder->CreateAdd(getLinearlyAccessibleBlocks(iBuilder, self, fromBlock), iBuilder->getSize(mOverflowBlocks));
308}
309
310// SwizzledCopybackBuffer Buffer
311
312void SwizzledCopybackBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
313    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType(), iBuilder->getSize(mBufferBlocks + mOverflowBlocks));
314}
315
316void SwizzledCopybackBuffer::createBlockAlignedCopy(IDISA::IDISA_Builder * const iBuilder, Value * targetBlockPtr, Value * sourceBlockPtr, Value * itemsToCopy) const {
317    Type * int8PtrTy = iBuilder->getInt8PtrTy();
318    DataLayout DL(iBuilder->getModule());
319    IntegerType * const intAddrTy = iBuilder->getIntPtrTy(DL);
320
321    Constant * blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
322    Function * f = iBuilder->GetInsertBlock()->getParent();
323    BasicBlock * wholeBlockCopy = BasicBlock::Create(iBuilder->getContext(), "wholeBlockCopy", f, 0);
324    BasicBlock * partialBlockCopy = BasicBlock::Create(iBuilder->getContext(), "partialBlockCopy", f, 0);
325    BasicBlock * copyDone = BasicBlock::Create(iBuilder->getContext(), "copyDone", f, 0);
326    const unsigned numStreams = getType()->getArrayNumElements();
327    const unsigned swizzleFactor = iBuilder->getBitBlockWidth()/mFieldWidth;
328    const auto elemTy = getType()->getArrayElementType();
329    const unsigned fieldWidth = isa<ArrayType>(elemTy) ? elemTy->getArrayNumElements() : 1;
330    Value * blocksToCopy = iBuilder->CreateUDiv(itemsToCopy, blockSize);
331    Value * partialItems = iBuilder->CreateURem(itemsToCopy, blockSize);
332    Value * partialBlockTargetPtr = iBuilder->CreateGEP(targetBlockPtr, blocksToCopy);
333    Value * partialBlockSourcePtr = iBuilder->CreateGEP(sourceBlockPtr, blocksToCopy);
334    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(blocksToCopy, iBuilder->getSize(0)), wholeBlockCopy, partialBlockCopy);
335
336    iBuilder->SetInsertPoint(wholeBlockCopy);
337    const unsigned alignment = iBuilder->getBitBlockWidth() / 8;
338    Value * copyLength = iBuilder->CreateSub(iBuilder->CreatePtrToInt(partialBlockTargetPtr, intAddrTy), iBuilder->CreatePtrToInt(targetBlockPtr, intAddrTy));
339    iBuilder->CreateMemMove(iBuilder->CreatePointerCast(targetBlockPtr, int8PtrTy), iBuilder->CreatePointerCast(sourceBlockPtr, int8PtrTy), copyLength, alignment);
340    iBuilder->CreateCondBr(iBuilder->CreateICmpUGT(partialItems, iBuilder->getSize(0)), partialBlockCopy, copyDone);
341    iBuilder->SetInsertPoint(partialBlockCopy);
342    Value * copyBits = iBuilder->CreateMul(itemsToCopy, iBuilder->getSize(fieldWidth * swizzleFactor));
343    Value * copyBytes = iBuilder->CreateLShr(iBuilder->CreateAdd(copyBits, iBuilder->getSize(7)), iBuilder->getSize(3));
344    for (unsigned strm = 0; strm < numStreams; strm += swizzleFactor) {
345        Value * strmTargetPtr = iBuilder->CreateGEP(partialBlockTargetPtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
346        Value * strmSourcePtr = iBuilder->CreateGEP(partialBlockSourcePtr, {iBuilder->getInt32(0), iBuilder->getInt32(strm)});
347        iBuilder->CreateMemMove(iBuilder->CreatePointerCast(strmTargetPtr, int8PtrTy), iBuilder->CreatePointerCast(strmSourcePtr, int8PtrTy), copyBytes, alignment);
348    }
349    iBuilder->CreateBr(copyDone);
350
351    iBuilder->SetInsertPoint(copyDone);
352}
353
354void SwizzledCopybackBuffer::createCopyBack(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * overFlowItems) const {
355    Value * overFlowAreaPtr = iBuilder->CreateGEP(self, iBuilder->getSize(mBufferBlocks));
356    createBlockAlignedCopy(iBuilder, self, overFlowAreaPtr, overFlowItems);
357}
358
359Value * SwizzledCopybackBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * blockIndex) const {
360    return iBuilder->CreateGEP(getBaseAddress(iBuilder, self), modByBufferBlocks(iBuilder, blockIndex));
361}
362
363Value * SwizzledCopybackBuffer::getLinearlyWritableItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromPosition) const {
364    return iBuilder->CreateAdd(getLinearlyAccessibleItems(iBuilder, self, fromPosition), iBuilder->getSize(mOverflowBlocks * iBuilder->getBitBlockWidth()));
365}
366
367Value * SwizzledCopybackBuffer::getLinearlyWritableBlocks(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * fromBlock) const {
368    return iBuilder->CreateAdd(getLinearlyAccessibleBlocks(iBuilder, self, fromBlock), iBuilder->getSize(mOverflowBlocks));
369}
370
371// Expandable Buffer
372
373void ExpandableBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
374    mStreamSetBufferPtr = iBuilder->CreateCacheAlignedAlloca(getType());
375    Value * const capacityPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
376    iBuilder->CreateStore(iBuilder->getSize(mInitialCapacity), capacityPtr);
377    Type * const bufferType = getType()->getStructElementType(1)->getPointerElementType();
378    Constant * const bufferWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(bufferType), iBuilder->getSizeTy(), false);
379    Constant * const size = ConstantExpr::getMul(iBuilder->getSize(mBufferBlocks * mInitialCapacity), bufferWidth);
380    const auto alignment = std::max(iBuilder->getCacheAlignment(), iBuilder->getBitBlockWidth() / 8);
381    Value * const ptr = iBuilder->CreateAlignedMalloc(size, alignment);
382    iBuilder->CreateMemZero(ptr, size, bufferType->getPrimitiveSizeInBits() / 8);
383    Value * const streamSetPtr = iBuilder->CreateGEP(mStreamSetBufferPtr, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
384    iBuilder->CreateStore(iBuilder->CreatePointerCast(ptr, bufferType->getPointerTo()), streamSetPtr);
385}
386
387std::pair<Value *, Value *> ExpandableBuffer::getInternalStreamBuffer(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, const bool readOnly) const {
388
389    // ENTRY
390    Value * const capacityPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
391    Value * const capacity = iBuilder->CreateLoad(capacityPtr);
392    Value * const streamSetPtr = iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
393    Value * const streamSet = iBuilder->CreateLoad(streamSetPtr);
394    blockIndex = modByBufferBlocks(iBuilder, blockIndex);
395
396    assert (streamIndex->getType() == capacity->getType());
397    Value * const cond = iBuilder->CreateICmpULT(streamIndex, capacity);
398
399    // Are we guaranteed that we can access this stream?
400    if (readOnly || isCapacityGuaranteed(streamIndex, mInitialCapacity)) {
401        iBuilder->CreateAssert(cond, "ExpandableBuffer: out-of-bounds stream access");
402        Value * offset = iBuilder->CreateAdd(iBuilder->CreateMul(blockIndex, capacity), streamIndex);
403        return {streamSet, offset};
404    }
405
406    BasicBlock * const entry = iBuilder->GetInsertBlock();
407    BasicBlock * const expand = BasicBlock::Create(iBuilder->getContext(), "expand", entry->getParent());
408    BasicBlock * const resume = BasicBlock::Create(iBuilder->getContext(), "resume", entry->getParent());
409
410    iBuilder->CreateLikelyCondBr(cond, resume, expand);
411
412    // EXPAND
413    iBuilder->SetInsertPoint(expand);
414
415    Type * elementType = getType()->getStructElementType(1)->getPointerElementType();
416    Constant * const vectorWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(elementType), capacity->getType(), false);
417
418    Value * newCapacity = iBuilder->CreateAdd(streamIndex, iBuilder->getSize(1));
419    newCapacity = iBuilder->CreateCeilLog2(newCapacity);
420    newCapacity = iBuilder->CreateShl(iBuilder->getSize(1), newCapacity, "newCapacity");
421
422    std::string tmp;
423    raw_string_ostream out(tmp);
424    out << "__expand";
425    elementType->print(out);
426    std::string name = out.str();
427
428    Module * const m = iBuilder->getModule();
429    Function * expandFunction = m->getFunction(name);
430
431    if (expandFunction == nullptr) {
432
433        const auto ip = iBuilder->saveIP();
434
435        FunctionType * fty = FunctionType::get(elementType->getPointerTo(), {elementType->getPointerTo(), iBuilder->getSizeTy(), iBuilder->getSizeTy()}, false);
436        expandFunction = Function::Create(fty, GlobalValue::PrivateLinkage, name, m);
437
438        auto args = expandFunction->arg_begin();
439        Value * streamSet = &*args++;
440        Value * capacity = &*args++;
441        Value * newCapacity = &*args;
442
443        BasicBlock * entry = BasicBlock::Create(iBuilder->getContext(), "entry", expandFunction);
444        iBuilder->SetInsertPoint(entry);
445
446        Value * size = iBuilder->CreateMul(newCapacity, iBuilder->getSize(mBufferBlocks));
447        const auto memAlign = std::max(iBuilder->getCacheAlignment(), iBuilder->getBitBlockWidth() / 8);
448
449        Value * newStreamSet = iBuilder->CreatePointerCast(iBuilder->CreateAlignedMalloc(iBuilder->CreateMul(size, vectorWidth), memAlign), elementType->getPointerTo());
450        Value * const diffCapacity = iBuilder->CreateMul(iBuilder->CreateSub(newCapacity, capacity), vectorWidth);
451
452        const auto alignment = elementType->getPrimitiveSizeInBits() / 8;
453        for (unsigned i = 0; i < mBufferBlocks; ++i) {
454            ConstantInt * const offset = iBuilder->getSize(i);
455            Value * srcOffset = iBuilder->CreateMul(capacity, offset);
456            Value * srcPtr = iBuilder->CreateGEP(streamSet, srcOffset);
457            Value * destOffset = iBuilder->CreateMul(newCapacity, offset);
458            Value * destPtr = iBuilder->CreateGEP(newStreamSet, destOffset);
459            iBuilder->CreateMemCpy(destPtr, srcPtr, iBuilder->CreateMul(capacity, vectorWidth), alignment);
460            Value * destZeroOffset = iBuilder->CreateAdd(destOffset, capacity);
461            Value * destZeroPtr = iBuilder->CreateGEP(newStreamSet, destZeroOffset);
462            iBuilder->CreateMemZero(destZeroPtr, diffCapacity, alignment);
463        }
464
465        iBuilder->CreateFree(streamSet);
466
467        iBuilder->CreateRet(newStreamSet);
468
469        iBuilder->restoreIP(ip);
470    }
471
472    Value * newStreamSet = iBuilder->CreateCall(expandFunction, {streamSet, capacity, newCapacity});
473    iBuilder->CreateStore(newStreamSet, streamSetPtr);
474    iBuilder->CreateStore(newCapacity, capacityPtr);
475
476    iBuilder->CreateBr(resume);
477
478    // RESUME
479    iBuilder->SetInsertPoint(resume);
480
481    PHINode * phiStreamSet = iBuilder->CreatePHI(streamSet->getType(), 2);
482    phiStreamSet->addIncoming(streamSet, entry);
483    phiStreamSet->addIncoming(newStreamSet, expand);
484
485    PHINode * phiCapacity = iBuilder->CreatePHI(capacity->getType(), 2);
486    phiCapacity->addIncoming(capacity, entry);
487    phiCapacity->addIncoming(newCapacity, expand);
488
489    Value * offset = iBuilder->CreateAdd(iBuilder->CreateMul(blockIndex, phiCapacity), streamIndex);
490
491    return {phiStreamSet, offset};
492}
493
494Value * ExpandableBuffer::getStreamBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, const bool readOnly) const {
495    Value * ptr, * offset;
496    std::tie(ptr, offset) = getInternalStreamBuffer(iBuilder, self, streamIndex, blockIndex, readOnly);
497    return iBuilder->CreateGEP(ptr, offset);
498}
499
500Value * ExpandableBuffer::getStreamPackPtr(IDISA::IDISA_Builder * const iBuilder, Value * self, Value * streamIndex, Value * blockIndex, Value * packIndex, const bool readOnly) const {
501    Value * ptr, * offset;
502    std::tie(ptr, offset) = getInternalStreamBuffer(iBuilder, self, streamIndex, blockIndex, readOnly);
503    return iBuilder->CreateGEP(ptr, {offset, packIndex});
504}
505
506Value * ExpandableBuffer::getStreamSetCount(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
507    return iBuilder->CreateLoad(iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(0)}));
508}
509
510Value * ExpandableBuffer::getBaseAddress(IDISA::IDISA_Builder * const iBuilder, Value * self) const {
511    iBuilder->CreateAssert(self, "ExpandableBuffer: instance cannot be null");
512    Value * const baseAddr = iBuilder->CreateLoad(iBuilder->CreateGEP(self, {iBuilder->getInt32(0), iBuilder->getInt32(1)}));
513    iBuilder->CreateAssert(self, "ExpandableBuffer: base address cannot be 0");
514    return baseAddr;
515}
516
517void ExpandableBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) const {
518    b->CreateFree(getBaseAddress(b.get(), mStreamSetBufferPtr));
519}
520
521Value * ExpandableBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const iBuilder, Value *, Value *) const {
522    report_fatal_error("Expandable buffers: getStreamSetBlockPtr is not supported.");
523}
524
525Value * ExpandableBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const iBuilder, Value * self, Value *) const {
526    report_fatal_error("Expandable buffers: getLinearlyAccessibleItems is not supported.");
527}
528
529SourceBuffer::SourceBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, unsigned MemoryAddressSpace, unsigned StructAddressSpace)
530: StreamSetBuffer(BufferKind::SourceBuffer, type, StructType::get(resolveStreamSetType(b, type)->getPointerTo(MemoryAddressSpace), b->getSizeTy(), b->getSizeTy(), nullptr), 0, StructAddressSpace) {
531    mUniqueID = "B";
532    if (MemoryAddressSpace != 0 || StructAddressSpace != 0) {
533        mUniqueID += "@" + std::to_string(MemoryAddressSpace) + ":" + std::to_string(StructAddressSpace);
534    }
535}
536
537ExternalBuffer::ExternalBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, llvm::Value * addr, unsigned AddressSpace)
538: StreamSetBuffer(BufferKind::ExternalBuffer, type, resolveStreamSetType(b, type), 0, AddressSpace) {
539    mUniqueID = "E";
540    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
541    mStreamSetBufferPtr = b->CreatePointerBitCastOrAddrSpaceCast(addr, getPointerType());
542}
543
544CircularBuffer::CircularBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
545: StreamSetBuffer(BufferKind::CircularBuffer, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
546    mUniqueID = "C" + std::to_string(bufferBlocks);
547    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
548}
549
550CircularBuffer::CircularBuffer(const BufferKind k, const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
551: StreamSetBuffer(k, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace) {
552
553}
554
555CircularCopybackBuffer::CircularCopybackBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned AddressSpace)
556: CircularBuffer(BufferKind::CircularCopybackBuffer, b, type, bufferBlocks, AddressSpace)
557, mOverflowBlocks(overflowBlocks) {
558    mUniqueID = "CC" + std::to_string(bufferBlocks);
559    if (mOverflowBlocks != 1) mUniqueID += "_" + std::to_string(mOverflowBlocks);
560    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
561}
562
563ExpandableBuffer::ExpandableBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, unsigned AddressSpace)
564: StreamSetBuffer(BufferKind::ExpandableBuffer, type, resolveExpandableStreamSetType(b, type), bufferBlocks, AddressSpace)
565, mInitialCapacity(type->getArrayNumElements()) {
566    mUniqueID = "XP" + std::to_string(bufferBlocks);
567    if (AddressSpace > 0) mUniqueID += "@" + std::to_string(AddressSpace);
568}
569
570SwizzledCopybackBuffer::SwizzledCopybackBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t bufferBlocks, size_t overflowBlocks, unsigned fieldwidth, unsigned AddressSpace)
571: StreamSetBuffer(BufferKind::SwizzledCopybackBuffer, type, resolveStreamSetType(b, type), bufferBlocks, AddressSpace), mOverflowBlocks(overflowBlocks), mFieldWidth(fieldwidth) {
572    mUniqueID = "SW" + std::to_string(fieldwidth) + ":" + std::to_string(bufferBlocks);
573    if (mOverflowBlocks != 1) {
574        mUniqueID += "_" + std::to_string(mOverflowBlocks);
575    }
576    if (AddressSpace > 0) {
577        mUniqueID += "@" + std::to_string(AddressSpace);
578    }
579}
580
581Value * DynamicBuffer::getBaseAddress(IDISA::IDISA_Builder * const b, Value * const handle) const {
582    b->CreateAssert(handle, "DynamicBuffer: instance cannot be null");
583    Value * const p = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::BaseAddress))});
584    Value * const addr = b->CreateLoad(p);
585    b->CreateAssert(addr, "DynamicBuffer: base address cannot be 0");
586    return addr;
587}
588
589Value * DynamicBuffer::getStreamSetBlockPtr(IDISA::IDISA_Builder * const b, Value * handle, Value * blockIndex) const {
590    Value * const wkgBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::WorkingBlocks))}));
591    return b->CreateGEP(getBaseAddress(b, handle), b->CreateURem(blockIndex, wkgBlocks));
592}
593
594Value * DynamicBuffer::getRawItemPointer(IDISA::IDISA_Builder * const b, Value * handle, Value * streamIndex, Value * absolutePosition) const {
595    Value * absBlock = b->CreateUDiv(absolutePosition, b->getSize(b->getBitBlockWidth()));
596    Value * blockPos = b->CreateURem(absolutePosition, b->getSize(b->getBitBlockWidth()));
597    Value * blockPtr = b->CreateGEP(getStreamSetBlockPtr(b, handle, absBlock), {b->getInt32(0), streamIndex});
598    const auto bw = mBaseType->getArrayElementType()->getScalarSizeInBits();
599    if (bw < 8) {
600        assert (bw  == 1 || bw == 2 || bw == 4);
601        blockPos = b->CreateUDiv(blockPos, ConstantInt::get(blockPos->getType(), 8 / bw));
602        blockPtr = b->CreatePointerCast(blockPtr, b->getInt8PtrTy());
603    } else {
604        blockPtr = b->CreatePointerCast(blockPtr, b->getIntNTy(bw)->getPointerTo());
605    }
606    return b->CreateGEP(blockPtr, blockPos);
607}
608
609Value * DynamicBuffer::getLinearlyAccessibleItems(IDISA::IDISA_Builder * const b, Value * handle, Value * fromPosition) const {
610    Constant * blockSize = b->getSize(b->getBitBlockWidth());
611    if (isa<ArrayType>(mType) && dyn_cast<ArrayType>(mType)->getNumElements() > 1) {
612        return b->CreateSub(blockSize, b->CreateURem(fromPosition, blockSize));
613    } else {
614        Value * const bufBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::WorkingBlocks))}));
615        Value * bufSize = b->CreateMul(bufBlocks, blockSize);
616        return b->CreateSub(bufSize, b->CreateURem(fromPosition, bufSize, "linearItems"));
617    }
618}
619
620Value * DynamicBuffer::getLinearlyAccessibleBlocks(IDISA::IDISA_Builder * const b, Value * handle, Value * fromBlock) const {
621    Value * const bufBlocks = b->CreateLoad(b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::WorkingBlocks))}));
622    return b->CreateSub(bufBlocks, b->CreateURem(fromBlock, bufBlocks), "linearBlocks");
623}
624
625void DynamicBuffer::allocateBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) {
626    Value * handle = b->CreateCacheAlignedAlloca(mBufferStructType);
627    size_t numStreams = 1;
628    if (isa<ArrayType>(mBaseType)) {
629        numStreams = mBaseType->getArrayNumElements();
630    }
631    const auto fieldWidth = mBaseType->getArrayElementType()->getScalarSizeInBits();
632    Value * bufSize = b->getSize((mBufferBlocks + mOverflowBlocks) * b->getBitBlockWidth() * numStreams * fieldWidth/8);
633    bufSize = b->CreateRoundUp(bufSize, b->getSize(b->getCacheAlignment()));
634    Value * bufBasePtrField = b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::BaseAddress))});
635    Value * bufPtr = b->CreatePointerCast(b->CreateCacheAlignedMalloc(bufSize), bufBasePtrField->getType()->getPointerElementType());
636    b->CreateStore(bufPtr, bufBasePtrField);
637    b->CreateStore(bufSize, b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::AllocatedCapacity))}));
638    b->CreateStore(b->getSize(mBufferBlocks), b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::WorkingBlocks))}));
639    b->CreateStore(b->getSize(-1), b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::Length))}));
640    b->CreateStore(b->getSize(0), b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::ProducedPosition))}));
641    b->CreateStore(b->getSize(0), b->CreateGEP(handle, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::ConsumedPosition))}));
642    mStreamSetBufferPtr = handle;
643}
644
645void DynamicBuffer::releaseBuffer(const std::unique_ptr<kernel::KernelBuilder> & b) const {
646    /* Free the dynamically allocated buffer, but not the stack-allocated buffer struct. */
647    b->CreateFree(b->CreateLoad(b->CreateGEP(mStreamSetBufferPtr, {b->getInt32(0), b->getInt32(int(DynamicBuffer::Field::BaseAddress))})));
648}
649
650
651DynamicBuffer::DynamicBuffer(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type, size_t initialCapacity, size_t overflow, unsigned swizzle, unsigned addrSpace)
652: StreamSetBuffer(BufferKind::DynamicBuffer, type, resolveStreamSetType(b, type), initialCapacity, addrSpace)
653, mBufferStructType(StructType::get(resolveStreamSetType(b, type)->getPointerTo(addrSpace), 
654                                    b->getSizeTy(), b->getSizeTy(), b->getSizeTy(), b->getSizeTy(), b->getSizeTy(), nullptr))
655, mSwizzleFactor(swizzle)
656, mOverflowBlocks(overflow)
657{
658    mUniqueID = "DB";
659    if (swizzle != 1) {
660        mUniqueID += "s" + std::to_string(swizzle);
661    }
662        if (overflow != 0) {
663        mUniqueID += "o" + std::to_string(overflow);
664    }
665    if (addrSpace != 0) {
666        mUniqueID += "@" + std::to_string(addrSpace);
667    }
668}
669
670
671inline StreamSetBuffer::StreamSetBuffer(BufferKind k, Type * baseType, Type * resolvedType, unsigned BufferBlocks, unsigned AddressSpace)
672: mBufferKind(k)
673, mType(resolvedType)
674, mBufferBlocks(BufferBlocks)
675, mAddressSpace(AddressSpace)
676, mStreamSetBufferPtr(nullptr)
677, mBaseType(baseType)
678, mProducer(nullptr) {
679
680}
681
682StreamSetBuffer::~StreamSetBuffer() { }
683
684// Helper routines
685ArrayType * resolveStreamSetType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type) {
686    unsigned numElements = 1;
687    if (LLVM_LIKELY(type->isArrayTy())) {
688        numElements = type->getArrayNumElements();
689        type = type->getArrayElementType();
690    }
691    if (LLVM_LIKELY(type->isVectorTy() && type->getVectorNumElements() == 0)) {
692        type = type->getVectorElementType();
693        if (LLVM_LIKELY(type->isIntegerTy())) {
694            const auto fieldWidth = cast<IntegerType>(type)->getBitWidth();
695            type = b->getBitBlockType();
696            if (fieldWidth != 1) {
697                type = ArrayType::get(type, fieldWidth);
698            }
699            return ArrayType::get(type, numElements);
700        }
701    }
702    std::string tmp;
703    raw_string_ostream out(tmp);
704    type->print(out);
705    out << " is an unvalid stream set buffer type.";
706    report_fatal_error(out.str());
707}
708
709StructType * resolveExpandableStreamSetType(const std::unique_ptr<kernel::KernelBuilder> & b, Type * type) {
710    if (LLVM_LIKELY(type->isArrayTy())) {
711        type = type->getArrayElementType();
712    }
713    if (LLVM_LIKELY(type->isVectorTy() && type->getVectorNumElements() == 0)) {
714        type = type->getVectorElementType();
715        if (LLVM_LIKELY(type->isIntegerTy())) {
716            const auto fieldWidth = cast<IntegerType>(type)->getBitWidth();
717            type = b->getBitBlockType();
718            if (fieldWidth != 1) {
719                type = ArrayType::get(type, fieldWidth);
720            }
721            return StructType::get(b->getSizeTy(), type->getPointerTo(), nullptr);
722        }
723    }
724    std::string tmp;
725    raw_string_ostream out(tmp);
726    type->print(out);
727    out << " is an unvalid stream set buffer type.";
728    report_fatal_error(out.str());
729}
Note: See TracBrowser for help on using the repository browser.