Ignore:
Timestamp:
Oct 13, 2016, 11:05:03 AM (3 years ago)
Author:
cameron
Message:

mmap and posix_memalign buffering options for u8u16

Location:
icGREP/icgrep-devel/icgrep
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/kernels/streamset.cpp

    r5190 r5191  
    99#include <IDISA/idisa_builder.h>
    1010#include <llvm/IR/Type.h>
    11 #include <iostream>
    12    
     11
    1312using namespace parabix;
    1413
     
    8887}
    8988
     89llvm::Value * StreamSetBuffer::getStreamSetBlockPointer(llvm::Value * bufferStructPtr, llvm::Value * blockNo) {
     90    Value * handle = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iBuffer_ptr)});
     91    return iBuilder->CreateGEP(iBuilder->CreateLoad(handle), {blockNo});
     92}
     93
     94
     95
     96
    9097// Single Block Buffer
    9198// For a single block buffer, the block pointer is always the buffer base pointer.
     
    96103
    97104
    98 // External Unbounded Buffer
     105// External File Buffer
    99106
    100107void ExternalFileBuffer::setStreamSetBuffer(llvm::Value * ptr, Value * fileSize) {
     
    137144}
    138145
    139 
    140146// Circular Stack Allocated Buffer
    141147
     
    149155}
    150156
    151 llvm::Value * LinearBuffer::getStreamSetBlockPointer(llvm::Value * bufferStructPtr, llvm::Value * blockNo) {
     157llvm::Value * LinearCopybackBuffer::getStreamSetBlockPointer(llvm::Value * bufferStructPtr, llvm::Value * blockNo) {
    152158    Constant * blockWidth = ConstantInt::get(iBuilder->getSizeTy(), iBuilder->getStride());
    153159    Value * consumerPos_ptr = iBuilder->CreateGEP(bufferStructPtr, {iBuilder->getInt32(0), iBuilder->getInt32(iConsumer_pos)});
     
    160166}
    161167
    162 void LinearBuffer::setConsumerPos(Value * bufferStructPtr, Value * new_consumer_pos) {
     168void LinearCopybackBuffer::setConsumerPos(Value * bufferStructPtr, Value * new_consumer_pos) {
    163169    Type * const i1 = iBuilder->getInt1Ty();
    164170    Type * const i8 = iBuilder->getInt8Ty();
  • icGREP/icgrep-devel/icgrep/kernels/streamset.h

    r5190 r5191  
    1111#include <IDISA/idisa_builder.h>
    1212#include <llvm/IR/Type.h>
    13    
     13
    1414namespace parabix {
    1515   
     
    4141class StreamSetBuffer {
    4242public:
    43     enum class BufferKind : unsigned {BlockBuffer, ExternalFileBuffer, CircularBuffer, LinearBuffer, ExpandingBuffer};
     43    enum class BufferKind : unsigned {BlockBuffer, ExternalFileBuffer, CircularBuffer, LinearCopybackBuffer};
    4444    inline BufferKind getBufferKind() const {return mBufferKind;}
    4545    inline StreamSetType& getBufferStreamSetType() {return mStreamSetType;}
     
    120120
    121121};
    122 
     122   
    123123class CircularBuffer : public StreamSetBuffer {
    124124public:
     
    137137// offset of the block containing the current consumer position is always zero.
    138138//
    139 class LinearBuffer : public StreamSetBuffer {
     139class LinearCopybackBuffer : public StreamSetBuffer {
    140140public:
    141     static inline bool classof(const StreamSetBuffer * b) {return b->getBufferKind() == BufferKind::LinearBuffer;}
     141    static inline bool classof(const StreamSetBuffer * b) {return b->getBufferKind() == BufferKind::LinearCopybackBuffer;}
    142142   
    143     LinearBuffer(IDISA::IDISA_Builder * b, StreamSetType ss_type, size_t bufferBlocks, unsigned AddressSpace = 0) :
    144         StreamSetBuffer(BufferKind::CircularBuffer, b, ss_type, bufferBlocks, AddressSpace) {}
     143    LinearCopybackBuffer(IDISA::IDISA_Builder * b, StreamSetType ss_type, size_t bufferBlocks, unsigned AddressSpace = 0) :
     144        StreamSetBuffer(BufferKind::LinearCopybackBuffer, b, ss_type, bufferBlocks, AddressSpace) {}
    145145   
    146146    llvm::Value * getStreamSetBlockPointer(llvm::Value * bufferStructPtr, llvm::Value * blockNo) override;
  • icGREP/icgrep-devel/icgrep/u8u16.cpp

    r5185 r5191  
    99#include <fstream>
    1010#include <sstream>
     11#include <stdlib.h>
    1112
    1213#include <llvm/IR/Function.h>
     
    4142#include <boost/filesystem.hpp>
    4243#include <boost/iostreams/device/mapped_file.hpp>
    43 
     44#include <boost/interprocess/anonymous_shared_memory.hpp>
     45#include <boost/interprocess/mapped_region.hpp>
    4446#include <fcntl.h>
    4547static cl::OptionCategory u8u16Options("u8u16 Options",
     
    4951
    5052static cl::opt<bool> segmentPipelineParallel("enable-segment-pipeline-parallel", cl::desc("Enable multithreading with segment pipeline parallelism."), cl::cat(u8u16Options));
     53static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(u8u16Options));
     54static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(u8u16Options));
    5155
    5256//
     
    223227using namespace parabix;
    224228
    225 const unsigned u16OutputBlocks = 64;
    226 
    227229Function * u8u16Pipeline(Module * mMod, IDISA::IDISA_Builder * iBuilder, pablo::PabloFunction * function) {
    228230    Type * mBitBlockType = iBuilder->getBitBlockType();
     
    244246    CircularBuffer DeletionCounts(iBuilder, StreamSetType(1, i1), segmentSize * bufferSegments );
    245247   
    246     LinearBuffer U16out(iBuilder, StreamSetType(1, i16), segmentSize * bufferSegments + 2);
     248    // Different choices for the output buffer depending on chosen option.
     249    ExternalFileBuffer U16external(iBuilder, StreamSetType(1, i16));
     250    LinearCopybackBuffer U16out(iBuilder, StreamSetType(1, i16), segmentSize * bufferSegments + 2);
    247251
    248252    s2pKernel  s2pk(iBuilder);
     
    257261   
    258262    p2s_16Kernel_withCompressedOutput p2sk(iBuilder);
    259     p2sk.generateKernel({&U16Bits, &DeletionCounts}, {&U16out});
    260263   
    261264    stdOutKernel stdoutK(iBuilder, 16);
    262     stdoutK.generateKernel({&U16out}, {});
    263 
     265   
     266    if (mMapBuffering || memAlignBuffering) {
     267        p2sk.generateKernel({&U16Bits, &DeletionCounts}, {&U16external});
     268        stdoutK.generateKernel({&U16external}, {});
     269    }
     270    else {
     271        p2sk.generateKernel({&U16Bits, &DeletionCounts}, {&U16out});
     272        stdoutK.generateKernel({&U16out}, {});
     273    }
    264274   
    265275    Type * const size_ty = iBuilder->getSizeTy();
    266276    Type * const voidTy = Type::getVoidTy(mMod->getContext());
    267277    Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
     278    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 16), 1), 0);
    268279    Type * const int32ty = iBuilder->getInt32Ty();
    269280    Type * const int8PtrTy = iBuilder->getInt8PtrTy();
     
    271282
    272283   
    273     Function * const main = cast<Function>(mMod->getOrInsertFunction("Main", voidTy, inputType, size_ty, nullptr));
     284    Function * const main = cast<Function>(mMod->getOrInsertFunction("Main", voidTy, inputType, outputType, size_ty, nullptr));
    274285    main->setCallingConv(CallingConv::C);
    275286    Function::arg_iterator args = main->arg_begin();
    276287   
    277288    Value * const inputStream = &*(args++);
    278     inputStream->setName("input");
     289    inputStream->setName("inputStream");
     290    Value * const outputStream = &*(args++);
     291    outputStream->setName("outputStream");
    279292    Value * const fileSize = &*(args++);
    280293    fileSize->setName("fileSize");
     
    288301    U16Bits.allocateBuffer();
    289302    DeletionCounts.allocateBuffer();
    290     U16out.allocateBuffer();
    291 
     303    if (mMapBuffering || memAlignBuffering) {
     304        U16external.setEmptyBuffer(outputStream);
     305    }
     306    else {
     307        U16out.allocateBuffer();
     308    }
    292309    Value * s2pInstance = s2pk.createInstance({});
    293310    Value * u8u16Instance = u8u16k.createInstance({});
     
    333350
    334351
    335 typedef void (*u8u16FunctionType)(char * byte_data, size_t filesize);
     352typedef void (*u8u16FunctionType)(char * byte_data, char * output_data, size_t filesize);
    336353
    337354static ExecutionEngine * u8u16Engine = nullptr;
     
    386403        mFileBuffer = const_cast<char *>(mFile.data());
    387404    }
    388     //std::cerr << "mFileSize =" << mFileSize << "\n";
    389     //std::cerr << "fn_ptr =" << std::hex << reinterpret_cast<intptr_t>(fn_ptr) << "\n";
    390 
    391     fn_ptr(mFileBuffer, mFileSize);
    392 
     405
     406    if (mMapBuffering) {
     407        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2*mFileSize));
     408        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
     409        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
     410        fn_ptr(mFileBuffer, static_cast<char*>(outputBuffer.get_address()), mFileSize);
     411    }
     412    else if (memAlignBuffering) {
     413        char * outputBuffer;
     414        posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2*mFileSize);
     415        fn_ptr(mFileBuffer, outputBuffer, mFileSize);
     416        free(reinterpret_cast<void *>(outputBuffer));
     417    }
     418    else {
     419        /* No external output buffer */
     420        fn_ptr(mFileBuffer, nullptr, mFileSize);
     421    }
    393422    mFile.close();
    394423   
Note: See TracChangeset for help on using the changeset viewer.