Ignore:
Timestamp:
Apr 22, 2017, 4:03:25 PM (2 years ago)
Author:
nmedfort
Message:

Removed non-functional CUDA code from icgrep and consolidated grep and multigrep mode into a single function; allowed segment parallel pipeline to utilize process as its initial thread; modified MMapSourceKernel to map and perform mmap directly and advise the OS to drop consumed data streams.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/base64.cpp

    r5402 r5418  
    1919#include <kernels/radix64.h>
    2020#include <kernels/stdout_kernel.h>
    21 #include <boost/filesystem.hpp>
    22 #include <boost/iostreams/device/mapped_file.hpp>
     21#include <boost/interprocess/mapped_region.hpp>
    2322#include <boost/interprocess/anonymous_shared_memory.hpp>
     23#include <sys/stat.h>
     24#include <fcntl.h>
    2425
    2526using namespace llvm;
     
    4344    Type * mBitBlockType = iBuilder->getBitBlockType();
    4445
    45     Type * const size_ty = iBuilder->getSizeTy();
    4646    Type * const voidTy = Type::getVoidTy(mod->getContext());
    47     Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
     47    Type * const int32Ty = iBuilder->getInt32Ty();
    4848    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
    4949   
    5050   
    51     Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, inputType, outputType, size_ty, nullptr));
     51    Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, int32Ty, outputType, nullptr));
    5252    main->setCallingConv(CallingConv::C);
    5353    Function::arg_iterator args = main->arg_begin();
    5454   
    55     Value * const inputStream = &*(args++);
    56     inputStream->setName("inputStream");
     55    Value * const fileDescriptor = &*(args++);
     56    fileDescriptor->setName("fileDescriptor");
    5757    Value * const outputStream = &*(args++);
    5858    outputStream->setName("outputStream");
    59     Value * const fileSize = &*(args++);
    60     fileSize->setName("fileSize");
    6159
    6260    //Round up to a multiple of 3.
     
    6563    const unsigned bufferSegments = codegen::BufferSegments;
    6664   
    67     ExternalFileBuffer ByteStream(iBuilder, iBuilder->getStreamSetTy(1, 8));
     65    SourceFileBuffer ByteStream(iBuilder, iBuilder->getStreamSetTy(1, 8));
    6866
    6967    CircularBuffer Expanded3_4Out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
     
    7270   
    7371    MMapSourceKernel mmapK(iBuilder, segmentSize);
    74     mmapK.setInitialArguments({fileSize});
     72    mmapK.setInitialArguments({fileDescriptor});
    7573    pxDriver.addKernelCall(mmapK, {}, {&ByteStream});
    7674   
     
    8987    iBuilder->SetInsertPoint(BasicBlock::Create(mod->getContext(), "entry", main,0));
    9088
    91     ByteStream.setStreamSetBuffer(inputStream);
     89    ByteStream.allocateBuffer();
    9290    Expanded3_4Out.allocateBuffer();
    9391    Radix64out.allocateBuffer();
     
    102100
    103101
    104 typedef void (*base64FunctionType)(char * byte_data, char * output_data, size_t filesize);
     102typedef void (*base64FunctionType)(const uint32_t fd, char * outputBuffer);
    105103
    106104base64FunctionType base64CodeGen(void) {
     
    117115}
    118116
     117size_t file_size(const int fd) {
     118    struct stat st;
     119    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
     120        st.st_size = 0;
     121    }
     122    return st.st_size;
     123}
     124
    119125void base64(base64FunctionType fn_ptr, const std::string & fileName) {
    120     std::string mFileName = fileName;
    121     size_t mFileSize;
    122     char * mFileBuffer;
    123126
    124     const boost::filesystem::path file(mFileName);
    125     if (exists(file)) {
    126         if (is_directory(file)) {
    127             return;
    128         }
    129     } else {
    130         std::cerr << "Error: cannot open " << mFileName << " for processing. Skipped.\n";
     127    const int fd = open(fileName.c_str(), O_RDONLY);
     128    if (LLVM_UNLIKELY(fd == -1)) {
     129        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
    131130        return;
    132131    }
    133    
    134     mFileSize = file_size(file);
    135     boost::iostreams::mapped_file_source mFile;
    136     if (mFileSize == 0) {
    137         mFileBuffer = nullptr;
    138     }
    139     else {
    140         try {
    141             mFile.open(mFileName);
    142         } catch (std::exception &e) {
    143             std::cerr << "Error: Boost mmap of " << mFileName << ": " << e.what() << std::endl;
    144             return;
    145         }
    146         mFileBuffer = const_cast<char *>(mFile.data());
    147     }
    148 
    149132    if (mMapBuffering) {
    150         boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2*mFileSize));
     133        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2 * file_size(fd)));
    151134        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
    152135        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
    153         fn_ptr(mFileBuffer, static_cast<char*>(outputBuffer.get_address()), mFileSize);
    154     }
    155     else if (memAlignBuffering) {
     136        fn_ptr(fd, static_cast<char*>(outputBuffer.get_address()));
     137    } else if (memAlignBuffering) {
    156138        char * outputBuffer;
    157         if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2*mFileSize)) {
     139        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2 * file_size(fd))) {
    158140            throw std::bad_alloc();
    159141        }
    160         fn_ptr(mFileBuffer, outputBuffer, mFileSize);
     142        fn_ptr(fd, outputBuffer);
    161143        free(reinterpret_cast<void *>(outputBuffer));
     144    } else { /* No external output buffer */
     145        fn_ptr(fd, nullptr);
    162146    }
    163     else {
    164         /* No external output buffer */
    165         fn_ptr(mFileBuffer, nullptr, mFileSize);
    166     }
    167     mFile.close();
     147    close(fd);
    168148   
    169149}
Note: See TracChangeset for help on using the changeset viewer.