Ignore:
Timestamp:
Apr 22, 2017, 4:03:25 PM (2 years ago)
Author:
nmedfort
Message:

Removed non-functional CUDA code from icgrep and consolidated grep and multigrep mode into a single function; allowed segment parallel pipeline to utilize process as its initial thread; modified MMapSourceKernel to map and perform mmap directly and advise the OS to drop consumed data streams.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/u8u16.cpp

    r5414 r5418  
    2424#include <pablo/pe_zeroes.h>
    2525#include <kernels/toolchain.h>
    26 #include <boost/iostreams/device/mapped_file.hpp>  // for mapped_file_source
    27 #include <boost/filesystem.hpp>
    28 #include <boost/interprocess/anonymous_shared_memory.hpp>
    2926#include "kernels/streamset.h"                     // for CircularBuffer
    3027#include <kernels/pipeline.h>
     
    3633#include "llvm/Support/Compiler.h"                 // for LLVM_UNLIKELY
    3734#include <pablo/builder.hpp>                       // for PabloBuilder
     35#include <boost/interprocess/anonymous_shared_memory.hpp>
     36#include <boost/interprocess/mapped_region.hpp>
    3837#include <iostream>
    3938
     
    271270    assert (iBuilder);
    272271
    273     Type * const size_ty = iBuilder->getSizeTy();
    274272    Type * const voidTy = iBuilder->getVoidTy();
    275273    Type * const bitBlockType = iBuilder->getBitBlockType();
    276     Type * const inputType = ArrayType::get(ArrayType::get(bitBlockType, 8), 1)->getPointerTo();
    277274    Type * const outputType = ArrayType::get(ArrayType::get(bitBlockType, 16), 1)->getPointerTo();
    278    
    279     Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, inputType, outputType, size_ty, nullptr));
     275
     276    Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, iBuilder->getInt32Ty(), outputType, nullptr));
    280277    main->setCallingConv(CallingConv::C);
    281278    Function::arg_iterator args = main->arg_begin();
    282    
    283     Value * const inputStream = &*(args++);
    284     inputStream->setName("inputStream");
     279
     280    Value * const fileDecriptor = &*(args++);
     281    fileDecriptor->setName("fileDecriptor");
    285282    Value * const outputStream = &*(args++);
    286283    outputStream->setName("outputStream");
    287     Value * const fileSize = &*(args++);
    288     fileSize->setName("fileSize");
    289284
    290285    iBuilder->SetInsertPoint(BasicBlock::Create(mod->getContext(), "entry", main,0));
    291286   
    292287    // File data from mmap
    293     StreamSetBuffer * ByteStream = pxDriver.addExternalBuffer(make_unique<ExternalFileBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)), inputStream);
     288    StreamSetBuffer * ByteStream = pxDriver.addBuffer(make_unique<SourceFileBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)));
    294289   
    295290    KernelBuilder * mmapK = pxDriver.addKernelInstance(make_unique<MMapSourceKernel>(iBuilder, segmentSize));
    296     mmapK->setInitialArguments({fileSize});
     291    mmapK->setInitialArguments({fileDecriptor});
    297292    pxDriver.makeKernelCall(mmapK, {}, {ByteStream});
    298293   
     
    371366    assert (iBuilder);
    372367   
    373     Type * const size_ty = iBuilder->getSizeTy();
    374368    Type * const voidTy = iBuilder->getVoidTy();
    375369    Type * const bitBlockType = iBuilder->getBitBlockType();
    376     Type * const inputType = ArrayType::get(ArrayType::get(bitBlockType, 8), 1)->getPointerTo();
    377370    Type * const outputType = ArrayType::get(ArrayType::get(bitBlockType, 16), 1)->getPointerTo();
    378371   
    379     Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, inputType, outputType, size_ty, nullptr));
     372    Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, iBuilder->getInt32Ty(), outputType, nullptr));
    380373    main->setCallingConv(CallingConv::C);
    381374    Function::arg_iterator args = main->arg_begin();
    382375   
    383     Value * const inputStream = &*(args++);
    384     inputStream->setName("inputStream");
     376    Value * const fileDecriptor = &*(args++);
     377    fileDecriptor->setName("fileDecriptor");
    385378    Value * const outputStream = &*(args++);
    386379    outputStream->setName("outputStream");
    387     Value * const fileSize = &*(args++);
    388     fileSize->setName("fileSize");
    389    
     380
    390381    iBuilder->SetInsertPoint(BasicBlock::Create(mod->getContext(), "entry", main,0));
    391382
    392383    // File data from mmap
    393     StreamSetBuffer * ByteStream = pxDriver.addExternalBuffer(make_unique<ExternalFileBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)), inputStream);
     384    StreamSetBuffer * ByteStream = pxDriver.addBuffer(make_unique<SourceFileBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)));
    394385   
    395386    KernelBuilder * mmapK = pxDriver.addKernelInstance(make_unique<MMapSourceKernel>(iBuilder, segmentSize));
    396     mmapK->setInitialArguments({fileSize});
     387    mmapK->setInitialArguments({fileDecriptor});
    397388    pxDriver.makeKernelCall(mmapK, {}, {ByteStream});
    398389   
     
    443434
    444435
    445 typedef void (*u8u16FunctionType)(char * byte_data, char * output_data, size_t filesize);
     436typedef void (*u8u16FunctionType)(uint32_t fd, char * output_data);
    446437
    447438u8u16FunctionType u8u16CodeGen(void) {
     
    465456}
    466457
     458size_t file_size(const int fd) {
     459    struct stat st;
     460    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
     461        st.st_size = 0;
     462    }
     463    return st.st_size;
     464}
     465
    467466void u8u16(u8u16FunctionType fn_ptr, const std::string & fileName) {
    468 
    469     const boost::filesystem::path file(fileName);
    470     if (exists(file)) {
    471         if (is_directory(file)) {
    472             return;
     467    const int fd = open(fileName.c_str(), O_RDONLY);
     468    if (LLVM_UNLIKELY(fd == -1)) {
     469        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
     470    } else {
     471        const auto fileSize = file_size(fd);
     472        if (mMapBuffering) {
     473            boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2 * fileSize));
     474            outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
     475            outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
     476            fn_ptr(fd, static_cast<char*>(outputBuffer.get_address()));
     477        } else if (memAlignBuffering) {
     478            char * outputBuffer;
     479            const auto r = posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2 * fileSize);
     480            if (LLVM_UNLIKELY(r != 0)) {
     481                throw std::runtime_error("posix_memalign failed with return code " + std::to_string(r));
     482            }
     483            fn_ptr(fd, outputBuffer);
     484            free(reinterpret_cast<void *>(outputBuffer));
     485        } else { /* No external output buffer */
     486            fn_ptr(fd, nullptr);
    473487        }
    474     } else {
    475         std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
    476         return;
    477     }
    478    
    479     size_t fileSize = file_size(file);
    480     boost::iostreams::mapped_file_source input;
    481 
    482     char * fileBuffer = nullptr;
    483     if (fileSize) {
    484         try {
    485             input.open(fileName);
    486             fileBuffer = const_cast<char *>(input.data());
    487         } catch (std::exception & e) {
    488             throw std::runtime_error("Boost mmap error: " + fileName + ": " + e.what());
    489         }       
    490     }
    491 
    492     if (mMapBuffering) {
    493         boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2*fileSize));
    494         outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
    495         outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
    496         fn_ptr(fileBuffer, static_cast<char*>(outputBuffer.get_address()), fileSize);
    497     } else if (memAlignBuffering) {
    498         char * outputBuffer;
    499         const auto r = posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2*fileSize);
    500         if (LLVM_UNLIKELY(r != 0)) {
    501             throw std::runtime_error("posix_memalign failed with return code " + std::to_string(r));
    502         }
    503         fn_ptr(fileBuffer, outputBuffer, fileSize);
    504         free(reinterpret_cast<void *>(outputBuffer));
    505     } else {
    506         /* No external output buffer */
    507         fn_ptr(fileBuffer, nullptr, fileSize);
    508     }
    509     input.close();
    510    
     488        close(fd);
     489    }
    511490}
    512491
Note: See TracChangeset for help on using the changeset viewer.