Changeset 5021


Ignore:
Timestamp:
May 3, 2016, 9:38:18 AM (17 months ago)
Author:
cameron
Message:

Large file fix: use boost mapped_file_source

Location:
icGREP/icgrep-devel/icgrep
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep_engine.cpp

    r5017 r5021  
    8282    const size_t fileSize = file_size(file);
    8383    if (fileSize > 0) {
    84         mapped_file file;
     84        mapped_file_source file;
    8585        try {
    86             file.open(fileName, mapped_file::priv, fileSize, 0);
    87         } catch (std::ios_base::failure e) {
     86            file.open(fileName);
     87        } catch (std::exception &e) {
    8888            throw std::runtime_error("Boost mmap error: " + fileName + ": " + e.what());
    8989        }
    90         char * const fileBuffer = file.data();
     90        char * fileBuffer = const_cast<char *>(file.data());
    9191        mGrepFunction(fileBuffer, fileSize, fileIdx, finalLineIsUnterminated(fileBuffer, fileSize));
    9292        file.close();
     93    }
     94    else {
     95        mGrepFunction(nullptr, 0, fileIdx, false);
    9396    }
    9497}
  • icGREP/icgrep-devel/icgrep/u8u16.cpp

    r5012 r5021  
    327327   
    328328    mFileSize = file_size(file);
    329     mapped_file mFile;
     329    mapped_file_source mFile;
    330330    if (mFileSize == 0) {
    331331        mFileBuffer = nullptr;
     
    333333    else {
    334334        try {
    335             mFile.open(mFileName, mapped_file::priv, mFileSize, 0);
    336         } catch (std::ios_base::failure e) {
     335            mFile.open(mFileName);
     336        } catch (std::exception &e) {
    337337            std::cerr << "Error: Boost mmap of " << mFileName << ": " << e.what() << std::endl;
    338338            return;
    339339        }
    340         mFileBuffer = mFile.data();
     340        mFileBuffer = const_cast<char *>(mFile.data());
    341341    }
    342342    //std::cerr << "mFileSize =" << mFileSize << "\n";
  • icGREP/icgrep-devel/icgrep/wc.cpp

    r5020 r5021  
    102102
    103103//
    104 //  Functions taken from toolchain.cpp and modified for casefold
    105 //  JIT_t_ExecutionEngine : remove object cache
    106 //  icgrep_Linking:   unneeded?
    107 //  all others: definitely unneeded
    108104//
    109105
    110 ExecutionEngine * JIT_to_ExecutionEngine (Module * m) {
     106ExecutionEngine * wcJIT_to_ExecutionEngine (Module * m) {
    111107
    112108    InitializeNativeTarget();
     
    192188
    193189
    194 class PipelineBuilder {
     190class wcPipelineBuilder {
    195191public:
    196     PipelineBuilder(llvm::Module * m, IDISA::IDISA_Builder * b);
    197    
    198     ~PipelineBuilder();
     192    wcPipelineBuilder(llvm::Module * m, IDISA::IDISA_Builder * b);
     193   
     194    ~wcPipelineBuilder();
    199195   
    200196    void CreateKernels(pablo::PabloFunction * function);
     
    211207
    212208
    213 Function *  PipelineBuilder::ExecuteKernels() {
     209using namespace pablo;
     210using namespace kernel;
     211
     212wcPipelineBuilder::wcPipelineBuilder(Module * m, IDISA::IDISA_Builder * b)
     213: mMod(m)
     214, iBuilder(b)
     215, mBitBlockType(b->getBitBlockType())
     216, mBlockSize(b->getBitBlockWidth()){
     217   
     218}
     219
     220wcPipelineBuilder::~wcPipelineBuilder(){
     221    delete mS2PKernel;
     222    delete mWC_Kernel;
     223}
     224
     225void wcPipelineBuilder::CreateKernels(PabloFunction * function){
     226    mS2PKernel = new KernelBuilder(iBuilder, "s2p", SegmentSize);
     227    mWC_Kernel = new KernelBuilder(iBuilder, "wc", SegmentSize);
     228   
     229    generateS2PKernel(mMod, iBuilder, mS2PKernel);
     230   
     231    pablo_function_passes(function);
     232   
     233    PabloCompiler pablo_compiler(mMod, iBuilder);
     234    try {
     235        pablo_compiler.setKernel(mWC_Kernel);
     236        pablo_compiler.compile(function);
     237        delete function;
     238        releaseSlabAllocatorMemory();
     239    } catch (std::runtime_error e) {
     240        delete function;
     241        releaseSlabAllocatorMemory();
     242        std::cerr << "Runtime error: " << e.what() << std::endl;
     243        exit(1);
     244    }
     245   
     246}
     247
     248
     249
     250
     251Function * wcPipelineBuilder::ExecuteKernels() {
    214252    Constant * report_counts_routine;
    215253    Type * const int64ty = iBuilder->getInt64Ty();
     
    335373    IDISA::IDISA_Builder * idb = GetIDISA_Builder(M);
    336374
    337     PipelineBuilder pipelineBuilder(M, idb);
     375    wcPipelineBuilder pipelineBuilder(M, idb);
    338376
    339377    Encoding encoding(Encoding::Type::UTF_8, 8);
     
    352390    //verifyModule(*M, &dbgs());
    353391    //std::cerr << "ExecuteKernels(); done\n";
    354     wcEngine = JIT_to_ExecutionEngine(M);
     392    wcEngine = wcJIT_to_ExecutionEngine(M);
    355393   
    356394    wcEngine->finalizeObject();
     
    377415   
    378416    fileSize = file_size(file);
    379     mapped_file mappedFile;
     417    mapped_file_source mappedFile;
    380418    if (fileSize == 0) {
    381419        fileBuffer = nullptr;
     
    383421    else {
    384422        try {
    385             mappedFile.open(fileName, mapped_file::priv, fileSize, 0);
    386         } catch (std::ios_base::failure e) {
     423            mappedFile.open(fileName);
     424        } catch (std::exception &e) {
    387425            std::cerr << "Error: Boost mmap of " << fileName << ": " << e.what() << std::endl;
    388426            return;
    389427        }
    390         fileBuffer = mappedFile.data();
    391     }
    392     //std::cerr << "mFileSize =" << mFileSize << "\n";
    393     //std::cerr << "fn_ptr =" << std::hex << reinterpret_cast<intptr_t>(fn_ptr) << "\n";
    394 
     428        fileBuffer = const_cast<char *>(mappedFile.data());
     429    }
    395430    fn_ptr(fileBuffer, fileSize, fileIdx);
    396431
     
    400435
    401436
    402 using namespace pablo;
    403 using namespace kernel;
    404 
    405 PipelineBuilder::PipelineBuilder(Module * m, IDISA::IDISA_Builder * b)
    406 : mMod(m)
    407 , iBuilder(b)
    408 , mBitBlockType(b->getBitBlockType())
    409 , mBlockSize(b->getBitBlockWidth()){
    410 
    411 }
    412 
    413 PipelineBuilder::~PipelineBuilder(){
    414     delete mS2PKernel;
    415     delete mWC_Kernel;
    416 }
    417 
    418 void PipelineBuilder::CreateKernels(PabloFunction * function){
    419     mS2PKernel = new KernelBuilder(iBuilder, "s2p", SegmentSize);
    420     mWC_Kernel = new KernelBuilder(iBuilder, "wc", SegmentSize);
    421 
    422     generateS2PKernel(mMod, iBuilder, mS2PKernel);
    423 
    424     pablo_function_passes(function);
    425 
    426     PabloCompiler pablo_compiler(mMod, iBuilder);
    427     try {
    428         pablo_compiler.setKernel(mWC_Kernel);
    429         pablo_compiler.compile(function);
    430         delete function;
    431         releaseSlabAllocatorMemory();
    432     } catch (std::runtime_error e) {
    433         delete function;
    434         releaseSlabAllocatorMemory();
    435         std::cerr << "Runtime error: " << e.what() << std::endl;
    436         exit(1);
    437     }
    438    
    439 }
    440437
    441438
     
    488485    delete wcEngine;
    489486   
    490     int maxCount = 0;
     487    size_t maxCount = 0;
    491488    if (CountLines) maxCount = TotalLines;
    492489    if (CountWords) maxCount = TotalWords;
Note: See TracChangeset for help on using the changeset viewer.