Changeset 5927 for icGREP


Ignore:
Timestamp:
Mar 22, 2018, 7:26:40 AM (14 months ago)
Author:
cameron
Message:

mmap - command line flag + overridden for stdin, devices, files of size 0 (incl virtual files such as /proc/cpuinfo)

Location:
icGREP/icgrep-devel/icgrep
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.cpp

    r5914 r5927  
    423423    const unsigned encodingBits = 8;
    424424
    425     Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getInt64Ty(), idb->getInt32Ty(), nullptr));
     425    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getInt64Ty(), idb->getInt8Ty(), idb->getInt32Ty(), nullptr));
    426426    mainFunc->setCallingConv(CallingConv::C);
    427427    idb->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", mainFunc, 0));
    428428    auto args = mainFunc->arg_begin();
    429429
     430    Value * const useMMap = &*(args++);
     431    useMMap->setName("useMMap");
    430432    Value * const fileDescriptor = &*(args++);
    431433    fileDescriptor->setName("fileDescriptor");
     
    433435    StreamSetBuffer * ByteStream = mGrepDriver->addBuffer<SourceBuffer>(idb, idb->getStreamSetTy(1, encodingBits));
    434436    kernel::Kernel * sourceK = mGrepDriver->addKernelInstance<kernel::FDSourceKernel>(idb);
    435     sourceK->setInitialArguments({fileDescriptor});
     437    sourceK->setInitialArguments({useMMap, fileDescriptor});
    436438    mGrepDriver->makeKernelCall(sourceK, {}, {ByteStream});
    437439
     
    516518    const unsigned encodingBits = 8;
    517519
    518     Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getInt64Ty(), idb->getInt32Ty(), idb->getIntAddrTy(), nullptr));
     520    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getInt64Ty(), idb->getInt8Ty(), idb->getInt32Ty(), idb->getIntAddrTy(), nullptr));
    519521    mainFunc->setCallingConv(CallingConv::C);
    520522    idb->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", mainFunc, 0));
    521523    auto args = mainFunc->arg_begin();
    522524
     525    Value * const useMMap = &*(args++);
     526    useMMap->setName("useMMap");
    523527    Value * const fileDescriptor = &*(args++);
    524528    fileDescriptor->setName("fileDescriptor");
     
    528532    StreamSetBuffer * ByteStream = mGrepDriver->addBuffer<SourceBuffer>(idb, idb->getStreamSetTy(1, encodingBits));
    529533    kernel::Kernel * sourceK = mGrepDriver->addKernelInstance<kernel::FDSourceKernel>(idb);
    530     sourceK->setInitialArguments({fileDescriptor});
     534    sourceK->setInitialArguments({useMMap, fileDescriptor});
    531535    mGrepDriver->makeKernelCall(sourceK, {}, {ByteStream});
    532536
     
    553557
    554558uint64_t GrepEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) {
    555     typedef uint64_t (*GrepFunctionType)(int32_t fileDescriptor);
     559    typedef uint64_t (*GrepFunctionType)(bool useMMap, int32_t fileDescriptor);
     560    using namespace boost::filesystem;
     561    path p(fileName);
     562    bool useMMap = grep::MmapFlag;
     563    if (p == "-") useMMap = false;
     564    if (!is_regular_file(p)) useMMap = false;
     565
    556566    auto f = reinterpret_cast<GrepFunctionType>(mGrepDriver->getMain());
    557567
     
    559569    if (fileDescriptor == -1) return 0;
    560570
    561     uint64_t grepResult = f(fileDescriptor);
     571    uint64_t grepResult = f(useMMap, fileDescriptor);
    562572    close(fileDescriptor);
    563573    return grepResult;
     
    589599
    590600uint64_t EmitMatchesEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) {
    591     typedef uint64_t (*GrepFunctionType)(int32_t fileDescriptor, intptr_t accum_addr);
     601    typedef uint64_t (*GrepFunctionType)(bool useMMap, int32_t fileDescriptor, intptr_t accum_addr);
     602    using namespace boost::filesystem;
     603    path p(fileName);
     604    bool useMMap = grep::MmapFlag;
     605    if (p == "-") useMMap = false;
     606    if (!is_regular_file(p)) useMMap = false;
    592607    auto f = reinterpret_cast<GrepFunctionType>(mGrepDriver->getMain());
    593 
    594608    int32_t fileDescriptor = openFile(fileName, mResultStrs[fileIdx]);
    595609    if (fileDescriptor == -1) return 0;
    596610    EmitMatch accum(linePrefix(fileName), mResultStrs[fileIdx]);
    597     f(fileDescriptor, reinterpret_cast<intptr_t>(&accum));
     611    f(useMMap, fileDescriptor, reinterpret_cast<intptr_t>(&accum));
    598612    close(fileDescriptor);
    599613    if (accum.mLineCount > 0) grepMatchFound = true;
  • icGREP/icgrep-devel/icgrep/grep_interface.cpp

    r5900 r5927  
    9696
    9797bool MmapFlag;
    98 static cl::opt<bool, true> MmapOption("mmap", cl::location(MmapFlag), cl::desc("Use mmap for file input."), cl::cat(Input_Options));
     98static cl::opt<bool, true> MmapOption("mmap", cl::location(MmapFlag),  cl::init(1), cl::desc("Use mmap for file input (default)."), cl::cat(Input_Options));
    9999
    100100std::string ExcludeFlag;
     
    267267        llvm::report_fatal_error("Sorry, -U is not yet supported.\n");
    268268    }
    269     if (NullDataFlag) {
    270         llvm::report_fatal_error("Sorry, -z is not yet supported.\n");
    271     }
    272269    if (ExcludeFlag!="") {
    273270        llvm::report_fatal_error("Sorry, -exclude is not yet supported.\n");
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

    r5915 r5927  
    11/*
    2  *  Copyright (c) 2017 International Characters.
     2 *  Copyright (c) 2018 International Characters.
    33 *  This software is licensed to the public under the Open Software License 3.0.
    44 */
     
    4343    PointerType * const codeUnitPtrTy = kb->getIntNTy(codeUnitWidth)->getPointerTo();
    4444    Value * const fileBuffer = kb->CreatePointerCast(kb->CreateFileSourceMMap(fd, fileSize), codeUnitPtrTy);
    45     kb->setScalarField("buffer", fileBuffer);   
    46     kb->setBaseAddress("sourceBuffer", fileBuffer);   
     45    kb->setScalarField("buffer", fileBuffer);
     46    kb->setBaseAddress("sourceBuffer", fileBuffer);
    4747    kb->CreateMAdvise(fileBuffer, fileSize, CBuilder::ADVICE_WILLNEED);
    4848    if (codeUnitWidth > 8) {
     
    237237    BasicBlock * finalizeMMap = kb->CreateBasicBlock("finalizeMMap");
    238238    BasicBlock * finalizeDone = kb->CreateBasicBlock("finalizeDone");
    239     // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic, otherwise use mmap logic.
    240     kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(STDIN_FILENO)), finalizeRead, finalizeMMap);
     239    kb->CreateCondBr(kb->CreateTrunc(kb->getScalarField("useMMap"), kb->getInt1Ty()), finalizeMMap, finalizeRead);
     240    kb->SetInsertPoint(finalizeMMap);
     241    MMapSourceKernel::unmapSourceBuffer(kb);
     242    kb->CreateBr(finalizeDone);
    241243    kb->SetInsertPoint(finalizeRead);
    242244    ReadSourceKernel::freeBuffer(kb);
    243245    kb->CreateBr(finalizeDone);
    244     kb->SetInsertPoint(finalizeMMap);
    245     MMapSourceKernel::unmapSourceBuffer(kb);
    246     kb->CreateBr(finalizeDone);
    247246    kb->SetInsertPoint(finalizeDone);
    248247}
     
    250249void FDSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & kb) {
    251250    BasicBlock * initializeRead = kb->CreateBasicBlock("initializeRead");
     251    BasicBlock * tryMMap = kb->CreateBasicBlock("tryMMap");
    252252    BasicBlock * initializeMMap = kb->CreateBasicBlock("initializeMMap");
    253253    BasicBlock * initializeDone = kb->CreateBasicBlock("initializeDone");
    254     // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic, otherwise use MMap logic.
    255     kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(STDIN_FILENO)), initializeRead, initializeMMap);
    256     kb->SetInsertPoint(initializeRead);
    257     ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, getStride(), kb);
    258     kb->CreateBr(initializeDone);
     254    // The source will use MMapSource or readSoure kernel logic depending on the useMMap
     255    // parameter, possibly overridden.
     256    Value * useMMap = kb->CreateTrunc(kb->getScalarField("useMMap"), kb->getInt1Ty());
     257    // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic.
     258    Value * fd = kb->getScalarField("fileDescriptor");
     259    useMMap = kb->CreateAnd(useMMap, kb->CreateICmpNE(fd, kb->getInt32(STDIN_FILENO)));
     260    kb->CreateCondBr(useMMap, tryMMap, initializeRead);
     261   
     262    kb->SetInsertPoint(tryMMap);
     263    // If the fileSize is 0, we may have a virtual file such as /proc/cpuinfo
     264    Value * fileSize = kb->CreateZExtOrTrunc(kb->CreateCall(mFileSizeFunction, fd), kb->getSizeTy());
     265    useMMap = kb->CreateICmpNE(fileSize, kb->getSize(0));
     266    kb->CreateCondBr(useMMap, initializeMMap, initializeRead);
    259267    kb->SetInsertPoint(initializeMMap);
    260268    MMapSourceKernel::generateInitializeMethod(mFileSizeFunction, mCodeUnitWidth, kb);
     269    kb->CreateBr(initializeDone);
     270
     271    kb->SetInsertPoint(initializeRead);
     272    // Ensure that readSource logic is used throughout.
     273    kb->setScalarField("useMMap", kb->getInt8(0));
     274    ReadSourceKernel::generateInitializeMethod(mCodeUnitWidth, getStride(), kb);
    261275    kb->CreateBr(initializeDone);
    262276    kb->SetInsertPoint(initializeDone);
     
    267281    BasicBlock * DoSegmentMMap = kb->CreateBasicBlock("DoSegmentMMap");
    268282    BasicBlock * DoSegmentDone = kb->CreateBasicBlock("DoSegmentDone");
    269     // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic, otherwise use MMap logic.
    270     kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(STDIN_FILENO)), DoSegmentRead, DoSegmentMMap);
     283    kb->CreateCondBr(kb->CreateTrunc(kb->getScalarField("useMMap"), kb->getInt1Ty()), DoSegmentMMap, DoSegmentRead);
     284    kb->SetInsertPoint(DoSegmentMMap);
     285    MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, kb);
     286    kb->CreateBr(DoSegmentDone);
    271287    kb->SetInsertPoint(DoSegmentRead);
    272288    ReadSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, getStride(), kb);
    273289    kb->CreateBr(DoSegmentDone);
    274     kb->SetInsertPoint(DoSegmentMMap);
    275     MMapSourceKernel::generateDoSegmentMethod(mCodeUnitWidth, kb);
    276     kb->CreateBr(DoSegmentDone);
    277290    kb->SetInsertPoint(DoSegmentDone);
    278291}
    279 
    280292
    281293/// MEMORY SOURCE KERNEL
     
    324336, {}
    325337, {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer", FixedRate(), Deferred()}}
    326 , {Binding{kb->getInt32Ty(), "fileDescriptor"}}
     338, {Binding{kb->getInt8Ty(), "useMMap"}, Binding{kb->getInt32Ty(), "fileDescriptor"}}
    327339, {}
    328340, {Binding{kb->getIntNTy(codeUnitWidth)->getPointerTo(), "buffer"}, Binding{kb->getSizeTy(), "fileSize"}})
Note: See TracChangeset for help on using the changeset viewer.