Changeset 5693 for icGREP


Ignore:
Timestamp:
Oct 17, 2017, 8:59:21 AM (18 months ago)
Author:
cameron
Message:

Unified FDsource kernel; filename - now interpreted as stdin

Location:
icGREP/icgrep-devel/icgrep
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep_engine.cpp

    r5692 r5693  
    9494   
    9595uint64_t GrepEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) const {
     96    if (fileName == "-") {
     97        return doGrep(STDIN_FILENO, fileIdx);
     98    }
    9699    struct stat sb;
    97100    const int32_t fd = open(fileName.c_str(), O_RDONLY);
     
    123126
    124127uint64_t GrepEngine::doGrep(const int32_t fileDescriptor, const uint32_t fileIdx) const {
    125     assert (mGrepDriver);
    126128    typedef uint64_t (*GrepFunctionType)(int32_t fileDescriptor, const uint32_t fileIdx);
    127129    auto f = reinterpret_cast<GrepFunctionType>(mGrepDriver->getMain());
     
    327329}
    328330
    329 void GrepEngine::grepCodeGen(std::vector<re::RE *> REs, const GrepModeType grepMode, GrepSource grepSource) {
     331void GrepEngine::grepCodeGen(std::vector<re::RE *> REs, const GrepModeType grepMode) {
    330332
    331333    assert (mGrepDriver == nullptr);
     
    340342    Type * const int32Ty = idb->getInt32Ty();
    341343
    342     kernel::Kernel * sourceK = nullptr;
    343    
    344344    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", int64Ty, idb->getInt32Ty(), int32Ty, nullptr));
    345345    mainFunc->setCallingConv(CallingConv::C);
     
    353353
    354354    StreamSetBuffer * ByteStream = mGrepDriver->addBuffer(make_unique<SourceBuffer>(idb, idb->getStreamSetTy(1, 8)));
    355 
    356     if (grepSource == GrepSource::File) {
    357         sourceK = mGrepDriver->addKernelInstance(make_unique<kernel::MMapSourceKernel>(idb, segmentSize));
    358     } else {
    359         sourceK = mGrepDriver->addKernelInstance(make_unique<kernel::ReadSourceKernel>(idb, segmentSize));
    360     }
     355    kernel::Kernel * sourceK = mGrepDriver->addKernelInstance(make_unique<kernel::FDSourceKernel>(idb, segmentSize));
    361356    sourceK->setInitialArguments({fileDescriptor});
    362 
    363357    mGrepDriver->makeKernelCall(sourceK, {}, {ByteStream});
    364358   
  • icGREP/icgrep-devel/icgrep/grep_engine.h

    r5692 r5693  
    3232    ~GrepEngine();
    3333
    34     void grepCodeGen(std::vector<re::RE *> REs, GrepModeType grepMode, GrepSource grepSource);
     34    void grepCodeGen(std::vector<re::RE *> REs, GrepModeType grepMode);
    3535
    3636    void doGrep(const std::string & fileName, std::string & PTXFilename) const;
  • icGREP/icgrep-devel/icgrep/icgrep.cpp

    r5692 r5693  
    178178
    179179    allFiles = getFullFileList(inputFiles);
    180     if ((allFiles.size() > 1) && !grep::NoFilenameFlag) {
     180    if (allFiles.empty()) {
     181        allFiles = { "-" };
     182    }
     183    else if ((allFiles.size() > 1) && !grep::NoFilenameFlag) {
    181184        grep::WithFilenameFlag = true;
    182185    }
     
    184187    grep::GrepEngine grepEngine;
    185188
    186     if (allFiles.empty()) {
    187 
    188         grepEngine.grepCodeGen(REs, grep::Mode, GrepSource::StdIn);
    189         allFiles = { "-" };
    190         grep::initFileResult(allFiles);
    191         grepEngine.doGrep(STDIN_FILENO, 0);
    192 
    193     } else {
    194189               
    195         grepEngine.grepCodeGen(REs, grep::Mode, GrepSource::File);
    196 
    197         grep::initFileResult(allFiles);
    198 
    199         if (Threads <= 1) {
    200             for (unsigned i = 0; i != allFiles.size(); ++i) {
    201                 grepEngine.doGrep(allFiles[i], i);
    202             }
    203         } else if (Threads > 1) {
    204             const unsigned numOfThreads = Threads; // <- convert the command line value into an integer to allow stack allocation
    205             pthread_t threads[numOfThreads];
    206 
    207             for(unsigned long i = 0; i < numOfThreads; ++i){
    208                 const int rc = pthread_create(&threads[i], nullptr, grep::DoGrepThreadFunction, (void *)&grepEngine);
    209                 if (rc) {
    210                     llvm::report_fatal_error("Failed to create thread: code " + std::to_string(rc));
    211                 }
    212             }
    213             for(unsigned i = 0; i < numOfThreads; ++i) {
    214                 void * status = nullptr;
    215                 const int rc = pthread_join(threads[i], &status);
    216                 if (rc) {
    217                     llvm::report_fatal_error("Failed to join thread: code " + std::to_string(rc));
    218                 }
    219             }
    220         }
    221 
    222     }
     190    grepEngine.grepCodeGen(REs, grep::Mode);
     191
     192    grep::initFileResult(allFiles);
     193
     194    if (Threads <= 1) {
     195        for (unsigned i = 0; i != allFiles.size(); ++i) {
     196            grepEngine.doGrep(allFiles[i], i);
     197        }
     198    } else if (Threads > 1) {
     199        const unsigned numOfThreads = Threads; // <- convert the command line value into an integer to allow stack allocation
     200        pthread_t threads[numOfThreads];
     201
     202        for(unsigned long i = 0; i < numOfThreads; ++i){
     203            const int rc = pthread_create(&threads[i], nullptr, grep::DoGrepThreadFunction, (void *)&grepEngine);
     204            if (rc) {
     205                llvm::report_fatal_error("Failed to create thread: code " + std::to_string(rc));
     206            }
     207        }
     208        for(unsigned i = 0; i < numOfThreads; ++i) {
     209            void * status = nullptr;
     210            const int rc = pthread_join(threads[i], &status);
     211            if (rc) {
     212                llvm::report_fatal_error("Failed to join thread: code " + std::to_string(rc));
     213            }
     214        }
     215    }
     216
    223217   
    224218    grep::PrintResults();
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

    r5634 r5693  
    331331
    332332ReadSourceKernel::ReadSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, unsigned blocksPerSegment, unsigned codeUnitWidth)
    333 : SegmentOrientedKernel("read_source"
     333: SegmentOrientedKernel("read_source"  + std::to_string(blocksPerSegment) + "@" + std::to_string(codeUnitWidth)
    334334, {}
    335335, {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
     
    342342}
    343343
     344// Hybrid MMap/Read source kernel
     345   
     346FDSourceKernel::FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & kb, unsigned blocksPerSegment, unsigned codeUnitWidth)
     347: SegmentOrientedKernel("FD_source" + std::to_string(blocksPerSegment) + "@" + std::to_string(codeUnitWidth)
     348, {}
     349, {Binding{kb->getStreamSetTy(1, codeUnitWidth), "sourceBuffer"}}
     350, {Binding{kb->getInt32Ty(), "fileDescriptor"}}
     351, {}
     352, {Binding{IntegerType::get(kb->getContext(), codeUnitWidth)->getPointerTo(), "buffer"}, Binding{kb->getSizeTy(), "capacity"},
     353    Binding{kb->getSizeTy(), "fileSize"}, Binding{kb->getInt8PtrTy(), "readableBuffer"}})
     354, mSegmentBlocks(blocksPerSegment)
     355, mCodeUnitWidth(codeUnitWidth)
     356, mFileSizeFunction(nullptr) {
     357   
     358}
     359
     360void FDSourceKernel::generateFinalizeMethod(const std::unique_ptr<KernelBuilder> & kb) {
     361    BasicBlock * finalizeRead = kb->CreateBasicBlock("finalizeRead");
     362    BasicBlock * finalizeMMap = kb->CreateBasicBlock("finalizeMMap");
     363    BasicBlock * finalizeDone = kb->CreateBasicBlock("finalizeDone");
     364    // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic, otherwise use mmap logic.
     365    kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(0)), finalizeRead, finalizeMMap);
     366    kb->SetInsertPoint(finalizeRead);
     367    reinterpret_cast<ReadSourceKernel *>(this)->ReadSourceKernel::generateFinalizeMethod(kb);
     368    kb->CreateBr(finalizeDone);
     369    kb->SetInsertPoint(finalizeMMap);
     370    reinterpret_cast<MMapSourceKernel *>(this)->MMapSourceKernel::generateFinalizeMethod(kb);
     371    kb->CreateBr(finalizeDone);
     372    kb->SetInsertPoint(finalizeDone);
     373}
     374
     375void FDSourceKernel::generateInitializeMethod(const std::unique_ptr<KernelBuilder> & kb) {
     376    BasicBlock * initializeRead = kb->CreateBasicBlock("initializeRead");
     377    BasicBlock * initializeMMap = kb->CreateBasicBlock("initializeMMap");
     378    BasicBlock * initializeDone = kb->CreateBasicBlock("initializeDone");
     379    // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic, otherwise use MMap logic.
     380    kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(0)), initializeRead, initializeMMap);
     381    kb->SetInsertPoint(initializeRead);
     382    reinterpret_cast<ReadSourceKernel *>(this)->ReadSourceKernel::generateInitializeMethod(kb);
     383    kb->CreateBr(initializeDone);
     384    kb->SetInsertPoint(initializeMMap);
     385    reinterpret_cast<MMapSourceKernel *>(this)->MMapSourceKernel::generateInitializeMethod(kb);
     386    kb->CreateBr(initializeDone);
     387    kb->SetInsertPoint(initializeDone);
     388}
     389
     390void FDSourceKernel::generateDoSegmentMethod(const std::unique_ptr<KernelBuilder> & kb) {
     391    BasicBlock * DoSegmentRead = kb->CreateBasicBlock("DoSegmentRead");
     392    BasicBlock * DoSegmentMMap = kb->CreateBasicBlock("DoSegmentMMap");
     393    BasicBlock * DoSegmentDone = kb->CreateBasicBlock("DoSegmentDone");
     394    // if the fileDescriptor is 0, the file is stdin, use readSource kernel logic, otherwise use MMap logic.
     395    kb->CreateCondBr(kb->CreateICmpEQ(kb->getScalarField("fileDescriptor"), kb->getInt32(0)), DoSegmentRead, DoSegmentMMap);
     396    kb->SetInsertPoint(DoSegmentRead);
     397    reinterpret_cast<ReadSourceKernel *>(this)->ReadSourceKernel::generateDoSegmentMethod(kb);
     398    kb->CreateBr(DoSegmentDone);
     399    kb->SetInsertPoint(DoSegmentMMap);
     400    reinterpret_cast<MMapSourceKernel *>(this)->MMapSourceKernel::generateDoSegmentMethod(kb);
     401    kb->CreateBr(DoSegmentDone);
     402    kb->SetInsertPoint(DoSegmentDone);
     403}
     404
     405
     406void FDSourceKernel::linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & kb) {
     407    mFileSizeFunction = kb->LinkFunction("file_size", &file_size);
     408}
     409   
     410   
    344411/// MEMORY SOURCE KERNEL
    345412
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.h

    r5464 r5693  
    1616    bool isCachable() const override { return true; }
    1717    bool hasSignature() const override { return false; }
    18 protected:
    1918    void linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
    2019    void generateInitializeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
     
    3231    bool isCachable() const override { return true; }
    3332    bool hasSignature() const override { return false; }
    34 protected:
    3533    void generateInitializeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
    3634    void generateDoSegmentMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
     
    4139};
    4240
     41class FDSourceKernel final : public SegmentOrientedKernel {
     42public:
     43    FDSourceKernel(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, unsigned blocksPerSegment = 1, unsigned codeUnitWidth = 8);
     44    bool isCachable() const override { return true; }
     45    bool hasSignature() const override { return false; }
     46    void linkExternalMethods(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
     47    void generateInitializeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
     48    void generateDoSegmentMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
     49    void generateFinalizeMethod(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) override;
     50protected:
     51    const unsigned          mSegmentBlocks;
     52    const unsigned          mCodeUnitWidth;
     53    llvm::Function *        mFileSizeFunction;
     54};
     55   
    4356class MemorySourceKernel final : public SegmentOrientedKernel {
    4457public:
Note: See TracChangeset for help on using the changeset viewer.