Changeset 4967


Ignore:
Timestamp:
Mar 10, 2016, 2:38:50 PM (3 years ago)
Author:
lindanl
Message:

Multiple input files can run on separate threads.

Location:
icGREP/icgrep-devel/icgrep
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep_engine.cpp

    r4961 r4967  
    5252
    5353
    54 bool GrepEngine::finalLineIsUnterminated() const {
     54bool GrepEngine::finalLineIsUnterminated(char * mFileBuffer, int mFileSize) const {
    5555    if (mFileSize == 0) return false;
    5656    unsigned char end_byte = static_cast<unsigned char>(mFileBuffer[mFileSize-1]);
     
    6868}
    6969
    70 bool GrepEngine::openMMap(const std::string & fileName) {
    71 
    72     mFileName = fileName;
     70void GrepEngine::doGrep(const std::string & fileName) {
     71    std::string mFileName = fileName;
     72    int mFileSize;
     73    char * mFileBuffer;
    7374
    7475#ifdef USE_BOOST_MMAP
     
    9394        } catch (std::ios_base::failure e) {
    9495            std::cerr << "Error: Boost mmap of " << mFileName << ": " << e.what() << std::endl;
    95             return false;
     96            return;
    9697        }
    9798        mFileBuffer = mFile.data();
     
    102103    if (fdSrc == -1) {
    103104        std::cerr << "Error: cannot open " << mFileName << " for processing. Skipped.\n";
    104         return false;
     105        return;
    105106    }
    106107    if (fstat(fdSrc, &infile_sb) == -1) {
    107108        std::cerr << "Error: cannot stat " << mFileName << " for processing. Skipped.\n";
    108109        close (fdSrc);
    109         return false;
     110        return;
    110111    }
    111112    if (S_ISDIR(infile_sb.st_mode)) {
    112113        close (fdSrc);
    113         return false;
     114        return;
    114115    }
    115116    mFileSize = infile_sb.st_size;
     
    128129                close (fdSrc);
    129130            }
    130             return false;
     131            return;
    131132        }
    132133    }
     
    134135
    135136#endif
    136     return true;  // success
    137 }
     137   
     138    uint64_t finalLineUnterminated = 0;
     139    if(finalLineIsUnterminated(mFileBuffer, mFileSize))
     140        finalLineUnterminated = 1;
     141   
     142    mMainFcn(mFileBuffer, mFileSize, mFileName.c_str(), finalLineUnterminated);
    138143
    139 void GrepEngine::closeMMap() {
    140144#ifdef USE_BOOST_MMAP
    141145    mFile.close();
    142146#else
    143147    munmap((void *)mFileBuffer, mFileSize);
    144 #endif  
     148#endif
    145149
    146150}
    147151
    148 void GrepEngine::doGrep() {
    149        
    150     llvm::raw_os_ostream out(std::cout);
    151    
    152     uint64_t finalLineUnterminated = 0;
    153     if(finalLineIsUnterminated())
    154         finalLineUnterminated = 1;
    155    
    156     mMainFcn(mFileBuffer, mFileSize, mFileName.c_str(), finalLineUnterminated);
    157    
    158     if (!mIsNameExpression) PrintTotalCount();
    159 }
    160152
    161153void GrepEngine::grepCodeGen(std::string moduleName, re::RE * re_ast, bool isNameExpression) {
    162154                           
    163     Module * M = new Module(moduleName, getGlobalContext());
     155    Module * M = new Module("moduleName", getGlobalContext());
    164156   
    165157    IDISA::IDISA_Builder * idb = GetIDISA_Builder(M);
     
    190182re::CC *  GrepEngine::grepCodepoints() {
    191183    setParsedCodePointSet();
    192     mFileBuffer = getUnicodeNameDataPtr();
    193     mFileSize = getUnicodeNameDataSize();
    194     mFileName = "Uname.txt";
    195     doGrep();
     184    char * mFileBuffer = getUnicodeNameDataPtr();
     185    int mFileSize = getUnicodeNameDataSize();
     186    std::string mFileName = "Uname.txt";
     187
     188    uint64_t finalLineUnterminated = 0;
     189    if(finalLineIsUnterminated(mFileBuffer, mFileSize))
     190        finalLineUnterminated = 1;   
     191    mMainFcn(mFileBuffer, mFileSize, mFileName.c_str(), finalLineUnterminated);
     192
    196193    return getParsedCodePointSet();
    197194}
  • icGREP/icgrep-devel/icgrep/grep_engine.h

    r4959 r4967  
    2424    void grepCodeGen(std::string moduleName, re::RE * re_ast, bool isNameExpression = false);
    2525   
    26     bool openMMap(const std::string & fileName);
    27    
    28     void closeMMap();
    29 
    30     void doGrep();
     26    void doGrep(const std::string & fileName);
    3127   
    3228    re::CC *  grepCodepoints();
     
    3834private:
    3935   
    40     bool finalLineIsUnterminated() const;
     36    bool finalLineIsUnterminated(char * fileBuffer, int fileSize) const;
    4137
    4238    main_fcn_T mMainFcn;
    4339   
    4440    bool mIsNameExpression;
    45     std::string mFileName;
    46     size_t mFileSize;
    47     char * mFileBuffer;
    4841    llvm::ExecutionEngine * mEngine;
    4942};
  • icGREP/icgrep-devel/icgrep/icgrep.cpp

    r4963 r4967  
    2626
    2727#include <boost/uuid/sha1.hpp>
     28#include <toolchain.h>
     29#include <atomic>
    2830
    2931static cl::OptionCategory aRegexSourceOptions("Regular Expression Options",
     
    4951static cl::opt<std::string> RegexFilename("f", cl::desc("Take regular expressions (one per line) from a file"), cl::value_desc("regex file"), cl::init(""), cl::cat(aRegexSourceOptions));
    5052static cl::opt<std::string> IRFileName("precompiled", cl::desc("Use precompiled regular expression"), cl::value_desc("LLVM IR file"), cl::init(""), cl::cat(aRegexSourceOptions));
     53
     54static cl::opt<int> Threads("t", cl::desc("Total number of threads."), cl::init(1));
    5155
    5256
     
    110114}
    111115
     116GrepEngine grepEngine;
     117std::atomic<int> fileCount;
     118
     119void *DoGrep(void *threadid)
     120{
     121    long tid;
     122    tid = (long)threadid;
     123    if (tid+1 < inputFiles.size())
     124        grepEngine.doGrep(inputFiles[tid+1]);
     125
     126    pthread_exit(NULL);
     127}
    112128
    113129int main(int argc, char *argv[]) {
     
    143159    std::string module_name = "grepcode:" + sha1sum(allREs) + ":" + std::to_string(globalFlags);
    144160   
    145     GrepEngine grepEngine;
    146161    grepEngine.grepCodeGen(module_name, re_ast);
    147    
    148     for (unsigned i = firstInputFile; i != inputFiles.size(); ++i) {
    149         if (grepEngine.openMMap(inputFiles[i])) {
    150             grepEngine.doGrep();
    151             grepEngine.closeMMap();
     162
     163
     164    initResult(inputFiles, inputFiles.size());
     165
     166    if(Threads == 1){
     167        for (unsigned i = firstInputFile; i != inputFiles.size(); ++i) {
     168            grepEngine.doGrep(inputFiles[i]);
     169        }       
     170    }
     171    else if (Threads > 1) {
     172
     173        pthread_t threads[100];
     174        int rc;
     175        long t;
     176        void *status;
     177
     178        for(t=0; t<Threads; t++){
     179          rc = pthread_create(&threads[t], NULL, DoGrep, (void *)t);
     180          if (rc){
     181             printf("ERROR; return code from pthread_create() is %d\n", rc);
     182             exit(-1);
     183          }
    152184        }
    153     }
     185
     186        for(t=0; t<Threads; t++) {
     187            rc = pthread_join(threads[t], &status);
     188            if (rc) {
     189                printf("ERROR; return code from pthread_join() is %d\n", rc);
     190                exit(-1);
     191            }
     192        }
     193    }
     194    else {
     195        std::cerr << "Invalid number of threads :" << Threads << std::endl;
     196    }
     197
     198    PrintResult();   
    154199   
    155200    return 0;
  • icGREP/icgrep-devel/icgrep/toolchain.cpp

    r4964 r4967  
    470470}
    471471
    472 int total_count = 0;
     472
     473
     474int * total_count;
     475std::stringstream * resultStrs;
     476std::vector<std::string> inputFiles;
     477
     478void initResult(std::vector<std::string> filenames, const int n){
     479
     480    inputFiles = filenames;
     481
     482    std::stringstream * ss = new std::stringstream[n];
     483    resultStrs = ss;
     484
     485    int * c = new int[n];
     486    total_count = c;
     487    for (int i=1; i<inputFiles.size(); i++){
     488        total_count[i-1] = 0;
     489    }
     490
     491}
    473492
    474493extern "C" {
    475494    void wrapped_report_match(uint64_t lineNum, uint64_t line_start, uint64_t line_end, const char * buffer, int filesize, char * filename) {
     495       
     496        int idx = 0;
     497        for (int i=1; i<inputFiles.size(); i++){
     498            if (inputFiles[i] == filename){
     499                idx = i-1;
     500                break;
     501            }
     502        }
     503
    476504        if(CountOnly){
    477             total_count++;
     505            total_count[idx]++;
    478506            return;
    479507        }
    480508
    481         llvm::raw_os_ostream out(std::cout);
    482509        if (ShowFileNames) {
    483             out << filename << ':';
     510            resultStrs[idx] << filename << ':';
    484511        }
    485512        if (ShowLineNumbers) {
    486             out << lineNum << ":";
     513            resultStrs[idx] << lineNum << ":";
    487514        }
    488515
     
    493520        if (line_end == filesize) {
    494521            // The match position is at end-of-file.   We have a final unterminated line.
    495             out.write(&buffer[line_start], line_end - line_start);
     522            resultStrs[idx].write(&buffer[line_start], line_end - line_start);
    496523            if (NormalizeLineBreaks) {
    497                 out << '\n';  // terminate it
     524                resultStrs[idx] << '\n';  // terminate it
    498525            }
    499526            return;
     
    508535                line_end -= 2;
    509536            }
    510             out.write(&buffer[line_start], line_end - line_start);
    511             out << '\n';
     537            resultStrs[idx].write(&buffer[line_start], line_end - line_start);
     538            resultStrs[idx] << '\n';
    512539        }
    513540        else{   
     
    520547                }
    521548            }
    522             out.write(&buffer[line_start], line_end - line_start + 1);
    523         }
    524     }
    525 }
    526 
    527 
    528 void PrintTotalCount(){
     549            resultStrs[idx].write(&buffer[line_start], line_end - line_start + 1);
     550        }
     551    }
     552}
     553
     554void PrintResult(){
    529555    if(CountOnly){
    530         std::cout << total_count << std::endl;
     556        for (int i=1; i<inputFiles.size(); i++){
     557            std::cout << total_count[i-1] << std::endl;
     558        }
     559        return;
     560    }
     561
     562    std::string out;
     563    for (int i=1; i<inputFiles.size(); i++){
     564        std::cout << resultStrs[i-1].str();
    531565    }
    532566}
  • icGREP/icgrep-devel/icgrep/toolchain.h

    r4952 r4967  
    2626void pablo_function_passes(pablo::PabloFunction * function);
    2727
     28Module * grepIRGen(std::string moduleName, re::RE * re_ast, bool isNameExpression = false);
     29
    2830ExecutionEngine * JIT_to_ExecutionEngine (Module * m);
    2931
    3032void icgrep_Linking(Module * m, ExecutionEngine * e);
    3133
    32 void PrintTotalCount();
    3334re::CC * getParsedCodePointSet();
    3435void setParsedCodePointSet();
    3536
     37void initResult(std::vector<std::string> filenames, const int n);
     38void PrintResult();
     39
    3640#endif
Note: See TracChangeset for help on using the changeset viewer.