Ignore:
Timestamp:
Oct 18, 2017, 12:59:35 PM (22 months ago)
Author:
cameron
Message:

Complete the scanmatch kernel conversion to use MatchAccumulator? objects

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep_engine.cpp

    r5693 r5695  
    3131#include <toolchain/NVPTXDriver.h>
    3232#include <iostream>
    33 #include <sstream>
    3433#include <cc/multiplex_CCs.h>
    3534#include <llvm/Support/raw_ostream.h>
     
    3837#include <fcntl.h>
    3938#include <errno.h>
    40 #include <mutex>
    4139
    4240using namespace parabix;
     
    4543namespace grep {
    4644
    47 static std::stringstream * resultStrs = nullptr;
    48 static std::vector<std::string> inputFiles;
    49 static std::vector<std::string> linePrefix;
    50 static bool grepMatchFound;
    51 
    52 size_t * startPoints = nullptr;
    53 size_t * accumBytes = nullptr;
    54 
    55 
    56 std::mutex count_mutex;
    57 size_t fileCount;
    5845
    5946// DoGrep thread function.
     
    6350    grep::GrepEngine * grepEngine = (grep::GrepEngine *)args;
    6451
    65     count_mutex.lock();
    66     fileIdx = fileCount;
    67     fileCount++;
    68     count_mutex.unlock();
    69 
    70     while (fileIdx < inputFiles.size()) {
    71         size_t grepResult = grepEngine->doGrep(inputFiles[fileIdx], fileIdx);
     52    grepEngine->count_mutex.lock();
     53    fileIdx = grepEngine->fileCount;
     54    grepEngine->fileCount++;
     55    grepEngine->count_mutex.unlock();
     56
     57    while (fileIdx < grepEngine->inputFiles.size()) {
     58        size_t grepResult = grepEngine->doGrep(grepEngine->inputFiles[fileIdx], fileIdx);
    7259       
    73         count_mutex.lock();
    74         if (grepResult > 0) grepMatchFound = true;
    75         fileIdx = fileCount;
    76         fileCount++;
    77         count_mutex.unlock();
    78         if (QuietMode && grepMatchFound) pthread_exit(nullptr);
     60        grepEngine->count_mutex.lock();
     61        if (grepResult > 0) grepEngine->grepMatchFound = true;
     62        fileIdx = grepEngine->fileCount;
     63        grepEngine->fileCount++;
     64        grepEngine->count_mutex.unlock();
     65        if (QuietMode && grepEngine->grepMatchFound) pthread_exit(nullptr);
    7966    }
    8067
    8168    pthread_exit(nullptr);
    8269}
     70   
     71   
     72    void NonNormalizingReportMatch::accumulate_match (const size_t lineNum, char * line_start, char * line_end) {
     73        if (!(WithFilenameFlag | LineNumberFlag) && (line_start == mPrevious_line_end + 1)) {
     74            // Consecutive matches: only one write call needed.
     75            mResultStr.write(mPrevious_line_end, line_end - mPrevious_line_end);
     76        }
     77        else {
     78            if (mLineCount > 0) {
     79                // deal with the final byte of the previous line.
     80                mResultStr.write(mPrevious_line_end, 1);
     81            }
     82            if (WithFilenameFlag) {
     83                mResultStr << mLinePrefix;
     84            }
     85            if (LineNumberFlag) {
     86                // Internally line numbers are counted from 0.  For display, adjust
     87                // the line number so that lines are numbered from 1.
     88                if (InitialTabFlag) {
     89                    mResultStr << lineNum+1 << "\t:";
     90                }
     91                else {
     92                    mResultStr << lineNum+1 << ":";
     93                }
     94            }
     95            mResultStr.write(line_start, line_end - line_start);
     96        }
     97        mPrevious_line_end = line_end;
     98        mLineCount++;
     99    }
     100   
     101    void NonNormalizingReportMatch::finalize_match(char * buffer_end) {
     102        if (mLineCount == 0) return;  // No matches.
     103        if (mPrevious_line_end < buffer_end) {
     104            mResultStr.write(mPrevious_line_end, 1);
     105        }
     106        else {
     107            // Likely unterminated final line.
     108            char last_byte = mPrevious_line_end[-1];
     109            if (last_byte == 0x0D) {
     110                // The final CR is acceptable as a line_end.
     111                return;
     112            }
     113            // Terminate the line with an LF
     114            // (Even if we had an incomplete UTF-8 sequence.)
     115            mResultStr << "\n";
     116        }
     117    }
     118   
     119   
    83120
    84121bool matchesNeedToBeMovedToEOL() {
     
    93130}
    94131   
    95 uint64_t GrepEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) const {
     132uint64_t GrepEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) {
    96133    if (fileName == "-") {
    97134        return doGrep(STDIN_FILENO, fileIdx);
     
    102139        if (!NoMessagesFlag  && !(Mode == QuietMode)) {
    103140            if (errno == EACCES) {
    104                 resultStrs[fileIdx] << "icgrep: " << fileName << ": Permission denied.\n";
     141                resultAccums[fileIdx]->mResultStr << "icgrep: " << fileName << ": Permission denied.\n";
    105142            }
    106143            else if (errno == ENOENT) {
    107                 resultStrs[fileIdx] << "icgrep: " << fileName << ": No such file.\n";
     144                resultAccums[fileIdx]->mResultStr << "icgrep: " << fileName << ": No such file.\n";
    108145            }
    109146            else {
    110                 resultStrs[fileIdx] << "icgrep: " << fileName << ": Failed.\n";
     147                resultAccums[fileIdx]->mResultStr << "icgrep: " << fileName << ": Failed.\n";
    111148            }
    112149        }
     
    115152    if (stat(fileName.c_str(), &sb) == 0 && S_ISDIR(sb.st_mode)) {
    116153        if (!NoMessagesFlag  && !(Mode == QuietMode)) {
    117             resultStrs[fileIdx] << "icgrep: " << fileName << ": Is a directory.\n";
     154            resultAccums[fileIdx]->mResultStr << "icgrep: " << fileName << ": Is a directory.\n";
    118155        }
    119156        close(fd);
     
    125162}
    126163
    127 uint64_t GrepEngine::doGrep(const int32_t fileDescriptor, const uint32_t fileIdx) const {
    128     typedef uint64_t (*GrepFunctionType)(int32_t fileDescriptor, const uint32_t fileIdx);
     164uint64_t GrepEngine::doGrep(const int32_t fileDescriptor, const uint32_t fileIdx) {
     165    typedef uint64_t (*GrepFunctionType)(int32_t fileDescriptor, intptr_t accum_addr);
    129166    auto f = reinterpret_cast<GrepFunctionType>(mGrepDriver->getMain());
    130167   
    131     uint64_t grepResult = f(fileDescriptor, fileIdx);
     168    uint64_t grepResult = f(fileDescriptor, reinterpret_cast<intptr_t>(resultAccums[fileIdx]));
    132169    if (grepResult > 0) grepMatchFound = true;
    133     else if ((Mode == NormalMode) && !resultStrs[fileIdx].str().empty()) grepMatchFound = true;
     170    else if ((Mode == NormalMode) && !resultAccums[fileIdx]->mResultStr.str().empty()) grepMatchFound = true;
    134171   
    135172    if (Mode == CountOnly) {
    136         resultStrs[fileIdx] << linePrefix[fileIdx] << grepResult << "\n";
     173        resultAccums[fileIdx]->mResultStr << resultAccums[fileIdx]->mLinePrefix << grepResult << "\n";
    137174    }
    138175    else if (Mode == FilesWithMatch || Mode == FilesWithoutMatch ) {
    139176        size_t requiredCount = Mode == FilesWithMatch ? 1 : 0;
    140177        if (grepResult == requiredCount) {
    141             resultStrs[fileIdx] << linePrefix[fileIdx];
     178            resultAccums[fileIdx]->mResultStr << resultAccums[fileIdx]->mLinePrefix;
    142179        }
    143180    }
     
    148185}
    149186
    150 void initFileResult(std::vector<std::string> filenames){
     187void GrepEngine::initFileResult(std::vector<std::string> filenames){
    151188    grepMatchFound = false;
    152189    const int n = filenames.size();
    153     linePrefix.resize(n);
    154190    if ((n > 1) && !NoFilenameFlag) {
    155191        WithFilenameFlag = true;
     
    172208    }
    173209    inputFiles = filenames;
    174     resultStrs = new std::stringstream[n];
    175210    for (unsigned i = 0; i < inputFiles.size(); ++i) {
     211        std::string linePrefix;
    176212        if (setLinePrefix) {
    177213            if (inputFiles[i] == "-") {
    178                 linePrefix[i] = LabelFlag + fileSuffix;
     214                linePrefix = LabelFlag + fileSuffix;
    179215            }
    180216            else {
    181                 linePrefix[i] = inputFiles[i] + fileSuffix;
    182             }
    183         }
    184     }
    185 }
    186 
    187 template<typename CodeUnit>
    188 void wrapped_report_match(const size_t lineNum, size_t line_start, size_t line_end, const CodeUnit * const buffer, const size_t filesize, const size_t fileIdx) {
    189 
    190 //    errs().write_hex((size_t)buffer) << " : " << lineNum << " (" << line_start << ", " << line_end << ", " << filesize << ")\n";
    191 
    192     assert (buffer);
    193     assert (line_start <= line_end);
    194     assert (line_end <= filesize);
    195 
    196     if (WithFilenameFlag) {
    197         resultStrs[fileIdx] << linePrefix[fileIdx];
    198     }
    199     if (LineNumberFlag) {
    200         // Internally line numbers are counted from 0.  For display, adjust
    201         // the line number so that lines are numbered from 1.
    202         if (InitialTabFlag) {
    203             resultStrs[fileIdx] << lineNum+1 << "\t:";
    204         }
    205         else {
    206             resultStrs[fileIdx] << lineNum+1 << ":";
    207         }
    208     }
    209 
    210     // If the line "starts" on the LF of a CRLF, it is actually the end of the last line.
    211     if ((buffer[line_start] == 0xA) && (line_start != line_end)) {
    212         ++line_start;
    213     }
    214 
    215     if (LLVM_UNLIKELY(line_end == filesize)) {
    216         // The match position is at end-of-file.   We have a final unterminated line.
    217         resultStrs[fileIdx].write((char *)&buffer[line_start], (line_end - line_start) * sizeof(CodeUnit));
    218         if (NormalizeLineBreaksFlag) {
    219             resultStrs[fileIdx] << '\n';  // terminate it
    220         }
    221     } else {
    222         const auto end_byte = buffer[line_end];
    223         if (grep::NormalizeLineBreaksFlag) {
    224             if (LLVM_UNLIKELY(end_byte == 0x85)) {
    225                 // Line terminated with NEL, on the second byte.  Back up 1.
    226                 line_end -= 1;
    227             } else if (LLVM_UNLIKELY(end_byte > 0xD)) {
    228                 // Line terminated with PS or LS, on the third byte.  Back up 2.
    229                 line_end -= 2;
    230             }
    231             resultStrs[fileIdx].write((char *)&buffer[line_start], (line_end - line_start) * sizeof(CodeUnit));
    232             resultStrs[fileIdx] << '\n';
    233         } else {
    234             if (end_byte == 0x0D) {
    235                 // Check for line_end on first byte of CRLF; we don't want to access past the end of buffer.
    236                 if ((line_end + 1) < filesize) {
    237                     if (buffer[line_end + 1] == 0x0A) {
    238                         // Found CRLF; preserve both bytes.
    239                         ++line_end;
    240                     }
    241                 }
    242             }
    243             resultStrs[fileIdx].write((char *)&buffer[line_start], (line_end - line_start + 1) * sizeof(CodeUnit));
    244         }
    245     }
    246 }
    247 
    248 void PrintResults(){
     217                linePrefix = inputFiles[i] + fileSuffix;
     218            }
     219        }
     220        resultAccums.push_back(new NonNormalizingReportMatch(linePrefix));
     221    }
     222}
     223
     224
     225void GrepEngine::PrintResults(){
    249226   
    250227    for (unsigned i = 0; i < inputFiles.size(); ++i){
    251         std::cout << resultStrs[i].str();
     228        std::cout << resultAccums[i]->mResultStr.str();
    252229    }
    253230    exit(grepMatchFound ? MatchFoundExitCode : MatchNotFoundExitCode);
     
    339316    const unsigned encodingBits = 8;
    340317
    341     Type * const int64Ty = idb->getInt64Ty();
    342     Type * const int32Ty = idb->getInt32Ty();
    343 
    344     Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", int64Ty, idb->getInt32Ty(), int32Ty, nullptr));
     318    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getInt64Ty(), idb->getInt32Ty(), idb->getIntAddrTy(), nullptr));
    345319    mainFunc->setCallingConv(CallingConv::C);
    346320    idb->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", mainFunc, 0));
     
    349323    Value * const fileDescriptor = &*(args++);
    350324    fileDescriptor->setName("fileDescriptor");
    351     Value * fileIdx = &*(args++);
    352     fileIdx->setName("fileIdx");
     325    Value * match_accumulator = &*(args++);
     326    match_accumulator->setName("match_accumulator");
    353327
    354328    StreamSetBuffer * ByteStream = mGrepDriver->addBuffer(make_unique<SourceBuffer>(idb, idb->getStreamSetTy(1, 8)));
     
    362336   
    363337    if (grepMode == NormalMode) {
    364         kernel::Kernel * scanMatchK = mGrepDriver->addKernelInstance(make_unique<kernel::ScanMatchKernel>(idb, GrepType::Normal, encodingBits));
    365         scanMatchK->setInitialArguments({fileIdx});
     338        kernel::Kernel * scanMatchK = mGrepDriver->addKernelInstance(make_unique<kernel::ScanMatchKernel>(idb));
     339        scanMatchK->setInitialArguments({match_accumulator});
    366340        mGrepDriver->makeKernelCall(scanMatchK, {Matches, LineBreakStream, ByteStream}, {});
    367         mGrepDriver->LinkFunction(*scanMatchK, "matcher", &wrapped_report_match<uint8_t>);
     341        mGrepDriver->LinkFunction(*scanMatchK, "accumulate_match_wrapper", &accumulate_match_wrapper);
     342        mGrepDriver->LinkFunction(*scanMatchK, "finalize_match_wrapper", &finalize_match_wrapper);
     343
     344       
    368345        mGrepDriver->generatePipelineIR();
    369346        mGrepDriver->deallocateBuffers();
     
    376353        idb->setKernel(matchCountK);
    377354        Value * matchedLineCount = idb->getAccumulator("countResult");
    378         matchedLineCount = idb->CreateZExt(matchedLineCount, int64Ty);
     355        matchedLineCount = idb->CreateZExt(matchedLineCount, idb->getInt64Ty());
    379356        mGrepDriver->deallocateBuffers();
    380357        idb->CreateRet(matchedLineCount);
     
    383360}
    384361
    385 GrepEngine::GrepEngine()
    386 : mGrepDriver(nullptr) {
    387 
    388 }
    389362
    390363GrepEngine::~GrepEngine() {
Note: See TracChangeset for help on using the changeset viewer.