Changeset 5964


Ignore:
Timestamp:
Apr 11, 2018, 10:27:58 AM (13 months ago)
Author:
cameron
Message:

Restructuring in preparation for command-line file include/exclude GLOBs

Location:
icGREP/icgrep-devel/icgrep
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.cpp

    r5963 r5964  
    136136    mInvertMatches(false),
    137137    mMaxCount(0),
     138    mGrepStdIn(false),
    138139    mGrepDriver(nullptr),
    139140    mNextFileToGrep(0),
     
    180181
    181182   
    182 void GrepEngine::initFileResult(std::vector<std::string> & filenames) {
    183     const unsigned n = filenames.size();
     183void GrepEngine::initFileResult(std::vector<boost::filesystem::path> & paths) {
     184    const unsigned n = paths.size();
    184185    mResultStrs.resize(n);
    185186    mFileStatus.resize(n, FileStatus::Pending);
    186     inputFiles = filenames;
     187    inputPaths = paths;
    187188}
    188189
     
    561562//  differently based on the engine type.
    562563
    563 uint64_t GrepEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) {
     564uint64_t GrepEngine::doGrep(const std::string & fileName, std::ostringstream & strm) {
    564565    typedef uint64_t (*GrepFunctionType)(bool useMMap, int32_t fileDescriptor);
    565566    using namespace boost::filesystem;
     
    571572    auto f = reinterpret_cast<GrepFunctionType>(mGrepDriver->getMain());
    572573
    573     int32_t fileDescriptor = openFile(fileName, mResultStrs[fileIdx]);
     574    int32_t fileDescriptor = openFile(fileName, strm);
    574575    if (fileDescriptor == -1) return 0;
    575576
     
    579580}
    580581
    581 uint64_t CountOnlyEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) {
    582     uint64_t grepResult = GrepEngine::doGrep(fileName, fileIdx);
    583     if (mShowFileNames) mResultStrs[fileIdx] << linePrefix(fileName);
    584     mResultStrs[fileIdx] << grepResult << "\n";
     582uint64_t CountOnlyEngine::doGrep(const std::string & fileName, std::ostringstream & strm) {
     583    uint64_t grepResult = GrepEngine::doGrep(fileName, strm);
     584    if (mShowFileNames) strm << linePrefix(fileName);
     585    strm << grepResult << "\n";
    585586    return grepResult;
    586587}
     
    596597}
    597598
    598 uint64_t MatchOnlyEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) {
    599     uint64_t grepResult = GrepEngine::doGrep(fileName, fileIdx);
     599uint64_t MatchOnlyEngine::doGrep(const std::string & fileName, std::ostringstream & strm) {
     600    uint64_t grepResult = GrepEngine::doGrep(fileName, strm);
    600601    if (grepResult == mRequiredCount) {
    601        mResultStrs[fileIdx] << linePrefix(fileName);
     602       strm << linePrefix(fileName);
    602603    }
    603604    return grepResult;
    604605}
    605606
    606 uint64_t EmitMatchesEngine::doGrep(const std::string & fileName, const uint32_t fileIdx) {
     607uint64_t EmitMatchesEngine::doGrep(const std::string & fileName, std::ostringstream & strm) {
    607608    typedef uint64_t (*GrepFunctionType)(bool useMMap, int32_t fileDescriptor, intptr_t accum_addr);
    608609    using namespace boost::filesystem;
     
    612613    if (!is_regular_file(p)) useMMap = false;
    613614    auto f = reinterpret_cast<GrepFunctionType>(mGrepDriver->getMain());
    614     int32_t fileDescriptor = openFile(fileName, mResultStrs[fileIdx]);
     615    int32_t fileDescriptor = openFile(fileName, strm);
    615616    if (fileDescriptor == -1) return 0;
    616     EmitMatch accum(linePrefix(fileName), mShowLineNumbers, mInitialTab, mResultStrs[fileIdx]);
     617    EmitMatch accum(linePrefix(fileName), mShowLineNumbers, mInitialTab, strm);
    617618    f(useMMap, fileDescriptor, reinterpret_cast<intptr_t>(&accum));
    618619    close(fileDescriptor);
     
    662663
    663664bool GrepEngine::searchAllFiles() {
    664     const unsigned numOfThreads = std::min(static_cast<unsigned>(Threads), static_cast<unsigned>(inputFiles.size()));
     665    const unsigned numOfThreads = std::min(static_cast<unsigned>(Threads), static_cast<unsigned>(inputPaths.size()));
    665666    std::vector<pthread_t> threads(numOfThreads);
    666667
     
    688689
    689690    unsigned fileIdx = mNextFileToGrep++;
    690     while (fileIdx < inputFiles.size()) {
     691    while (fileIdx < inputPaths.size()) {
    691692        if (codegen::DebugOptionIsSet(codegen::TraceCounts)) {
    692             errs() << "Tracing " << inputFiles[fileIdx] << "\n";
    693         }
    694         const auto grepResult = doGrep(inputFiles[fileIdx], fileIdx);
     693            errs() << "Tracing " << inputPaths[fileIdx].string() << "\n";
     694        }
     695        const auto grepResult = doGrep(inputPaths[fileIdx].string(), mResultStrs[fileIdx]);
    695696        mFileStatus[fileIdx] = FileStatus::GrepComplete;
    696697        if (grepResult > 0) {
     
    707708
    708709    unsigned printIdx = mNextFileToPrint++;
    709     while (printIdx < inputFiles.size()) {
     710    while (printIdx < inputPaths.size()) {
    710711        const bool readyToPrint = ((printIdx == 0) || (mFileStatus[printIdx - 1] == FileStatus::PrintComplete)) && (mFileStatus[printIdx] == FileStatus::GrepComplete);
    711712        if (readyToPrint) {
     
    727728        // Always perform one final cache cleanup step.
    728729        mGrepDriver->performIncrementalCacheCleanupStep();
     730        if (mGrepStdIn) {
     731            std::ostringstream s;
     732            const auto grepResult = doGrep("-", s);
     733            llvm::outs() << s.str();
     734            if (grepResult) grepMatchFound = true;
     735        }
    729736    }
    730737    return nullptr;
     
    736743    mGrepRecordBreak(GrepRecordBreakKind::LF),
    737744    mCaseInsensitive(false),
    738     mGrepDriver(nullptr),
    739     grepMatchFound(false) {}
     745    mGrepDriver(nullptr) {}
    740746   
    741747InternalSearchEngine::~InternalSearchEngine() {
     
    748754    Module * M = idb->getModule();
    749755   
    750     const unsigned encodingBits = 8;
    751756    const unsigned segmentSize = codegen::BufferSegments * codegen::SegmentSize * codegen::ThreadNum;
    752757   
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.h

    r5963 r5964  
    1515#include <atomic>
    1616#include <util/aligned_allocator.h>
     17#include <boost/filesystem.hpp>
    1718
    1819namespace re { class CC; }
     
    5354
    5455    void setMaxCount(int m) {mMaxCount = m;}
     56    void setGrepStdIn() {mGrepStdIn = true;}
    5557    void setInvertMatches() {mInvertMatches = true;}
    5658    void setCaseInsensitive()  {mCaseInsensitive = true;}
     
    5961
    6062    void setRecordBreak(GrepRecordBreakKind b);
    61     void initFileResult(std::vector<std::string> & filenames);
     63    void initFileResult(std::vector<boost::filesystem::path> & filenames);
    6264    void initREs(std::vector<re::RE *> & REs);
    6365    virtual void grepCodeGen();
     
    6870    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(parabix::StreamSetBuffer * ByteStream);
    6971
    70     virtual uint64_t doGrep(const std::string & fileName, const uint32_t fileIdx);
     72    virtual uint64_t doGrep(const std::string & fileName, std::ostringstream & strm);
    7173    int32_t openFile(const std::string & fileName, std::ostringstream & msgstrm);
    7274
     
    8587    bool mInvertMatches;
    8688    int mMaxCount;
     89    bool mGrepStdIn;
    8790   
    8891    Driver * mGrepDriver;
     
    9093    std::atomic<unsigned> mNextFileToGrep;
    9194    std::atomic<unsigned> mNextFileToPrint;
    92     std::vector<std::string> inputFiles;
     95    std::vector<boost::filesystem::path> inputPaths;
    9396    std::vector<std::ostringstream> mResultStrs;
    9497    std::vector<FileStatus> mFileStatus;
     
    135138    void grepCodeGen() override;
    136139private:
    137     uint64_t doGrep(const std::string & fileName, const uint32_t fileIdx) override;
     140    uint64_t doGrep(const std::string & fileName, std::ostringstream & strm) override;
    138141};
    139142
     
    142145    CountOnlyEngine();
    143146private:
    144     uint64_t doGrep(const std::string & fileName, const uint32_t fileIdx) override;
     147    uint64_t doGrep(const std::string & fileName, std::ostringstream & strm) override;
    145148};
    146149
     
    149152    MatchOnlyEngine(bool showFilesWithoutMatch, bool useNullSeparators);
    150153private:
    151     uint64_t doGrep(const std::string & fileName, const uint32_t fileIdx) override;
     154    uint64_t doGrep(const std::string & fileName, std::ostringstream & strm) override;
    152155    unsigned mRequiredCount;
    153156};
     
    177180
    178181    Driver * mGrepDriver;
    179     bool grepMatchFound;
    180182};
    181183   
  • icGREP/icgrep-devel/icgrep/grep_interface.cpp

    r5945 r5964  
    238238        llvm::report_fatal_error("Sorry, -U is not yet supported.\n");
    239239    }
    240     if (argv::ExcludeFlag!="") {
    241         llvm::report_fatal_error("Sorry, -exclude is not yet supported.\n");
    242     }
    243     if (argv::ExcludeFromFlag!="") {
    244         llvm::report_fatal_error("Sorry, -exclude-from is not yet supported.\n");
    245     }
    246     if (argv::ExcludeDirFlag!="") {
    247         llvm::report_fatal_error("Sorry, -exclude-dir is not yet supported.\n");
    248     }
    249     if (argv::IncludeFlag!="") {
    250         llvm::report_fatal_error("Sorry, -include is not yet supported.\n");
    251     }   
    252240    if (ByteOffsetFlag) {
    253241        llvm::report_fatal_error("Sorry, -b is not yet supported.\n");
  • icGREP/icgrep-devel/icgrep/icgrep.cpp

    r5945 r5964  
    2424#include <re/re_toolchain.h>
    2525#include <pablo/pablo_toolchain.h>
     26#include <boost/filesystem.hpp>
    2627#include <util/file_select.h>
    2728#include <sys/stat.h>
     
    105106}
    106107
     108namespace fs = boost::filesystem;
    107109
    108110int main(int argc, char *argv[]) {
     
    112114    auto REs = readExpressions();
    113115
    114     std::vector<std::string> allFiles = argv::getFullFileList(inputFiles);
    115     if (allFiles.empty()) {
    116         allFiles = { "-" };
     116    std::vector<fs::path> allFiles = argv::getFullFileList(inputFiles);
     117    if (inputFiles.empty()) {
     118        argv::UseStdIn = true;
    117119    }
    118120    else if ((allFiles.size() > 1) && !argv::NoFilenameFlag) {
     
    153155    }
    154156    grepEngine->setStdinLabel(argv::LabelFlag);
     157    if (argv::UseStdIn) grepEngine->setGrepStdIn();
    155158    if (argv::NoMessagesFlag) grepEngine->suppressFileMessages();
    156159    if (argv::MmapFlag) grepEngine->setPreferMMap();
  • icGREP/icgrep-devel/icgrep/util/file_select.cpp

    r5945 r5964  
    6464static cl::alias DirectoriesAlias("directories", cl::desc("Alias for -d"), cl::aliasopt(DirectoriesOption));
    6565
    66    
     66bool UseStdIn;
     67
    6768re::RE * getFileExcludePattern() {
    6869    std::vector<re::RE *> patterns;
     
    131132}
    132133
    133 void getSubdirectoryFiles(fs::path dirpath, std::vector<std::string> & collectedFiles) {
     134    void getSubdirectoryFiles(fs::path dirpath, std::vector<fs::path> & collectedFiles) {
    134135    boost::system::error_code errc;
    135136    fs::directory_iterator di(dirpath, errc);
     
    137138    if (errc) {
    138139        // If we cannot enter the directory, keep it in the list of files.
    139         collectedFiles.push_back(dirpath.string());
     140        collectedFiles.push_back(dirpath);
    140141        return;
    141142    }
     
    152153        } else {
    153154            if (!skip_path(e)) {
    154                 collectedFiles.push_back(e.string());
     155                collectedFiles.push_back(e);
    155156            }
    156157        }
    157158        di.increment(errc);
    158159        if (errc) {
    159             collectedFiles.push_back(e.string());
     160            collectedFiles.push_back(e);
    160161        }
    161162    }
    162163}
    163164
    164 std::vector<std::string> getFullFileList(cl::list<std::string> & inputFiles) {
    165     std::vector<std::string> expanded_paths;
     165std::vector<fs::path> getFullFileList(cl::list<std::string> & inputFiles) {
     166    std::vector<fs::path> expanded_paths;
    166167    boost::system::error_code errc;
    167168    for (const std::string & f : inputFiles) {
    168         //        if (f == "-") {
    169         //            continue;
    170         //        }
     169        if (f == "-") {
     170            argv::UseStdIn = true;
     171            continue;
     172        }
    171173        fs::path p(f);
    172174        if (skip_path(p)) {
     
    178180            }
    179181        } else {
    180             expanded_paths.push_back(p.string());
     182            expanded_paths.push_back(p);
    181183        }
    182184    }
  • icGREP/icgrep-devel/icgrep/util/file_select.h

    r5944 r5964  
    1414#include <vector>
    1515#include <llvm/Support/CommandLine.h>
     16#include <boost/filesystem.hpp>
     17
    1618namespace re {class RE;}
    1719
     
    4648bool includeIsDefault();
    4749
    48 std::vector<std::string> getFullFileList(llvm::cl::list<std::string> & inputFiles);
     50extern bool UseStdIn;
     51   
     52std::vector<boost::filesystem::path> getFullFileList(llvm::cl::list<std::string> & inputFiles);
    4953}
    5054#endif
  • icGREP/icgrep-devel/icgrep/wc.cpp

    r5946 r5964  
    2727#include <fcntl.h>
    2828#include <util/file_select.h>
     29#include <boost/filesystem.hpp>
     30namespace fs = boost::filesystem;
    2931
    3032using namespace llvm;
     
    3436static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(wcFlags));
    3537
    36 std::vector<std::string> allFiles;
     38std::vector<fs::path> allFiles;
    3739
    3840enum CountOptions {
     
    210212
    211213void wc(WordCountFunctionType fn_ptr, const int64_t fileIdx) {
    212     std::string fileName = allFiles[fileIdx];
     214    std::string fileName = allFiles[fileIdx].string();
    213215    struct stat sb;
    214216    const int fd = open(fileName.c_str(), O_RDONLY);
     
    300302            std::cout << byteCount[i];
    301303        }
    302         std::cout << " " << allFiles[i] << std::endl;
     304        std::cout << " " << allFiles[i].string() << std::endl;
    303305    }
    304306    if (inputFiles.size() > 1) {
Note: See TracChangeset for help on using the changeset viewer.