Changeset 5944 for icGREP/icgrep-devel


Ignore:
Timestamp:
Apr 5, 2018, 2:39:58 PM (15 months ago)
Author:
cameron
Message:

Common command line file selection utility for icgrep, wc ...

Location:
icGREP/icgrep-devel/icgrep
Files:
2 added
6 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/CMakeLists.txt

    r5938 r5944  
    9191ENDIF()
    9292
     93SET (RE_PARSERS_SRC re/parsers/PCRE_parser.cpp re/parsers/ERE_parser.cpp re/parsers/BRE_parser.cpp re/parsers/Prosite_parser.cpp re/parsers/fixed_string_parser.cpp re/parsers/GLOB_parser.cpp re/parsers/parser.cpp)
     94
    9395add_library(CodeGen ${TOOLCHAIN_SRC} ${DRIVER_SRC} ${OBJECT_CACHE_SRC} ${KERNEL_SRC} ${IDISA_SRC})
    9496add_library(PabloADT ${PABLO_SRC})
     
    9698add_library(CCADT re/re_re.cpp re/re_cc.cpp cc/cc_compiler.cpp cc/alphabet.cpp cc/multiplex_CCs.cpp UCD/unicode_set.cpp)
    9799# RegExpADT is the core library for representing, parsing and printing regular expressions
    98 add_library(RegExpADT re/re_rep.cpp re/re_diff.cpp re/re_intersect.cpp re/re_range.cpp re/re_assertion.cpp re/printer_re.cpp re/parsers/PCRE_parser.cpp re/parsers/ERE_parser.cpp re/parsers/BRE_parser.cpp re/parsers/Prosite_parser.cpp re/parsers/fixed_string_parser.cpp re/parsers/GLOB_parser.cpp)
    99 add_library(RegExpCompiler re/casing.cpp re/to_utf8.cpp re/parsers/parser.cpp re/re_memoizer.cpp re/re_nullable.cpp re/re_simplifier.cpp re/re_star_normal.cpp re/re_minimizer.cpp re/re_local.cpp re/re_compiler.cpp re/re_analysis.cpp re/re_toolchain.cpp re/re_name_resolve.cpp re/re_name_gather.cpp re/collect_ccs.cpp re/re_multiplex.cpp re/re_utility.cpp re/grapheme_clusters.cpp re/exclude_CC.cpp re/replaceCC.cpp)
     100add_library(RegExpADT re/re_rep.cpp re/re_diff.cpp re/re_intersect.cpp re/re_range.cpp re/re_assertion.cpp re/printer_re.cpp re/re_utility.cpp)
     101add_library(RegExpCompiler ${RE_PARSERS_SRC} re/casing.cpp re/to_utf8.cpp re/re_memoizer.cpp re/re_nullable.cpp re/re_simplifier.cpp re/re_star_normal.cpp re/re_minimizer.cpp re/re_local.cpp re/re_compiler.cpp re/re_analysis.cpp re/re_toolchain.cpp re/re_name_resolve.cpp re/re_name_gather.cpp re/collect_ccs.cpp re/re_multiplex.cpp re/re_utility.cpp re/grapheme_clusters.cpp re/exclude_CC.cpp re/replaceCC.cpp)
    100102add_library(UCDlib UCD/CaseFolding.cpp utf8_encoder.cpp utf16_encoder.cpp UCD/ucd_compiler.cpp UCD/PropertyObjects.cpp UCD/resolve_properties.cpp)
    101103add_library(GrepEngine  ${GREP_CORE_SRC} grep/grep_engine.cpp kernels/cc_kernel.cpp kernels/cc_scan_kernel.cpp kernels/charclasses.cpp kernels/streams_merge.cpp kernels/until_n.cpp kernels/UCD_property_kernel.cpp kernels/grapheme_kernel.cpp)
     
    115117target_link_libraries (GrepEngine RegExpCompiler CodeGen)
    116118
    117 add_executable(icgrep icgrep.cpp grep_interface.cpp)
     119add_executable(icgrep icgrep.cpp util/file_select.cpp grep_interface.cpp)
    118120add_executable(u8u16 u8u16.cpp)
    119121add_executable(base64 base64.cpp kernels/radix64.cpp)
    120 add_executable(wc wc.cpp)
     122add_executable(wc wc.cpp util/file_select.cpp)
    121123add_executable(editd editd/editd.cpp editd/pattern_compiler.cpp editd/editdscan_kernel.cpp editd/editd_gpu_kernel.cpp editd/editd_cpu_kernel.cpp kernels/streams_merge.cpp kernels/cc_kernel.cpp)
    122124add_executable(lz4d lz4d.cpp lz4FrameDecoder.cpp kernels/cc_kernel.cpp kernels/lz4/lz4_index_decoder.cpp kernels/lz4/lz4_bytestream_decoder.cpp)
     
    133135target_link_libraries (u8u16 PabloADT CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
    134136target_link_libraries (base64 PabloADT CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
    135 target_link_libraries (wc PabloADT CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
     137target_link_libraries (wc GrepEngine UCDlib PabloADT RegExpCompiler CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
    136138target_link_libraries (editd PabloADT CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
    137139target_link_libraries (lz4d PabloADT CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.cpp

    r5941 r5944  
    4646#include <llvm/Support/raw_ostream.h>
    4747#include <util/aligned_allocator.h>
     48#include <util/file_select.h>
    4849#include <sys/stat.h>
    4950#include <fcntl.h>
     
    579580    using namespace boost::filesystem;
    580581    path p(fileName);
    581     bool useMMap = grep::MmapFlag;
     582    bool useMMap = argv::MmapFlag;
    582583    if (p == "-") useMMap = false;
    583584    if (!is_regular_file(p)) useMMap = false;
     
    621622    using namespace boost::filesystem;
    622623    path p(fileName);
    623     bool useMMap = grep::MmapFlag;
     624    bool useMMap = argv::MmapFlag;
    624625    if (p == "-") useMMap = false;
    625626    if (!is_regular_file(p)) useMMap = false;
  • icGREP/icgrep-devel/icgrep/grep_interface.cpp

    r5940 r5944  
    1010#include <llvm/Support/Signals.h>
    1111#include <llvm/Support/raw_ostream.h>
     12#include <util/file_select.h>
    1213#include <toolchain/toolchain.h>
    1314#include <re/parsers/parser.h>
     
    7475static cl::OptionCategory Input_Options("B. Input Options", "These options control the input.");
    7576
    76 bool RecursiveFlag;
    77 static cl::opt<bool, true> RecursiveOption("r", cl::location(RecursiveFlag), cl::desc("Recursively process files within directories, (but follow only top-level symlinks unless -R)."), cl::cat(Input_Options), cl::Grouping);
    78 static cl::alias RecursiveAlias("recursive", cl::desc("Alias for -r"), cl::aliasopt(RecursiveOption));
    79 
    80 bool DereferenceRecursiveFlag;
    81 static cl::opt<bool, true> DereferenceRecursiveOption("R", cl::location(DereferenceRecursiveFlag), cl::desc("Recursively process files within directories, following symlinks at all levels."), cl::cat(Input_Options), cl::Grouping);
    82 static cl::alias DereferenceRecursiveAlias("dereference-recursive", cl::desc("Alias for -R"), cl::aliasopt(DereferenceRecursiveOption));
    83 
    8477bool TextFlag;
    8578static cl::opt<bool, true> TextOption("a", cl::location(TextFlag), cl::desc("Treat each input file as text, even if it is a binary file."), cl::cat(Input_Options), cl::Grouping);
     
    10093bool UnicodeLinesFlag;
    10194static cl::opt<bool, true> UnicodeLinesOption("Unicode-lines", cl::location(UnicodeLinesFlag), cl::desc("Enable Unicode line breaks (LF/VT/FF/CR/NEL/LS/PS/CRLF)"), cl::cat(Input_Options));
    102 
    103 bool MmapFlag;
    104 static cl::opt<bool, true> MmapOption("mmap", cl::location(MmapFlag),  cl::init(1), cl::desc("Use mmap for file input (default)."), cl::cat(Input_Options));
    105 
    106 std::string ExcludeFlag;
    107 static cl::opt<std::string, true> ExcludeOption("exclude", cl::location(ExcludeFlag), cl::desc("Exclude files matching the given filename GLOB pattern."), cl::cat(Input_Options));
    108 
    109 std::string ExcludeFromFlag;
    110 static cl::opt<std::string, true> ExcludeFromOption("exclude-from", cl::location(ExcludeFromFlag), cl::desc("Exclude files matching filename GLOB patterns from the given file."), cl::cat(Input_Options));
    111 
    112 std::string ExcludeDirFlag;
    113 static cl::opt<std::string, true> ExcludeDirOption("exclude-dir", cl::location(ExcludeDirFlag), cl::desc("Exclude directories matching the given pattern."), cl::cat(Input_Options));
    114 
    115 std::string IncludeFlag;
    116 static cl::opt<std::string, true> IncludeOption("include", cl::location(IncludeFlag), cl::desc("Include only files matching the given filename GLOB pattern."), cl::cat(Input_Options));
    117 
    118 DevDirAction DevicesFlag;
    119 static cl::opt<DevDirAction, true> DevicesOption("D", cl::desc("Processing mode for devices:"),
    120                                                  cl::values(clEnumValN(Read, "read", "Treat devices as files to be searched."),
    121                                                             clEnumValN(Skip, "skip", "Silently skip devices.")
    122                                                             CL_ENUM_VAL_SENTINEL), cl::cat(Input_Options), cl::location(DevicesFlag), cl::init(Read));
    123 static cl::alias DevicesAlias("devices", cl::desc("Alias for -D"), cl::aliasopt(DevicesOption));
    124 
    125 DevDirAction DirectoriesFlag;
    126 static cl::opt<DevDirAction, true> DirectoriesOption("d", cl::desc("Processing mode for directories:"),
    127                                                      cl::values(clEnumValN(Read, "read", "Print an error message for any listed directories."),
    128                                                                 clEnumValN(Skip, "skip", "Silently skip directories."),
    129                                                                 clEnumValN(Recurse, "recurse", "Recursive process directories, equivalent to -r.")
    130                                                                 CL_ENUM_VAL_SENTINEL), cl::cat(Input_Options), cl::location(DirectoriesFlag), cl::init(Read));
    131 static cl::alias DirectoriesAlias("directories", cl::desc("Alias for -d"), cl::aliasopt(DirectoriesOption));
    13295
    13396BinaryFilesMode BinaryFilesFlag;
     
    139102   
    140103
    141    
    142 re::RE * getFileExcludePattern() {
    143     std::vector<re::RE *> patterns;
    144     if (grep::ExcludeFlag != "") {
    145         re::RE * glob = re::RE_Parser::parse(grep::ExcludeFlag, re::DEFAULT_MODE, re::RE_Syntax::FileGLOB);
    146         patterns.push_back(glob);
    147     }
    148    
    149     if (grep::ExcludeFromFlag != "") {
    150         std::ifstream globFile(grep::ExcludeFromFlag.c_str());
    151         std::string r;
    152         if (globFile.is_open()) {
    153             while (std::getline(globFile, r)) {
    154                 re::RE * glob = re::RE_Parser::parse(r, re::DEFAULT_MODE, re::RE_Syntax::FileGLOB);
    155                 patterns.push_back(glob);
    156             }
    157             globFile.close();
    158         }
    159     }
    160     if (patterns.empty()) return nullptr;
    161     return re::makeAlt(patterns.begin(), patterns.end());
    162 }
    163 
    164 re::RE * getDirectoryExcludePattern() {
    165     if (grep::ExcludeDirFlag != "") {
    166         return re::RE_Parser::parse(grep::ExcludeDirFlag, re::DEFAULT_MODE, re::RE_Syntax::FileGLOB);
    167     }
    168     return nullptr;
    169 }
    170 
    171 re::RE * getFileIncludePattern() {
    172     if (grep::IncludeFlag != "") {
    173         return re::RE_Parser::parse(grep::IncludeFlag, re::DEFAULT_MODE, re::RE_Syntax::FileGLOB);
    174     }
    175     return nullptr;
    176 }
    177 
    178 // Include is the default unless a -include= option exists and is prior to any -exclude
    179 // or -exclude-dir option.
    180 bool includeIsDefault() {
    181     if (IncludeFlag == "") return true;
    182     if ((ExcludeFlag != "") && (ExcludeOption.getPosition() < IncludeOption.getPosition())) return true;
    183     if ((ExcludeDirFlag != "") && (ExcludeDirOption.getPosition() < IncludeOption.getPosition())) return true;
    184     return false;
    185 }
    186104   
    187105/*
     
    301219    llvm::install_fatal_error_handler(&icgrep_error_handler);
    302220    codegen::ParseCommandLineOptions(argc, argv, {&RE_Options, &Input_Options, &Output_Options, re::re_toolchain_flags(), pablo::pablo_toolchain_flags(), codegen::codegen_flags()});
    303     if (RecursiveFlag || DereferenceRecursiveFlag) {
    304         DirectoriesFlag = Recurse;
     221    if (argv::RecursiveFlag || argv::DereferenceRecursiveFlag) {
     222        argv::DirectoriesFlag = argv::Recurse;
    305223    }
    306224   
     
    320238        llvm::report_fatal_error("Sorry, -U is not yet supported.\n");
    321239    }
    322     if (ExcludeFlag!="") {
     240    if (argv::ExcludeFlag!="") {
    323241        llvm::report_fatal_error("Sorry, -exclude is not yet supported.\n");
    324242    }
    325     if (ExcludeFromFlag!="") {
     243    if (argv::ExcludeFromFlag!="") {
    326244        llvm::report_fatal_error("Sorry, -exclude-from is not yet supported.\n");
    327245    }
    328     if (ExcludeDirFlag!="") {
     246    if (argv::ExcludeDirFlag!="") {
    329247        llvm::report_fatal_error("Sorry, -exclude-dir is not yet supported.\n");
    330248    }
    331     if (IncludeFlag!="") {
     249    if (argv::IncludeFlag!="") {
    332250        llvm::report_fatal_error("Sorry, -include is not yet supported.\n");
    333251    }   
  • icGREP/icgrep-devel/icgrep/grep_interface.h

    r5940 r5944  
    4141
    4242/*
    43  *  B.  Grep input sources and interpretation.
     43 *  B.  Grep input options.
    4444 */
    4545
    4646// Use DirectoriesFlag==Recurse to test for recursive mode.
    47 //extern bool RecursiveFlag;
    48 extern bool DereferenceRecursiveFlag; // -R
    4947extern bool TextFlag; // -a
    5048extern bool BinaryFlag; // -U
    5149extern bool NullDataFlag; // -z
    5250extern bool UnicodeLinesFlag; // -Unicode-lines
    53 
    54 extern bool MmapFlag; // -mmap
    55 
    56 extern std::string ExcludeFlag; // -exclude
    57 extern std::string ExcludeFromFlag; // -exclude-from
    58 extern std::string ExcludeDirFlag; // -exclude-dir
    59 extern std::string IncludeFlag; // -include
    60 
    61 // File exclude pattern parsed from ExcludeFlag and ExcludeFromFlag
    62 re::RE * getFileExcludePattern();
    63 
    64 // File exclude pattern parsed from ExcludeDirFlag.
    65 re::RE * getDirectoryExcludePattern();
    66 
    67 // File include pattern parsed from InludeFlag
    68 re::RE * getFileIncludePattern();
    69 
    70 // Determine whether include is the default for files that do not match any pattern.
    71 bool includeIsDefault();
    7251
    7352
     
    8665extern ColoringType ColorFlag;
    8766
    88 enum DevDirAction {Read, Skip, Recurse};
    89 extern DevDirAction DevicesFlag;
    90 extern DevDirAction DirectoriesFlag;
    91    
    9267enum BinaryFilesMode {Binary, WithoutMatch, Text};
    9368extern BinaryFilesMode BinaryFilesFlag;
  • icGREP/icgrep-devel/icgrep/icgrep.cpp

    r5938 r5944  
    2424#include <re/re_toolchain.h>
    2525#include <pablo/pablo_toolchain.h>
    26 #include <boost/filesystem.hpp>
     26#include <util/file_select.h>
    2727#include <sys/stat.h>
    2828#include <fcntl.h>
     
    106106
    107107
    108 // This is a stub, to be expanded later.
    109 bool excludeDirectory(boost::filesystem::path dirpath) { return dirpath.filename() == ".svn";}
    110 
    111 // Determine whether to skip a path based on -D skip or -d skip settings.
    112 bool skip_path(boost::filesystem::path p) {
    113     using namespace boost::filesystem;
    114     switch (status(p).type()) {
    115         case directory_file: return grep::DirectoriesFlag == grep::Skip;
    116         case block_file:
    117         case character_file:
    118         case fifo_file:
    119         case socket_file:
    120             return grep::DevicesFlag == grep::Skip;
    121         default:
    122             return false;
    123     }
    124 }
    125 
    126 std::vector<std::string> getFullFileList(cl::list<std::string> & inputFiles) {
    127     using namespace boost::filesystem;
    128     symlink_option follow_symlink = grep::DereferenceRecursiveFlag ? symlink_option::recurse : symlink_option::none;
    129     std::vector<std::string> expanded_paths;
    130     boost::system::error_code errc;
    131     for (const std::string & f : inputFiles) {
    132         //        if (f == "-") {
    133         //            continue;
    134         //        }
    135         path p(f);
    136         if (skip_path(p)) {
    137             continue;
    138         }
    139         if (LLVM_UNLIKELY((grep::DirectoriesFlag == grep::Recurse) && is_directory(p))) {
    140             if (!excludeDirectory(p)) {
    141                 recursive_directory_iterator di(p, follow_symlink, errc), end;
    142                 if (errc) {
    143                     // If we cannot enter the directory, keep it in the list of files.
    144                     expanded_paths.push_back(f);
    145                     continue;
    146                 }
    147                 while (di != end) {
    148                     auto & e = di->path();
    149                     if (is_directory(e)) {
    150                         if (LLVM_UNLIKELY(excludeDirectory(e))) {
    151                             di.no_push();
    152                         }
    153                     } else {
    154                         if (!skip_path(e)) expanded_paths.push_back(e.string());
    155                     }
    156                     di.increment(errc);
    157                     if (errc) {
    158                         expanded_paths.push_back(e.string());
    159                     }
    160                 }
    161             }
    162         } else {
    163             expanded_paths.push_back(p.string());
    164         }
    165     }
    166     return expanded_paths;
    167 }
    168 
    169108int main(int argc, char *argv[]) {
    170109
     
    173112    auto REs = readExpressions();
    174113
    175     std::vector<std::string> allFiles = getFullFileList(inputFiles);
     114    std::vector<std::string> allFiles = argv::getFullFileList(inputFiles);
    176115    if (allFiles.empty()) {
    177116        allFiles = { "-" };
  • icGREP/icgrep-devel/icgrep/wc.cpp

    r5925 r5944  
    2525#include <toolchain/cpudriver.h>
    2626#include <fcntl.h>
    27 #include <boost/filesystem.hpp>
     27#include <util/file_select.h>
    2828
    2929using namespace llvm;
     
    4949
    5050static int defaultDisplayColumnWidth = 7;  // default field width
    51 
    52 
    53 bool RecursiveFlag;
    54 static cl::opt<bool, true> RecursiveOption("r", cl::location(RecursiveFlag), cl::desc("Recursively process files within directories, (but follow only top-level symlinks unless -R)."), cl::cat(wcFlags), cl::Grouping);
    55 static cl::alias RecursiveAlias("recursive", cl::desc("Alias for -r"), cl::aliasopt(RecursiveOption));
    56 
    57 bool DereferenceRecursiveFlag;
    58 static cl::opt<bool, true> DereferenceRecursiveOption("R", cl::location(DereferenceRecursiveFlag), cl::desc("Recursively process files within directories, following symlinks at all levels."), cl::cat(wcFlags), cl::Grouping);
    59 static cl::alias DereferenceRecursiveAlias("dereference-recursive", cl::desc("Alias for -R"), cl::aliasopt(DereferenceRecursiveOption));
    60 
    61 
    62 // This is a stub, to be expanded later.
    63 bool excludeDirectory(boost::filesystem::path dirpath) { return dirpath.filename() == ".svn";}
    64 
    65 // Determine whether to skip a path based on -D skip or -d skip settings.
    66 bool skip_path(boost::filesystem::path p) {
    67     using namespace boost::filesystem;
    68     switch (status(p).type()) {
    69         case directory_file: return !(RecursiveFlag|DereferenceRecursiveFlag);
    70         case block_file:
    71         case character_file:
    72         case fifo_file:
    73         case socket_file:
    74             return true;
    75         default:
    76             return false;
    77     }
    78 }
    79 
    80 
    81 std::vector<std::string> getFullFileList(cl::list<std::string> & inputFiles) {
    82     using namespace boost::filesystem;
    83     symlink_option follow_symlink = DereferenceRecursiveFlag ? symlink_option::recurse : symlink_option::none;
    84     std::vector<std::string> expanded_paths;
    85     boost::system::error_code errc;
    86     for (const std::string & f : inputFiles) {
    87         //        if (f == "-") {
    88         //            continue;
    89         //        }
    90         path p(f);
    91         if (skip_path(p)) {
    92             continue;
    93         }
    94         if (LLVM_UNLIKELY((RecursiveFlag || DereferenceRecursiveFlag) && is_directory(p))) {
    95             if (!excludeDirectory(p)) {
    96                 recursive_directory_iterator di(p, follow_symlink, errc), end;
    97                 if (errc) {
    98                     // If we cannot enter the directory, keep it in the list of files.
    99                     expanded_paths.push_back(f);
    100                     continue;
    101                 }
    102                 while (di != end) {
    103                     auto & e = di->path();
    104                     if (is_directory(e)) {
    105                         if (LLVM_UNLIKELY(excludeDirectory(e))) {
    106                             di.no_push();
    107                         }
    108                     } else {
    109                         if (!skip_path(e)) expanded_paths.push_back(e.string());
    110                     }
    111                     di.increment(errc);
    112                     if (errc) {
    113                         expanded_paths.push_back(e.string());
    114                     }
    115                 }
    116             }
    117         } else {
    118             expanded_paths.push_back(p.string());
    119         }
    120     }
    121     return expanded_paths;
    122 }
    123 
    124 
    12551
    12652
     
    320246    auto wordCountFunctionPtr = reinterpret_cast<WordCountFunctionType>(pxDriver.getMain());
    321247
    322     allFiles = getFullFileList(inputFiles);
     248    allFiles = argv::getFullFileList(inputFiles);
    323249    const auto fileCount = allFiles.size();
    324250    lineCount.resize(fileCount);
Note: See TracChangeset for help on using the changeset viewer.