Ignore:
Timestamp:
Mar 11, 2018, 9:30:05 AM (15 months ago)
Author:
cameron
Message:

Unicode-lines option; set default linebreak to LF

Location:
icGREP/icgrep-devel/icgrep
Files:
6 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.cpp

    r5897 r5900  
    147147    mNextFileToPrint(0),
    148148    grepMatchFound(false),
    149     mGrepRecordBreak(GrepRecordBreakKind::Unicode),
     149    mGrepRecordBreak(GrepRecordBreakKind::LF),
    150150    mMoveMatchesToEOL(true),
    151151    mEngineThread(pthread_self()) {}
  • icGREP/icgrep-devel/icgrep/grep_interface.cpp

    r5753 r5900  
    9191static cl::opt<bool, true> NullDataOption("z", cl::location(NullDataFlag), cl::desc("Use the NUL character (codepoint 00) as the line-break character for input."), cl::cat(Input_Options), cl::Grouping);
    9292static cl::alias NullDataAlias("null-data", cl::desc("Alias for -z"), cl::aliasopt(NullDataOption));
     93
     94bool UnicodeLinesFlag;
     95static cl::opt<bool, true> UnicodeLinesOption("Unicode-lines", cl::location(UnicodeLinesFlag), cl::desc("Enable Unicode line breaks (LF/VT/FF/CR/NEL/LS/PS/CRLF)"), cl::cat(Input_Options));
    9396
    9497bool MmapFlag;
  • icGREP/icgrep-devel/icgrep/grep_interface.h

    r5705 r5900  
    5050extern bool BinaryFlag; // -U
    5151extern bool NullDataFlag; // -z
     52extern bool UnicodeLinesFlag; // -Unicode-lines
     53
    5254extern bool MmapFlag; // -mmap
     55
    5356extern std::string ExcludeFlag; // -exclude
    5457extern std::string ExcludeFromFlag; // -exclude-from
  • icGREP/icgrep-devel/icgrep/icgrep.cpp

    r5894 r5900  
    3737static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<regex> <input file ...>"), cl::OneOrMore);
    3838
    39 static cl::opt<bool> UnixBreaks("Unix-line-breaks", cl::desc("Enable Unix line breaks"));
    4039static cl::opt<bool> ByteMode("enable-byte-mode", cl::desc("Process regular expressions in byte mode"));
    4140
     
    198197    }
    199198               
    200     if (UnixBreaks) {
    201         grepEngine->setRecordBreak(grep::GrepRecordBreakKind::LF);
     199    if (grep::UnicodeLinesFlag) {
     200        grepEngine->setRecordBreak(grep::GrepRecordBreakKind::Unicode);
    202201    } else if (grep::NullDataFlag) {
    203202        grepEngine->setRecordBreak(grep::GrepRecordBreakKind::Null);
     203    } else {
     204        grepEngine->setRecordBreak(grep::GrepRecordBreakKind::LF);
    204205    }
    205206   
  • icGREP/icgrep-devel/icgrep/kernels/grep_kernel.cpp

    r5890 r5900  
    11/*
    2  *  Copyright (c) 2017 International Characters.
     2 *  Copyright (c) 2018 International Characters.
    33 *  This software is licensed to the public under the Open Software License 3.0.
    44 */
     
    135135    it.createAssign(nonFinal, it.createAnd(nonFinal, u8valid));
    136136    pb.createAssign(nonFinal, pb.createOr(nonFinal, CRLF));
    137     PabloAST * unterminatedLineAtEOF = pb.createAtEOF(pb.createAdvance(pb.createNot(LineBreak), 1), "unterminatedLineAtEOF");
     137    //PabloAST * unterminatedLineAtEOF = pb.createAtEOF(pb.createAdvance(pb.createNot(LineBreak), 1), "unterminatedLineAtEOF");
    138138   
    139139    Var * const required = getOutputStreamVar("nonFinal");
    140140    pb.createAssign(pb.createExtract(required, pb.getInteger(0)), nonFinal);
    141     pb.createAssign(pb.createExtract(getOutputStreamVar("UnicodeLB"), pb.getInteger(0)), pb.createOr(LineBreak, unterminatedLineAtEOF, "EOL"));
     141    pb.createAssign(pb.createExtract(getOutputStreamVar("UnicodeLB"), pb.getInteger(0)), LineBreak);//pb.createOr(LineBreak, unterminatedLineAtEOF, "EOL"));
    142142}
    143143
  • icGREP/icgrep-devel/icgrep/re/re_compiler.cpp

    r5899 r5900  
    8686
    8787inline MarkerType RE_Compiler::compile(RE * const re, PabloBuilder & pb) {
    88     return process(re, makeMarker(FinalPostPositionUnit, pb.createOnes()), pb);
     88    return process(re, makeMarker(InitialPostPositionUnit, pb.createOnes()), pb);
    8989}
    9090   
Note: See TracChangeset for help on using the changeset viewer.