Changeset 5892 for icGREP


Ignore:
Timestamp:
Mar 8, 2018, 10:50:56 AM (13 months ago)
Author:
cameron
Message:

Restructuring: integrating grep_engine and grep_pipeline

Location:
icGREP/icgrep-devel/icgrep
Files:
1 added
2 deleted
6 edited
2 moved

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/CMakeLists.txt

    r5887 r5892  
    5959SET(OBJECT_CACHE_SRC toolchain/object_cache.cpp)
    6060
    61 SET(GREP_CORE_SRC toolchain/grep_pipeline.cpp kernels/linebreak_kernel.cpp kernels/grep_kernel.cpp kernels/scanmatchgen.cpp )
     61SET(GREP_CORE_SRC kernels/linebreak_kernel.cpp kernels/grep_kernel.cpp kernels/scanmatchgen.cpp )
    6262
    6363SET(TOOLCHAIN_SRC toolchain/toolchain.cpp toolchain/pipeline.cpp)
     
    9797# RegExpADT is the core library for representing, parsing and printing regular expressions
    9898add_library(RegExpADT re/re_rep.cpp re/re_diff.cpp re/re_intersect.cpp re/re_range.cpp re/re_assertion.cpp re/printer_re.cpp re/re_parser_pcre.cpp re/re_parser_ere.cpp re/re_parser_bre.cpp re/re_parser_prosite.cpp re/parse_fixed_strings.cpp)
    99 add_library(RegExpCompiler re/casing.cpp re/to_utf8.cpp re/re_parser.cpp re/re_memoizer.cpp re/re_nullable.cpp re/re_simplifier.cpp re/re_star_normal.cpp re/re_minimizer.cpp re/re_local.cpp re/re_compiler.cpp re/re_analysis.cpp re/re_toolchain.cpp re/re_name_resolve.cpp re/re_name_gather.cpp re/re_collect_unicodesets.cpp re/re_multiplex.cpp re/re_utility.cpp re/grapheme_clusters.cpp re/exclude_CC.cpp ${GREP_CORE_SRC})
     99add_library(RegExpCompiler re/casing.cpp re/to_utf8.cpp re/re_parser.cpp re/re_memoizer.cpp re/re_nullable.cpp re/re_simplifier.cpp re/re_star_normal.cpp re/re_minimizer.cpp re/re_local.cpp re/re_compiler.cpp re/re_analysis.cpp re/re_toolchain.cpp re/re_name_resolve.cpp re/re_name_gather.cpp re/re_collect_unicodesets.cpp re/re_multiplex.cpp re/re_utility.cpp re/grapheme_clusters.cpp re/exclude_CC.cpp)
    100100add_library(UCDlib UCD/CaseFolding.cpp utf8_encoder.cpp utf16_encoder.cpp UCD/ucd_compiler.cpp UCD/PropertyObjects.cpp UCD/resolve_properties.cpp)
     101add_library(GrepEngine  ${GREP_CORE_SRC} grep/grep_engine.cpp kernels/cc_kernel.cpp kernels/cc_scan_kernel.cpp kernels/charclasses.cpp kernels/streams_merge.cpp kernels/until_n.cpp kernels/UCD_property_kernel.cpp kernels/grapheme_kernel.cpp)
    101102
    102103# force the compiler to compile the object cache to ensure that the versioning information is up to date
     
    109110target_link_libraries (CCADT PabloADT)
    110111target_link_libraries (RegExpADT PabloADT CCADT)
    111 target_link_libraries (UCDlib RegExpCompiler RegExpADT PabloADT CCADT)
     112target_link_libraries (UCDlib GrepEngine RegExpCompiler RegExpADT PabloADT CCADT)
    112113target_link_libraries (RegExpCompiler UCDlib CCADT RegExpADT)
    113 
    114 add_executable(icgrep icgrep.cpp grep_interface.cpp grep_engine.cpp kernels/u8u32_kernel.cpp kernels/delmask_kernel.cpp kernels/cc_kernel.cpp kernels/cc_scan_kernel.cpp kernels/charclasses.cpp kernels/linebreak_kernel.cpp kernels/streams_merge.cpp kernels/grep_kernel.cpp kernels/until_n.cpp kernels/UCD_property_kernel.cpp kernels/grapheme_kernel.cpp)
     114target_link_libraries (GrepEngine RegExpCompiler CodeGen)
     115
     116add_executable(icgrep icgrep.cpp grep_interface.cpp)
    115117add_executable(u8u16 u8u16.cpp)
    116118add_executable(base64 base64.cpp kernels/radix64.cpp)
     
    126128set_target_properties(core PROPERTIES EXCLUDE_FROM_ALL 1)
    127129
    128 target_link_libraries (icgrep UCDlib PabloADT RegExpCompiler CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
     130target_link_libraries (icgrep GrepEngine UCDlib PabloADT RegExpCompiler CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
    129131target_link_libraries (u8u16 PabloADT CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
    130132target_link_libraries (base64 PabloADT CCADT CodeGen ${REQ_LLVM_LIBRARIES} ${Boost_LIBRARIES} ${CUDA_LIB})
  • icGREP/icgrep-devel/icgrep/UCD/PropertyObjects.cpp

    r5865 r5892  
    1111#include <sstream>
    1212#include <llvm/Support/ErrorHandling.h>
    13 #include <toolchain/grep_pipeline.h>
     13#include <grep/grep_engine.h>
    1414#include <util/aligned_allocator.h>
    1515#include <re/re_analysis.h>
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.cpp

    r5891 r5892  
    11/*
    2  *  Copyright (c) 2017 International Characters.
     2 *  Copyright (c) 2018 International Characters.
    33 *  This software is licensed to the public under the Open Software License 3.0.
    44 *  icgrep is a trademark of International Characters.
     
    6060static cl::opt<bool> PropertyKernels("enable-property-kernels", cl::desc("Enable Unicode property kernels."), cl::init(false));
    6161
     62
    6263namespace grep {
     64   
     65
     66void accumulate_match_wrapper(intptr_t accum_addr, const size_t lineNum, char * line_start, char * line_end) {
     67    reinterpret_cast<MatchAccumulator *>(accum_addr)->accumulate_match(lineNum, line_start, line_end);
     68}
     69
     70void finalize_match_wrapper(intptr_t accum_addr, char * buffer_end) {
     71    reinterpret_cast<MatchAccumulator *>(accum_addr)->finalize_match(buffer_end);
     72}
     73
     74void grepBuffer(re::RE * pattern, const char * search_buffer, size_t bufferLength, MatchAccumulator * accum) {
     75    const unsigned segmentSize = codegen::BufferSegments * codegen::SegmentSize * codegen::ThreadNum;
     76   
     77    pattern = resolveCaseInsensitiveMode(pattern, false);
     78    pattern = regular_expression_passes(pattern);
     79   
     80   
     81    ParabixDriver pxDriver("codepointEngine");
     82    auto & idb = pxDriver.getBuilder();
     83    Module * M = idb->getModule();
     84   
     85    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getVoidTy(), idb->getInt8PtrTy(), idb->getSizeTy(), nullptr));
     86    mainFunc->setCallingConv(CallingConv::C);
     87    auto args = mainFunc->arg_begin();
     88    Value * const buffer = &*(args++);
     89    buffer->setName("buffer");
     90    Value * length = &*(args++);
     91    length->setName("length");
     92   
     93    idb->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", mainFunc, 0));
     94   
     95    StreamSetBuffer * ByteStream = pxDriver.addBuffer<SourceBuffer>(idb, idb->getStreamSetTy(1, 8));
     96    kernel::Kernel * sourceK = pxDriver.addKernelInstance<kernel::MemorySourceKernel>(idb, idb->getInt8PtrTy());
     97    sourceK->setInitialArguments({buffer, length});
     98    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
     99   
     100   
     101    StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(8, 1), segmentSize);
     102    kernel::Kernel * s2pk = pxDriver.addKernelInstance<kernel::S2PKernel>(idb);
     103    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
     104   
     105    StreamSetBuffer * LineFeedStream = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
     106    kernel::Kernel * linefeedK = pxDriver.addKernelInstance<kernel::LineFeedKernelBuilder>(idb, Binding{idb->getStreamSetTy(8), "basis", FixedRate(), Principal()});
     107    pxDriver.makeKernelCall(linefeedK, {BasisBits}, {LineFeedStream});
     108   
     109    StreamSetBuffer * LineBreakStream = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
     110   
     111    kernel::Kernel * requiredStreamsK = pxDriver.addKernelInstance<kernel::RequiredStreams_UTF8>(idb);
     112    StreamSetBuffer * RequiredStreams = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
     113    pxDriver.makeKernelCall(requiredStreamsK, {BasisBits, LineFeedStream}, {RequiredStreams, LineBreakStream});
     114   
     115    StreamSetBuffer * MatchResults = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
     116    kernel::Kernel * icgrepK = pxDriver.addKernelInstance<kernel::ICGrepKernel>(idb, pattern, std::vector<std::string>{"UTF8_LB", "UTF8_nonfinal"});
     117    pxDriver.makeKernelCall(icgrepK, {BasisBits, LineBreakStream, RequiredStreams}, {MatchResults});
     118   
     119    StreamSetBuffer * MatchedLines = pxDriver.addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
     120    kernel::Kernel * matchedLinesK = pxDriver.addKernelInstance<kernel::MatchedLinesKernel>(idb);
     121    pxDriver.makeKernelCall(matchedLinesK, {MatchResults, LineBreakStream}, {MatchedLines});
     122   
     123    kernel::Kernel * scanMatchK = pxDriver.addKernelInstance<kernel::ScanMatchKernel>(idb);
     124    scanMatchK->setInitialArguments({ConstantInt::get(idb->getIntAddrTy(), reinterpret_cast<intptr_t>(accum))});
     125    pxDriver.makeKernelCall(scanMatchK, {MatchedLines, LineBreakStream, ByteStream}, {});
     126    pxDriver.LinkFunction(*scanMatchK, "accumulate_match_wrapper", &accumulate_match_wrapper);
     127    pxDriver.LinkFunction(*scanMatchK, "finalize_match_wrapper", &finalize_match_wrapper);
     128   
     129    pxDriver.generatePipelineIR();
     130    pxDriver.deallocateBuffers();
     131    idb->CreateRetVoid();
     132    pxDriver.finalizeObject();
     133   
     134    typedef void (*GrepFunctionType)(const char * buffer, const size_t length);
     135    auto f = reinterpret_cast<GrepFunctionType>(pxDriver.getMain());
     136    f(search_buffer, bufferLength);
     137}
     138
     139
    63140
    64141// Grep Engine construction and initialization.
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.h

    r5891 r5892  
    88#include <grep_interface.h>
    99#include <kernels/streamset.h>
    10 #include <toolchain/grep_pipeline.h>
    1110#include <cc/multiplex_CCs.h>
    1211#include <string>
     
    2221
    2322namespace grep {
     23    class MatchAccumulator {
     24    public:
     25        MatchAccumulator() {}
     26        virtual void accumulate_match(const size_t lineNum, char * line_start, char * line_end) = 0;
     27        virtual void finalize_match(char * buffer_end) {}  // default: no op
     28    };
     29   
     30    void accumulate_match_wrapper(intptr_t accum_addr, const size_t lineNum, char * line_start, char * line_end);
     31   
     32    void finalize_match_wrapper(intptr_t accum_addr, char * buffer_end);
     33   
     34    void grepBuffer(re::RE * pattern, const char * buffer, size_t bufferLength, MatchAccumulator * accum);
    2435
    2536class GrepEngine {
  • icGREP/icgrep-devel/icgrep/icgrep.cpp

    r5786 r5892  
    11/*
    2  *  Copyright (c) 2014-7 International Characters.
     2 *  Copyright (c) 2014-8 International Characters.
    33 *  This software is licensed to the public under the Open Software License 3.0.
    44 *  icgrep is a trademark of International Characters.
     
    1717#include <re/re_parser.h>
    1818#include <re/re_utility.h>
    19 #include <grep_engine.h>
     19#include <grep/grep_engine.h>
    2020#include <grep_interface.h>
    2121#include <fstream>
  • icGREP/icgrep-devel/icgrep/kernels/scanmatchgen.cpp

    r5832 r5892  
    1010#include <IR_Gen/FunctionTypeBuilder.h>
    1111#include <llvm/Support/raw_ostream.h>
    12 #include <grep_engine.h>
     12#include <grep/grep_engine.h>
    1313
    1414using namespace llvm;
  • icGREP/icgrep-devel/icgrep/re/re_analysis.cpp

    r5891 r5892  
    440440}
    441441
    442 size_t byteTestComplexity(RE * re) {
     442unsigned byteTestComplexity(RE * re) {
    443443    ByteTestComplexity btc_object;
    444444    btc_object.gatherTests(re);
  • icGREP/icgrep-devel/icgrep/re/re_analysis.h

    r5891 r5892  
    3030bool hasAssertion(const RE * re);
    3131   
    32 size_t byteTestComplexity(RE * re);
     32unsigned byteTestComplexity(RE * re);
    3333   
    3434void UndefinedNameError (const Name * n);
Note: See TracChangeset for help on using the changeset viewer.