Ignore:
Timestamp:
Feb 23, 2016, 9:07:05 AM (3 years ago)
Author:
lindanl
Message:

new version using the kernels.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/toolchain.cpp

    r4937 r4939  
    99#include <fstream>
    1010
    11 #include "basis_bits.h"
    1211#include "utf_encoding.h"
    1312#include "pablo/pablo_compiler.h"
     
    2221#include <llvm/Support/TargetSelect.h>
    2322#include <llvm/Support/Host.h>
    24 #include <llvm/Support/FileSystem.h>
    25 
    2623
    2724#include <IDISA/idisa_avx_builder.h>
     
    4239#include <pablo/optimizers/pablo_simplifier.hpp>
    4340#include <pablo/optimizers/codemotionpass.h>
    44 #ifdef ENABLE_MULTIPLEXING
    4541#include <pablo/passes/flattenassociativedfg.h>
    4642#include <pablo/passes/factorizedfg.h>
     43#ifdef ENABLE_MULTIPLEXING
    4744#include <pablo/optimizers/pablo_automultiplexing.hpp>
    4845#include <pablo/optimizers/pablo_bddminimization.h>
     
    5552#include <pablo/printer_pablos.h>
    5653
    57 #include <hrtime.h>
    58 #include <do_grep.h>
     54#include "do_grep.h"
    5955
    6056using namespace pablo;
     57
     58static cl::OptionCategory bGrepOutputOptions("Output Options",
     59                                      "These options control the output.");
     60
     61static cl::opt<bool> CountOnly("c", cl::desc("Count and display the matching lines per file only."), cl::cat(bGrepOutputOptions));
     62static cl::alias CountOnlyLong("count", cl::desc("Alias for -c"), cl::aliasopt(CountOnly));
     63static cl::opt<bool> NormalizeLineBreaks("normalize-line-breaks", cl::desc("Normalize line breaks to std::endl."), cl::init(false),  cl::cat(bGrepOutputOptions));
     64
     65static cl::opt<bool> ShowFileNames("H", cl::desc("Show the file name with each matching line."), cl::cat(bGrepOutputOptions));
     66static cl::alias ShowFileNamesLong("with-filename", cl::desc("Alias for -H"), cl::aliasopt(ShowFileNames));
     67
     68static cl::opt<bool> ShowLineNumbers("n", cl::desc("Show the line number with each matching line."), cl::cat(bGrepOutputOptions));
     69static cl::alias ShowLineNumbersLong("line-number", cl::desc("Alias for -n"), cl::aliasopt(ShowLineNumbers));
     70
    6171
    6272static cl::OptionCategory cRegexOutputOptions("Regex Dump Options",
     
    6878static cl::opt<bool> PrintUTF8REs("print-utf8-REs", cl::init(false), cl::desc("print out UTF-8 REs"), cl::cat(cRegexOutputOptions));
    6979static cl::opt<bool> PrintSimplifiedREs("print-simplified-REs", cl::init(false), cl::desc("print out final simplified REs"), cl::cat(cRegexOutputOptions));
    70 static cl::OptionCategory dPabloDumpOptions("Pablo Dump Options", "These options control printing of intermediate Pablo code.");
     80static cl::OptionCategory dPabloDumpOptions("Pablo Dump Options",
     81                                            "These options control printing of intermediate Pablo code.");
    7182
    7283static cl::opt<bool> PrintOptimizedREcode("print-pablo", cl::init(false), cl::desc("print final optimized Pablo code"), cl::cat(dPabloDumpOptions));
    7384static cl::opt<bool> PrintCompiledCCcode("print-CC-pablo", cl::init(false), cl::desc("print Pablo output from character class compiler"), cl::cat(dPabloDumpOptions));
    7485static cl::opt<bool> PrintCompiledREcode("print-RE-pablo", cl::init(false), cl::desc("print Pablo output from the regular expression compiler"), cl::cat(dPabloDumpOptions));
    75 static cl::opt<std::string> PabloOutputFilename("print-pablo-output", cl::init(""), cl::desc("output Pablo filename"), cl::cat(dPabloDumpOptions));
    7686
    7787static cl::OptionCategory cPabloOptimizationsOptions("Pablo Optimizations", "These options control Pablo optimization passes.");
    7888
    79 static cl::opt<bool> DisableSimplification("disable-simplification", cl::init(false),
    80                                      cl::desc("Disable Pablo Simplification pass (not recommended)"),
     89static cl::opt<bool> DisablePabloCSE("disable-CSE", cl::init(false),
     90                                     cl::desc("Disable Pablo common subexpression elimination/dead code elimination"),
    8191                                     cl::cat(cPabloOptimizationsOptions));
    82 
    8392static cl::opt<bool> PabloSinkingPass("sinking", cl::init(false),
    8493                                      cl::desc("Moves all instructions into the innermost legal If-scope so that they are only executed when needed."),
    8594                                      cl::cat(cPabloOptimizationsOptions));
    8695
    87 static cl::OptionCategory cMachineCodeOptimization("Machine Code Optimizations", "These options control back-end compilier optimization levels.");
    88 
    89 
    90 static cl::opt<char> OptLevel("O", cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] (default = '-O0')"),
    91                               cl::cat(cMachineCodeOptimization), cl::Prefix, cl::ZeroOrMore, cl::init('0'));
    92 
    9396#ifdef ENABLE_MULTIPLEXING
    9497static cl::opt<bool> PrintUnloweredCode("print-unlowered-pablo", cl::init(false), cl::desc("print Pablo output prior to lowering. "), cl::cat(dPabloDumpOptions));
     
    101104                                         cl::desc("coalesce associative functions prior to optimization passes."),
    102105                                         cl::cat(cPabloOptimizationsOptions));
    103 
    104106static cl::opt<bool> EnablePreDistribution("pre-dist", cl::init(false),
    105                                          cl::desc("apply distribution law optimization prior to multiplexing."),
     107                                         cl::desc("apply distribution law optimization."),
    106108                                         cl::cat(cPabloOptimizationsOptions));
    107 
    108109static cl::opt<bool> EnablePostDistribution("post-dist", cl::init(false),
    109                                          cl::desc("apply distribution law optimization after multiplexing."),
    110                                          cl::cat(cPabloOptimizationsOptions));
    111 
    112 static cl::opt<bool> EnablePrePassScheduling("pre-pass-scheduling", cl::init(false),
    113                                          cl::desc("apply pre-pass scheduling prior to LLVM IR generation."),
     110                                         cl::desc("apply distribution law optimization."),
    114111                                         cl::cat(cPabloOptimizationsOptions));
    115112#endif
     
    160157}
    161158
    162 #ifdef PRINT_TIMING_INFORMATION
    163 #define READ_CYCLE_COUNTER(name) name = read_cycle_counter();
    164 #else
    165 #define READ_CYCLE_COUNTER(name)
    166 #endif
    167 
    168 #ifdef PRINT_TIMING_INFORMATION
    169 unsigned COUNT_STATEMENTS(const PabloFunction * const entry) {
    170     std::stack<const Statement *> scope;
    171     unsigned statements = 0;
    172     // Scan through and collect all the advances, calls, scanthrus and matchstars ...
    173     for (const Statement * stmt = entry->getEntryBlock()->front(); ; ) {
    174         while ( stmt ) {
    175             ++statements;
    176             if (LLVM_UNLIKELY(isa<If>(stmt) || isa<While>(stmt))) {
    177                 // Set the next statement to be the first statement of the inner scope and push the
    178                 // next statement of the current statement into the scope stack.
    179                 const PabloBlock * const nested = isa<If>(stmt) ? cast<If>(stmt)->getBody() : cast<While>(stmt)->getBody();
    180                 scope.push(stmt->getNextNode());
    181                 stmt = nested->front();
    182                 assert (stmt);
    183                 continue;
    184             }
    185             stmt = stmt->getNextNode();
    186         }
    187         if (scope.empty()) {
    188             break;
    189         }
    190         stmt = scope.top();
    191         scope.pop();
    192     }
    193     return statements;
    194 }
    195 
    196 unsigned COUNT_ADVANCES(const PabloFunction * const entry) {
    197 
    198     std::stack<const Statement *> scope;
    199     unsigned advances = 0;
    200 
    201     // Scan through and collect all the advances, calls, scanthrus and matchstars ...
    202     for (const Statement * stmt = entry->getEntryBlock()->front(); ; ) {
    203         while ( stmt ) {
    204             if (isa<Advance>(stmt)) {
    205                 ++advances;
    206             }
    207             else if (LLVM_UNLIKELY(isa<If>(stmt) || isa<While>(stmt))) {
    208                 // Set the next statement to be the first statement of the inner scope and push the
    209                 // next statement of the current statement into the scope stack.
    210                 const PabloBlock * const nested = isa<If>(stmt) ? cast<If>(stmt)->getBody() : cast<While>(stmt)->getBody();
    211                 scope.push(stmt->getNextNode());
    212                 stmt = nested->front();
    213                 assert (stmt);
    214                 continue;
    215             }
    216             stmt = stmt->getNextNode();
    217         }
    218         if (scope.empty()) {
    219             break;
    220         }
    221         stmt = scope.top();
    222         scope.pop();
    223     }
    224     return advances;
    225 }
    226 
    227 using DistributionMap = boost::container::flat_map<unsigned, unsigned>;
    228 
    229 DistributionMap SUMMARIZE_VARIADIC_DISTRIBUTION(const PabloFunction * const entry) {
    230     std::stack<const Statement *> scope;
    231     DistributionMap distribution;
    232     // Scan through and collect all the advances, calls, scanthrus and matchstars ...
    233     for (const Statement * stmt = entry->getEntryBlock()->front(); ; ) {
    234         while ( stmt ) {
    235             if (isa<Variadic>(stmt)) {
    236                 auto f = distribution.find(stmt->getNumOperands());
    237                 if (f == distribution.end()) {
    238                     distribution.emplace(stmt->getNumOperands(), 1);
    239                 } else {
    240                     f->second += 1;
    241                 }
    242             }
    243             else if (LLVM_UNLIKELY(isa<If>(stmt) || isa<While>(stmt))) {
    244                 // Set the next statement to be the first statement of the inner scope and push the
    245                 // next statement of the current statement into the scope stack.
    246                 const PabloBlock * const nested = isa<If>(stmt) ? cast<If>(stmt)->getBody() : cast<While>(stmt)->getBody();
    247                 scope.push(stmt->getNextNode());
    248                 stmt = nested->front();
    249                 assert (stmt);
    250                 continue;
    251             }
    252             stmt = stmt->getNextNode();
    253         }
    254         if (scope.empty()) {
    255             break;
    256         }
    257         stmt = scope.top();
    258         scope.pop();
    259     }
    260     return distribution;
    261 }
    262 #endif
    263 
    264 void pablo_function_passes(PabloFunction * function) {   
     159void pablo_function_passes(PabloFunction * function) {
    265160    // Scan through the pablo code and perform DCE and CSE
    266 
    267 #ifdef PRINT_TIMING_INFORMATION
    268     timestamp_t simplification_start = 0, simplification_end = 0;
    269     timestamp_t coalescing_start = 0, coalescing_end = 0;
    270     timestamp_t sinking_start = 0, sinking_end = 0;
    271     timestamp_t pre_distribution_start = 0, pre_distribution_end = 0;
    272     timestamp_t multiplexing_start = 0, multiplexing_end = 0;
    273     timestamp_t post_distribution_start = 0, post_distribution_end = 0;
    274     timestamp_t lowering_start = 0, lowering_end = 0;
    275     timestamp_t scheduling_start = 0, scheduling_end = 0;
    276     DistributionMap distribution;
    277     const timestamp_t optimization_start = read_cycle_counter();
    278 #endif
    279     if (!DisableSimplification) {
    280         READ_CYCLE_COUNTER(simplification_start);
     161    if (!DisablePabloCSE) {
    281162        Simplifier::optimize(*function);
    282         READ_CYCLE_COUNTER(simplification_end);
    283163    }
    284164#ifdef ENABLE_MULTIPLEXING
    285     if (EnableLowering || EnablePreDistribution || EnablePostDistribution) {
    286         READ_CYCLE_COUNTER(coalescing_start);
    287         CanonicalizeDFG::transform(*function);
    288         READ_CYCLE_COUNTER(coalescing_end);
    289     }
     165    if (EnableLowering || EnablePreDistribution || EnablePostDistribution || EnableMultiplexing) {
     166        FlattenAssociativeDFG::transform(*function);
     167    }
     168#endif
     169    if (PabloSinkingPass) {
     170        CodeMotionPass::optimize(*function);
     171    }
     172#ifdef ENABLE_MULTIPLEXING   
    290173    if (EnablePreDistribution) {
    291         READ_CYCLE_COUNTER(pre_distribution_start);
    292174        DistributivePass::optimize(*function);
    293         READ_CYCLE_COUNTER(pre_distribution_end);
    294175    }
    295176    if (EnableMultiplexing) {
    296         READ_CYCLE_COUNTER(multiplexing_start);
    297177        MultiplexingPass::optimize(*function);
    298         READ_CYCLE_COUNTER(multiplexing_end);
    299         if (EnableLowering || EnablePreDistribution || EnablePostDistribution) {
    300             CanonicalizeDFG::transform(*function);
    301         }
    302178    }
    303179    if (EnablePostDistribution) {
    304         READ_CYCLE_COUNTER(post_distribution_start);
    305180        DistributivePass::optimize(*function);
    306         READ_CYCLE_COUNTER(post_distribution_end);
    307     }
    308 #endif
    309     if (PabloSinkingPass) {
    310         READ_CYCLE_COUNTER(sinking_start);
    311         CodeMotionPass::optimize(*function);
    312         READ_CYCLE_COUNTER(sinking_end);
    313     }
    314 #ifdef ENABLE_MULTIPLEXING
     181    }
     182    SchedulingPrePass::optimize(*function);
    315183    if (PrintUnloweredCode) {
    316184        //Print to the terminal the AST that was generated by the pararallel bit-stream compiler.
     
    318186        cerr << "Unlowered Pablo AST:\n";
    319187        PabloPrinter::print(*function, cerr);
    320     }   
    321     #ifdef PRINT_TIMING_INFORMATION
    322     distribution = SUMMARIZE_VARIADIC_DISTRIBUTION(function);
    323     #endif
    324     if (EnableLowering || EnablePreDistribution || EnablePostDistribution) {
    325         READ_CYCLE_COUNTER(lowering_start);
     188    }
     189    if (EnableLowering || EnablePreDistribution || EnablePostDistribution || EnableMultiplexing) {
    326190        FactorizeDFG::transform(*function);
    327         READ_CYCLE_COUNTER(lowering_end);
    328     }
    329     if (EnablePrePassScheduling) {
    330         READ_CYCLE_COUNTER(scheduling_start);
    331         SchedulingPrePass::optimize(*function);
    332         READ_CYCLE_COUNTER(scheduling_end);
    333     }
    334 #endif
    335 #ifdef PRINT_TIMING_INFORMATION
    336     const timestamp_t optimization_end = read_cycle_counter();
     191    }
    337192#endif
    338193    if (PrintOptimizedREcode) {
    339         if (PabloOutputFilename.empty()) {
    340             //Print to the terminal the AST that was generated by the pararallel bit-stream compiler.
    341             llvm::raw_os_ostream cerr(std::cerr);
    342             cerr << "Final Pablo AST:\n";
    343             PabloPrinter::print(*function, cerr);
    344         } else {
    345             std::error_code error;
    346             llvm::raw_fd_ostream out(PabloOutputFilename, error, sys::fs::OpenFlags::F_None);
    347             PabloPrinter::print(*function, out);
    348         }
    349     }
    350 #ifdef PRINT_TIMING_INFORMATION
    351     std::cerr << "PABLO OPTIMIZATION TIME: " << (optimization_end - optimization_start) << std::endl;
    352     std::cerr << "  SIMPLIFICATION TIME: " << (simplification_end - simplification_start) << std::endl;
    353     std::cerr << "  COALESCING TIME: " << (coalescing_end - coalescing_start) << std::endl;
    354     std::cerr << "  SINKING TIME: " << (sinking_end - sinking_start) << std::endl;
    355     std::cerr << "  PRE-DISTRIBUTION TIME: " << (pre_distribution_end - pre_distribution_start) << std::endl;
    356     std::cerr << "  MULTIPLEXING TIME: " << (multiplexing_end - multiplexing_start) << std::endl;
    357     std::cerr << "  MULTIPLEXING SEED: " << MultiplexingPass::SEED << std::endl;
    358     std::cerr << "  MULTIPLEXING NODES USED: " << MultiplexingPass::NODES_USED << std::endl;
    359     std::cerr << "  MULTIPLEXING NODES ALLOCATED: " << MultiplexingPass::NODES_ALLOCATED << std::endl;
    360     std::cerr << "  LOWERING TIME: " << (lowering_end - lowering_start) << std::endl;
    361     std::cerr << "  POST-DISTRIBUTION TIME: " << (post_distribution_end - post_distribution_start) << std::endl;
    362     std::cerr << "  SCHEDULING TIME: " << (scheduling_end - scheduling_start) << std::endl;
    363     std::cerr << "PABLO STATEMENTS: " << COUNT_STATEMENTS(function) << std::endl;
    364     std::cerr << "PABLO ADVANCES: " << COUNT_ADVANCES(function) << std::endl;
    365     std::cerr << "PRE-LOWERING VARIADIC DISTRIBUTION: ";
    366     bool join = false;
    367     for (auto dist : distribution) {
    368         if (join) {
    369             std::cerr << ';';
    370         }
    371         std::cerr << dist.first << '|' << dist.second;
    372         join = true;
    373     }
    374     std::cerr << std::endl;
    375 #endif
     194        PabloVerifier::verify(*function, "post-optimization");
     195        //Print to the terminal the AST that was generated by the pararallel bit-stream compiler.
     196        llvm::raw_os_ostream cerr(std::cerr);
     197        cerr << "Final Pablo AST:\n";
     198        PabloPrinter::print(*function, cerr);
     199    }
    376200}
    377201
     
    411235    builder.setErrorStr(&errMessage);
    412236    builder.setMCPU(sys::getHostCPUName());
    413     CodeGenOpt::Level optLevel = CodeGenOpt::Level::None;
    414     switch (OptLevel) {
    415         case '0': optLevel = CodeGenOpt::None; break;
    416         case '1': optLevel = CodeGenOpt::Less; break;
    417         case '2': optLevel = CodeGenOpt::Default; break;
    418         case '3': optLevel = CodeGenOpt::Aggressive; break;
    419         default: errs() << OptLevel << " is an invalid optimization level.\n";
    420     }
    421     builder.setOptLevel(optLevel);
     237    builder.setOptLevel(CodeGenOpt::Level::None);
    422238
    423239#if (BLOCK_SIZE == 256)
     
    438254}
    439255
     256int total_count = 0;
     257
    440258extern "C" {
    441     void wrapped_report_match(uint64_t recordNum, uint64_t recordStart, uint64_t recordEnd) {
    442         printf("line %lu: (%lu, %lu)\n", recordNum, recordStart, recordEnd);
    443     }
    444 }
    445 
     259    void wrapped_report_match(uint64_t lineNum, uint64_t line_start, uint64_t line_end, const char * buffer, int filesize, char * filename) {
     260        if(CountOnly){
     261            total_count++;
     262            return;
     263        }
     264
     265        llvm::raw_os_ostream out(std::cout);
     266        if (ShowFileNames) {
     267            out << filename << ':';
     268        }
     269        if (ShowLineNumbers) {
     270            out << lineNum << ":";
     271        }
     272
     273        if ((buffer[line_start] == 0xA) && (line_start != line_end)) {
     274            // The line "starts" on the LF of a CRLF.  Really the end of the last line.
     275            line_start++;
     276        }
     277        if (line_end == filesize) {
     278            // The match position is at end-of-file.   We have a final unterminated line.
     279            out.write(&buffer[line_start], line_end - line_start);
     280            if (NormalizeLineBreaks) {
     281                out << '\n';  // terminate it
     282            }
     283            return;
     284        }
     285        unsigned char end_byte = (unsigned char)buffer[line_end];
     286        if (NormalizeLineBreaks) {
     287            if (end_byte == 0x85) {
     288                // Line terminated with NEL, on the second byte.  Back up 1.
     289                line_end--;
     290            } else if (end_byte > 0xD) {
     291                // Line terminated with PS or LS, on the third byte.  Back up 2.
     292                line_end -= 2;
     293            }
     294            out.write(&buffer[line_start], line_end - line_start);
     295            out << '\n';
     296        }
     297        else{   
     298            if (end_byte == 0x0D) {
     299                // Check for line_end on first byte of CRLF;  note that we don't
     300                // want to access past the end of buffer.
     301                if ((line_end + 1 < filesize) && (buffer[line_end + 1] == 0x0A)) {
     302                    // Found CRLF; preserve both bytes.
     303                    line_end++;
     304                }
     305            }
     306            out.write(&buffer[line_start], line_end - line_start + 1);
     307        }
     308    }
     309}
     310
     311
     312void PrintTotalCount(){
     313    if(CountOnly){
     314        std::cout << total_count << std::endl;
     315    }
     316}
     317
     318re::CC * parsedCodePointSet;
     319
     320extern "C" {
     321    void insert_codepoints(uint64_t lineNum, uint64_t line_start, uint64_t line_end, const char * buffer) {
     322       re::codepoint_t c = 0;
     323        ssize_t line_pos = line_start;
     324        while (isxdigit(buffer[line_pos])) {
     325            if (isdigit(buffer[line_pos])) {
     326                c = (c << 4) | (buffer[line_pos] - '0');
     327            }
     328            else {
     329                c = (c << 4) | (tolower(buffer[line_pos]) - 'a' + 10);
     330            }
     331            line_pos++;
     332        }
     333        assert(((line_pos - line_start) >= 4) && ((line_pos - line_start) <= 6)); // UCD format 4 to 6 hex digits.       
     334        parsedCodePointSet->insert(c);
     335    }
     336}
     337
     338void setParsedCodePointSet(){
     339    parsedCodePointSet = re::makeCC();
     340}
     341
     342re::CC * getParsedCodePointSet(){
     343    return parsedCodePointSet;
     344}
    446345
    447346extern "C" {
     
    465364            e->addGlobalMapping(cast<GlobalValue>(it), (void *)&wrapped_report_match);
    466365        }
     366        if (fnName == "insert_codepoints") {
     367            e->addGlobalMapping(cast<GlobalValue>(it), (void *)&insert_codepoints);
     368        }
    467369#ifndef DISABLE_PREGENERATED_UCD_FUNCTIONS
    468370        else {
Note: See TracChangeset for help on using the changeset viewer.