source: icGREP/icgrep-devel/icgrep/lz4_grep.cpp @ 6025

Last change on this file since 6025 was 6008, checked in by nmedfort, 17 months ago

Removed temporary buffers from pipeline and placed them in the source kernels.

File size: 4.1 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/Linker/Linker.h>
10#include <llvm/Support/PrettyStackTrace.h>
11#include <llvm/Support/Signals.h>
12#include <llvm/Support/ManagedStatic.h>
13#include <IR_Gen/idisa_target.h>
14#include <boost/filesystem.hpp>
15#include <boost/iostreams/device/mapped_file.hpp>
16
17#include <lz4FrameDecoder.h>
18#include <cc/cc_compiler.h>
19#include <toolchain/toolchain.h>
20#include <kernels/cc_kernel.h>
21#include <kernels/streamset.h>
22#include <kernels/s2p_kernel.h>
23#include <kernels/kernel_builder.h>
24#include <toolchain/cpudriver.h>
25
26#include <iostream>
27#include <lz4/LZ4GrepGenerator.h>
28
29
30#include <cstdio>
31#include <vector>
32#include <llvm/Support/CommandLine.h>
33#include <llvm/Support/ErrorHandling.h>
34#include <llvm/Support/Signals.h>
35#include <llvm/Support/raw_ostream.h>
36#include <re/re_alt.h>
37#include <re/re_seq.h>
38#include <re/re_start.h>
39#include <re/re_end.h>
40#include <re/parsers/parser.h>
41#include <re/re_utility.h>
42#include <grep/grep_engine.h>
43#include <grep_interface.h>
44#include <fstream>
45#include <string>
46#include <re/re_toolchain.h>
47#include <pablo/pablo_toolchain.h>
48#include <boost/filesystem.hpp>
49#include <iostream> // MEEE
50
51
52namespace re { class CC; }
53
54using namespace llvm;
55using namespace parabix;
56using namespace kernel;
57
58static cl::OptionCategory lz4GrepFlags("Command Flags", "lz4d options");
59static cl::opt<std::string> regexString(cl::Positional, cl::desc("<regex>"), cl::Required, cl::cat(lz4GrepFlags));
60static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4GrepFlags));
61static cl::opt<bool> countOnly("count-only", cl::desc("Only count the match result"), cl::init(false), cl::cat(lz4GrepFlags));
62//static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(lz4GrepFlags));
63//static cl::opt<bool> overwriteOutput("f", cl::desc("Overwrite existing output file."), cl::init(false), cl::cat(lz4GrepFlags));
64
65
66static cl::OptionCategory lz4GrepDebugFlags("LZ4 Grep Debug Flags", "lz4d debug options");
67//static cl::opt<bool> extractOnly("extract-only", cl::desc("Only extract literal data to output file"), cl::init(false), cl::cat(lz4GrepDebugFlags));
68//static cl::opt<bool> extractAndDepositOnly("extract-and-deposit-only", cl::desc("Only extract and deposit literal data to output file"), cl::init(false), cl::cat(lz4GrepDebugFlags));
69
70
71int main(int argc, char *argv[]) {
72    // This boilerplate provides convenient stack traces and clean LLVM exit
73    // handling. It also initializes the built in support for convenient
74    // command line option handling.
75    sys::PrintStackTraceOnErrorSignal(argv[0]);
76    llvm::PrettyStackTraceProgram X(argc, argv);
77    llvm_shutdown_obj shutdown;
78    codegen::ParseCommandLineOptions(argc, argv, {&lz4GrepFlags, &lz4GrepDebugFlags, codegen::codegen_flags()});
79    std::string fileName = inputFile;
80    LZ4FrameDecoder lz4Frame(fileName);
81    if (!lz4Frame.isValid()) {
82        errs() << "Invalid LZ4 file.\n";
83        return -1;
84    }
85
86    boost::iostreams::mapped_file_source mappedFile;
87    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
88    mappedFile.open(fileName , lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
89    //char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
90    char *fileBuffer = const_cast<char *>(mappedFile.data());
91    re::RE * re_ast = re::RE_Parser::parse(regexString, re::MULTILINE_MODE_FLAG);
92    LZ4GrepGenerator g;
93    if (countOnly) {
94        g.generateCountOnlyGrepPipeline(re_ast);
95        auto main = g.getMainFunc();
96        main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
97    } else {
98        g.generateScanMatchGrepPipeline(re_ast);
99        g.invokeScanMatchGrep(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
100
101    }
102
103
104    mappedFile.close();
105    return 0;
106}
Note: See TracBrowser for help on using the repository browser.