source: icGREP/icgrep-devel/icgrep/lz4_grep.cpp @ 6137

Last change on this file since 6137 was 6137, checked in by xwa163, 10 months ago
  1. LZ4 ScanMatch? pipeline
  2. Refactor LZ4 Generator
  3. Adjust some naming
File size: 5.1 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/Linker/Linker.h>
10#include <llvm/Support/PrettyStackTrace.h>
11#include <llvm/Support/Signals.h>
12#include <llvm/Support/ManagedStatic.h>
13#include <IR_Gen/idisa_target.h>
14#include <boost/filesystem.hpp>
15#include <boost/iostreams/device/mapped_file.hpp>
16
17#include <lz4/lz4_frame_decoder.h>
18#include <cc/cc_compiler.h>
19#include <toolchain/toolchain.h>
20#include <kernels/cc_kernel.h>
21#include <kernels/streamset.h>
22#include <kernels/s2p_kernel.h>
23#include <kernels/kernel_builder.h>
24#include <toolchain/cpudriver.h>
25
26#include <iostream>
27#include <lz4/lz4_grep_generator.h>
28
29
30#include <re/re_alt.h>
31#include <re/re_start.h>
32#include <re/re_end.h>
33#include <re/re_utility.h>
34#include <re/re_toolchain.h>
35#include <pablo/pablo_toolchain.h>
36#include <llvm/Support/raw_ostream.h>
37
38namespace re { class CC; }
39
40using namespace llvm;
41using namespace parabix;
42using namespace kernel;
43
44static cl::OptionCategory lz4GrepFlags("Command Flags", "lz4d options");
45static cl::opt<std::string> regexString(cl::Positional, cl::desc("<regex>"), cl::Required, cl::cat(lz4GrepFlags));
46static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4GrepFlags));
47static cl::opt<bool> countOnly("count-only", cl::desc("Only count the match result"), cl::init(false), cl::cat(lz4GrepFlags));
48static cl::opt<bool> enableMultiplexing("enable-multiplexing", cl::desc("Enable CC multiplexing."), cl::init(false), cl::cat(lz4GrepFlags));
49
50static cl::OptionCategory lz4GrepDebugFlags("LZ4 Grep Debug Flags", "lz4d debug options");
51static cl::opt<bool> parallelDecompression("parallel-decompression", cl::desc("Use parallel Approach for LZ4 Decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
52static cl::opt<bool> swizzledDecompression("swizzled-decompression", cl::desc("Use swizzle approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
53static cl::opt<bool> bitStreamDecompression("bitstream-decompression", cl::desc("Use bit stream approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
54static cl::opt<bool> enableGather("enable-gather", cl::desc("Enable gather intrinsics"), cl::init(false), cl::cat(lz4GrepDebugFlags));
55static cl::opt<bool> enableScatter("enable-scatter", cl::desc("Enable scatter intrinsics"), cl::init(false), cl::cat(lz4GrepDebugFlags));
56static cl::opt<int> minParallelLevel("min-parallel-level", cl::desc("Mininum parallel level"), cl::init(1), cl::cat(lz4GrepDebugFlags));
57
58
59int main(int argc, char *argv[]) {
60    // This boilerplate provides convenient stack traces and clean LLVM exit
61    // handling. It also initializes the built in support for convenient
62    // command line option handling.
63    sys::PrintStackTraceOnErrorSignal(argv[0]);
64    llvm::PrettyStackTraceProgram X(argc, argv);
65    llvm_shutdown_obj shutdown;
66    codegen::ParseCommandLineOptions(argc, argv, {&lz4GrepFlags, &lz4GrepDebugFlags, codegen::codegen_flags()});
67    std::string fileName = inputFile;
68    LZ4FrameDecoder lz4Frame(fileName);
69    if (!lz4Frame.isValid()) {
70        errs() << "Invalid LZ4 file.\n";
71        return -1;
72    }
73
74    boost::iostreams::mapped_file_source mappedFile;
75    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
76    mappedFile.open(fileName , lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
77    //char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
78    char *fileBuffer = const_cast<char *>(mappedFile.data());
79    re::RE * re_ast = re::RE_Parser::parse(regexString, re::MULTILINE_MODE_FLAG);
80    LZ4GrepGenerator g(enableMultiplexing);
81    if (countOnly) {
82        if (parallelDecompression) {
83            g.generateParallelAioPipeline(re_ast, enableGather, enableScatter, minParallelLevel);
84        } else if (swizzledDecompression) {
85            if (enableMultiplexing) {
86                g.generateMultiplexingSwizzledAioPipeline(re_ast);
87            } else {
88                g.generateSwizzledAioPipeline(re_ast);
89            }
90        } else if (bitStreamDecompression) {
91            if (enableMultiplexing) {
92                g.generateMultiplexingBitStreamAioPipeline(re_ast);
93            } else {
94                g.generateBitStreamAioPipeline(re_ast);
95            }
96        } else {
97            if (enableMultiplexing) {
98                g.generateByteStreamMultiplexingAioPipeline(re_ast);
99            } else {
100                g.generateAioPipeline(re_ast);
101            }
102        }
103
104        auto main = g.getCountOnlyGrepMainFunction();
105        uint64_t countResult = main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
106        llvm::outs() << countResult << "\n";
107    } else {
108        g.generateScanMatchGrepPipeline(re_ast);
109        g.invokeScanMatchGrep(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
110
111    }
112
113
114    mappedFile.close();
115    return 0;
116}
Note: See TracBrowser for help on using the repository browser.