source: icGREP/icgrep-devel/icgrep/lz4_grep.cpp @ 6119

Last change on this file since 6119 was 6119, checked in by xwa163, 10 months ago
  1. Add some BasisSetNumbering? option to fix bug of multiplexing
  2. Use BiigEndian? BitNumbering? for lz4 and lzparabix related pipeline
  3. Support multiplexing in LZ4BitStreamAio pipeline
File size: 5.5 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/Linker/Linker.h>
10#include <llvm/Support/PrettyStackTrace.h>
11#include <llvm/Support/Signals.h>
12#include <llvm/Support/ManagedStatic.h>
13#include <IR_Gen/idisa_target.h>
14#include <boost/filesystem.hpp>
15#include <boost/iostreams/device/mapped_file.hpp>
16
17#include <lz4FrameDecoder.h>
18#include <cc/cc_compiler.h>
19#include <toolchain/toolchain.h>
20#include <kernels/cc_kernel.h>
21#include <kernels/streamset.h>
22#include <kernels/s2p_kernel.h>
23#include <kernels/kernel_builder.h>
24#include <toolchain/cpudriver.h>
25
26#include <iostream>
27#include <lz4/LZ4GrepGenerator.h>
28
29
30#include <re/re_alt.h>
31#include <re/re_start.h>
32#include <re/re_end.h>
33#include <re/re_utility.h>
34#include <re/re_toolchain.h>
35#include <pablo/pablo_toolchain.h>
36#include <llvm/Support/raw_ostream.h>
37
38namespace re { class CC; }
39
40using namespace llvm;
41using namespace parabix;
42using namespace kernel;
43
44static cl::OptionCategory lz4GrepFlags("Command Flags", "lz4d options");
45static cl::opt<std::string> regexString(cl::Positional, cl::desc("<regex>"), cl::Required, cl::cat(lz4GrepFlags));
46static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4GrepFlags));
47static cl::opt<bool> countOnly("count-only", cl::desc("Only count the match result"), cl::init(false), cl::cat(lz4GrepFlags));
48static cl::opt<bool> enableMultiplexing("enable-multiplexing", cl::desc("Enable CC multiplexing."), cl::init(false), cl::cat(lz4GrepFlags));
49
50static cl::OptionCategory lz4GrepDebugFlags("LZ4 Grep Debug Flags", "lz4d debug options");
51static cl::opt<bool> aio("aio", cl::desc("Use All-in-One Approach for LZ4 Decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
52static cl::opt<bool> parallelDecompression("parallel-decompression", cl::desc("Use parallel Approach for LZ4 Decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
53static cl::opt<bool> swizzledDecompression("swizzled-decompression", cl::desc("Use swizzle approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
54static cl::opt<bool> bitStreamDecompression("bitstream-decompression", cl::desc("Use bit stream approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
55static cl::opt<bool> enableGather("enable-gather", cl::desc("Enable gather intrinsics"), cl::init(false), cl::cat(lz4GrepDebugFlags));
56static cl::opt<bool> enableScatter("enable-scatter", cl::desc("Enable scatter intrinsics"), cl::init(false), cl::cat(lz4GrepDebugFlags));
57static cl::opt<int> minParallelLevel("min-parallel-level", cl::desc("Mininum parallel level"), cl::init(1), cl::cat(lz4GrepDebugFlags));
58
59
60int main(int argc, char *argv[]) {
61    // This boilerplate provides convenient stack traces and clean LLVM exit
62    // handling. It also initializes the built in support for convenient
63    // command line option handling.
64    sys::PrintStackTraceOnErrorSignal(argv[0]);
65    llvm::PrettyStackTraceProgram X(argc, argv);
66    llvm_shutdown_obj shutdown;
67    codegen::ParseCommandLineOptions(argc, argv, {&lz4GrepFlags, &lz4GrepDebugFlags, codegen::codegen_flags()});
68    std::string fileName = inputFile;
69    LZ4FrameDecoder lz4Frame(fileName);
70    if (!lz4Frame.isValid()) {
71        errs() << "Invalid LZ4 file.\n";
72        return -1;
73    }
74
75    boost::iostreams::mapped_file_source mappedFile;
76    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
77    mappedFile.open(fileName , lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
78    //char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
79    char *fileBuffer = const_cast<char *>(mappedFile.data());
80    re::RE * re_ast = re::RE_Parser::parse(regexString, re::MULTILINE_MODE_FLAG);
81    LZ4GrepGenerator g(enableMultiplexing);
82    if (aio) {
83        if (parallelDecompression) {
84            g.generateParallelAioPipeline(re_ast, enableGather, enableScatter, minParallelLevel);
85        } else if (swizzledDecompression) {
86            if (enableMultiplexing) {
87                g.generateMultiplexingSwizzledAioPipeline(re_ast);
88            } else {
89                g.generateSwizzledAioPipeline(re_ast);
90            }
91        } else if (bitStreamDecompression) {
92            if (enableMultiplexing) {
93                g.generateMultiplexingBitStreamAioPipeline(re_ast);
94            } else {
95                g.generateBitStreamAioPipeline(re_ast);
96            }
97
98        } else {
99            g.generateAioPipeline(re_ast);
100        }
101
102        auto main = g.getCountOnlyGrepMainFunction();
103        uint64_t countResult = main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
104        llvm::outs() << countResult << "\n";
105    } else if (countOnly) {
106        if (swizzledDecompression) {
107            g.generateSwizzledCountOnlyGrepPipeline(re_ast);
108        } else {
109            g.generateCountOnlyGrepPipeline(re_ast, enableGather);
110        }
111
112        auto main = g.getCountOnlyGrepMainFunction();
113        uint64_t countResult = main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
114        llvm::outs() << countResult << "\n";
115    } else {
116        g.generateScanMatchGrepPipeline(re_ast);
117        g.invokeScanMatchGrep(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
118
119    }
120
121
122    mappedFile.close();
123    return 0;
124}
Note: See TracBrowser for help on using the repository browser.