source: icGREP/icgrep-devel/icgrep/lz4_grep.cpp @ 6132

Last change on this file since 6132 was 6132, checked in by xwa163, 13 months ago
  1. More experiment on lz4 grep
  2. Improve performance of lzparabix grep
File size: 5.7 KB
RevLine 
[5955]1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/Linker/Linker.h>
10#include <llvm/Support/PrettyStackTrace.h>
11#include <llvm/Support/Signals.h>
12#include <llvm/Support/ManagedStatic.h>
13#include <IR_Gen/idisa_target.h>
14#include <boost/filesystem.hpp>
15#include <boost/iostreams/device/mapped_file.hpp>
16
17#include <lz4FrameDecoder.h>
18#include <cc/cc_compiler.h>
19#include <toolchain/toolchain.h>
20#include <kernels/cc_kernel.h>
21#include <kernels/streamset.h>
22#include <kernels/s2p_kernel.h>
23#include <kernels/kernel_builder.h>
24#include <toolchain/cpudriver.h>
25
26#include <iostream>
27#include <lz4/LZ4GrepGenerator.h>
28
29
30#include <re/re_alt.h>
31#include <re/re_start.h>
32#include <re/re_end.h>
33#include <re/re_utility.h>
34#include <re/re_toolchain.h>
35#include <pablo/pablo_toolchain.h>
[6036]36#include <llvm/Support/raw_ostream.h>
[5955]37
38namespace re { class CC; }
39
40using namespace llvm;
41using namespace parabix;
42using namespace kernel;
43
44static cl::OptionCategory lz4GrepFlags("Command Flags", "lz4d options");
45static cl::opt<std::string> regexString(cl::Positional, cl::desc("<regex>"), cl::Required, cl::cat(lz4GrepFlags));
46static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4GrepFlags));
47static cl::opt<bool> countOnly("count-only", cl::desc("Only count the match result"), cl::init(false), cl::cat(lz4GrepFlags));
[6026]48static cl::opt<bool> enableMultiplexing("enable-multiplexing", cl::desc("Enable CC multiplexing."), cl::init(false), cl::cat(lz4GrepFlags));
49
[5955]50static cl::OptionCategory lz4GrepDebugFlags("LZ4 Grep Debug Flags", "lz4d debug options");
[6059]51static cl::opt<bool> aio("aio", cl::desc("Use All-in-One Approach for LZ4 Decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
[6064]52static cl::opt<bool> parallelDecompression("parallel-decompression", cl::desc("Use parallel Approach for LZ4 Decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
[6039]53static cl::opt<bool> swizzledDecompression("swizzled-decompression", cl::desc("Use swizzle approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
[6118]54static cl::opt<bool> bitStreamDecompression("bitstream-decompression", cl::desc("Use bit stream approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
[6081]55static cl::opt<bool> enableGather("enable-gather", cl::desc("Enable gather intrinsics"), cl::init(false), cl::cat(lz4GrepDebugFlags));
56static cl::opt<bool> enableScatter("enable-scatter", cl::desc("Enable scatter intrinsics"), cl::init(false), cl::cat(lz4GrepDebugFlags));
[6111]57static cl::opt<int> minParallelLevel("min-parallel-level", cl::desc("Mininum parallel level"), cl::init(1), cl::cat(lz4GrepDebugFlags));
[5955]58
59
60int main(int argc, char *argv[]) {
61    // This boilerplate provides convenient stack traces and clean LLVM exit
62    // handling. It also initializes the built in support for convenient
63    // command line option handling.
64    sys::PrintStackTraceOnErrorSignal(argv[0]);
65    llvm::PrettyStackTraceProgram X(argc, argv);
66    llvm_shutdown_obj shutdown;
67    codegen::ParseCommandLineOptions(argc, argv, {&lz4GrepFlags, &lz4GrepDebugFlags, codegen::codegen_flags()});
68    std::string fileName = inputFile;
69    LZ4FrameDecoder lz4Frame(fileName);
70    if (!lz4Frame.isValid()) {
71        errs() << "Invalid LZ4 file.\n";
72        return -1;
73    }
74
75    boost::iostreams::mapped_file_source mappedFile;
76    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
77    mappedFile.open(fileName , lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
78    //char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
79    char *fileBuffer = const_cast<char *>(mappedFile.data());
80    re::RE * re_ast = re::RE_Parser::parse(regexString, re::MULTILINE_MODE_FLAG);
[6026]81    LZ4GrepGenerator g(enableMultiplexing);
[6059]82    if (aio) {
[6064]83        if (parallelDecompression) {
[6111]84            g.generateParallelAioPipeline(re_ast, enableGather, enableScatter, minParallelLevel);
[6059]85        } else if (swizzledDecompression) {
[6118]86            if (enableMultiplexing) {
87                g.generateMultiplexingSwizzledAioPipeline(re_ast);
88            } else {
89                g.generateSwizzledAioPipeline(re_ast);
90            }
91        } else if (bitStreamDecompression) {
[6119]92            if (enableMultiplexing) {
93                g.generateMultiplexingBitStreamAioPipeline(re_ast);
94            } else {
95                g.generateBitStreamAioPipeline(re_ast);
96            }
[6059]97        } else {
[6132]98            if (enableMultiplexing) {
99                g.generateByteStreamMultiplexingAioPipeline(re_ast);
100            } else {
101                g.generateAioPipeline(re_ast);
102            }
[6059]103        }
104
[6062]105        auto main = g.getCountOnlyGrepMainFunction();
106        uint64_t countResult = main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
107        llvm::outs() << countResult << "\n";
[6059]108    } else if (countOnly) {
[6039]109        if (swizzledDecompression) {
110            g.generateSwizzledCountOnlyGrepPipeline(re_ast);
111        } else {
[6040]112            g.generateCountOnlyGrepPipeline(re_ast, enableGather);
[6039]113        }
114
[6065]115        auto main = g.getCountOnlyGrepMainFunction();
116        uint64_t countResult = main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
117        llvm::outs() << countResult << "\n";
[5955]118    } else {
119        g.generateScanMatchGrepPipeline(re_ast);
120        g.invokeScanMatchGrep(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
121
122    }
123
124
125    mappedFile.close();
126    return 0;
127}
Note: See TracBrowser for help on using the repository browser.