source: icGREP/icgrep-devel/icgrep/lz4_grep.cpp @ 6118

Last change on this file since 6118 was 6118, checked in by xwa163, 10 months ago

Implement bitstream decompression version of LZ4 Aio Kernel

File size: 5.4 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/Linker/Linker.h>
10#include <llvm/Support/PrettyStackTrace.h>
11#include <llvm/Support/Signals.h>
12#include <llvm/Support/ManagedStatic.h>
13#include <IR_Gen/idisa_target.h>
14#include <boost/filesystem.hpp>
15#include <boost/iostreams/device/mapped_file.hpp>
16
17#include <lz4FrameDecoder.h>
18#include <cc/cc_compiler.h>
19#include <toolchain/toolchain.h>
20#include <kernels/cc_kernel.h>
21#include <kernels/streamset.h>
22#include <kernels/s2p_kernel.h>
23#include <kernels/kernel_builder.h>
24#include <toolchain/cpudriver.h>
25
26#include <iostream>
27#include <lz4/LZ4GrepGenerator.h>
28
29
30#include <re/re_alt.h>
31#include <re/re_start.h>
32#include <re/re_end.h>
33#include <re/re_utility.h>
34#include <re/re_toolchain.h>
35#include <pablo/pablo_toolchain.h>
36#include <llvm/Support/raw_ostream.h>
37
38namespace re { class CC; }
39
40using namespace llvm;
41using namespace parabix;
42using namespace kernel;
43
44static cl::OptionCategory lz4GrepFlags("Command Flags", "lz4d options");
45static cl::opt<std::string> regexString(cl::Positional, cl::desc("<regex>"), cl::Required, cl::cat(lz4GrepFlags));
46static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4GrepFlags));
47static cl::opt<bool> countOnly("count-only", cl::desc("Only count the match result"), cl::init(false), cl::cat(lz4GrepFlags));
48static cl::opt<bool> enableMultiplexing("enable-multiplexing", cl::desc("Enable CC multiplexing."), cl::init(false), cl::cat(lz4GrepFlags));
49
50static cl::OptionCategory lz4GrepDebugFlags("LZ4 Grep Debug Flags", "lz4d debug options");
51static cl::opt<bool> aio("aio", cl::desc("Use All-in-One Approach for LZ4 Decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
52static cl::opt<bool> parallelDecompression("parallel-decompression", cl::desc("Use parallel Approach for LZ4 Decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
53static cl::opt<bool> swizzledDecompression("swizzled-decompression", cl::desc("Use swizzle approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
54static cl::opt<bool> bitStreamDecompression("bitstream-decompression", cl::desc("Use bit stream approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
55static cl::opt<bool> enableGather("enable-gather", cl::desc("Enable gather intrinsics"), cl::init(false), cl::cat(lz4GrepDebugFlags));
56static cl::opt<bool> enableScatter("enable-scatter", cl::desc("Enable scatter intrinsics"), cl::init(false), cl::cat(lz4GrepDebugFlags));
57static cl::opt<int> minParallelLevel("min-parallel-level", cl::desc("Mininum parallel level"), cl::init(1), cl::cat(lz4GrepDebugFlags));
58
59
60
61int main(int argc, char *argv[]) {
62    // This boilerplate provides convenient stack traces and clean LLVM exit
63    // handling. It also initializes the built in support for convenient
64    // command line option handling.
65    sys::PrintStackTraceOnErrorSignal(argv[0]);
66    llvm::PrettyStackTraceProgram X(argc, argv);
67    llvm_shutdown_obj shutdown;
68    codegen::ParseCommandLineOptions(argc, argv, {&lz4GrepFlags, &lz4GrepDebugFlags, codegen::codegen_flags()});
69    std::string fileName = inputFile;
70    LZ4FrameDecoder lz4Frame(fileName);
71    if (!lz4Frame.isValid()) {
72        errs() << "Invalid LZ4 file.\n";
73        return -1;
74    }
75
76    boost::iostreams::mapped_file_source mappedFile;
77    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
78    mappedFile.open(fileName , lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
79    //char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
80    char *fileBuffer = const_cast<char *>(mappedFile.data());
81    re::RE * re_ast = re::RE_Parser::parse(regexString, re::MULTILINE_MODE_FLAG);
82    LZ4GrepGenerator g(enableMultiplexing);
83    if (aio) {
84        if (parallelDecompression) {
85            g.generateParallelAioPipeline(re_ast, enableGather, enableScatter, minParallelLevel);
86        } else if (swizzledDecompression) {
87            if (enableMultiplexing) {
88                g.generateMultiplexingSwizzledAioPipeline(re_ast);
89            } else {
90                g.generateSwizzledAioPipeline(re_ast);
91            }
92        } else if (bitStreamDecompression) {
93            g.generateBitStreamAioPipeline(re_ast);
94        } else {
95            g.generateAioPipeline(re_ast);
96        }
97
98        auto main = g.getCountOnlyGrepMainFunction();
99        uint64_t countResult = main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
100        llvm::outs() << countResult << "\n";
101    } else if (countOnly) {
102        if (swizzledDecompression) {
103            g.generateSwizzledCountOnlyGrepPipeline(re_ast);
104        } else {
105            g.generateCountOnlyGrepPipeline(re_ast, enableGather);
106        }
107
108        auto main = g.getCountOnlyGrepMainFunction();
109        uint64_t countResult = main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
110        llvm::outs() << countResult << "\n";
111    } else {
112        g.generateScanMatchGrepPipeline(re_ast);
113        g.invokeScanMatchGrep(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
114
115    }
116
117
118    mappedFile.close();
119    return 0;
120}
Note: See TracBrowser for help on using the repository browser.