source: icGREP/icgrep-devel/icgrep/lz4_grep.cpp @ 6079

Last change on this file since 6079 was 6065, checked in by xwa163, 17 months ago
  1. Fix some typo in LZ4 Grep Extract and Deposit pipeline
  2. Small fix for LZ4ParallelByteStreamAIOKernel
File size: 4.8 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/Linker/Linker.h>
10#include <llvm/Support/PrettyStackTrace.h>
11#include <llvm/Support/Signals.h>
12#include <llvm/Support/ManagedStatic.h>
13#include <IR_Gen/idisa_target.h>
14#include <boost/filesystem.hpp>
15#include <boost/iostreams/device/mapped_file.hpp>
16
17#include <lz4FrameDecoder.h>
18#include <cc/cc_compiler.h>
19#include <toolchain/toolchain.h>
20#include <kernels/cc_kernel.h>
21#include <kernels/streamset.h>
22#include <kernels/s2p_kernel.h>
23#include <kernels/kernel_builder.h>
24#include <toolchain/cpudriver.h>
25
26#include <iostream>
27#include <lz4/LZ4GrepGenerator.h>
28
29
30#include <re/re_alt.h>
31#include <re/re_start.h>
32#include <re/re_end.h>
33#include <re/re_utility.h>
34#include <re/re_toolchain.h>
35#include <pablo/pablo_toolchain.h>
36#include <llvm/Support/raw_ostream.h>
37
38namespace re { class CC; }
39
40using namespace llvm;
41using namespace parabix;
42using namespace kernel;
43
44static cl::OptionCategory lz4GrepFlags("Command Flags", "lz4d options");
45static cl::opt<std::string> regexString(cl::Positional, cl::desc("<regex>"), cl::Required, cl::cat(lz4GrepFlags));
46static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4GrepFlags));
47static cl::opt<bool> countOnly("count-only", cl::desc("Only count the match result"), cl::init(false), cl::cat(lz4GrepFlags));
48static cl::opt<bool> enableMultiplexing("enable-multiplexing", cl::desc("Enable CC multiplexing."), cl::init(false), cl::cat(lz4GrepFlags));
49
50static cl::OptionCategory lz4GrepDebugFlags("LZ4 Grep Debug Flags", "lz4d debug options");
51static cl::opt<bool> aio("aio", cl::desc("Use All-in-One Approach for LZ4 Decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
52static cl::opt<bool> parallelDecompression("parallel-decompression", cl::desc("Use parallel Approach for LZ4 Decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
53static cl::opt<bool> swizzledDecompression("swizzled-decompression", cl::desc("Use swizzle approach for decompression"), cl::init(false), cl::cat(lz4GrepDebugFlags));
54static cl::opt<bool> enableGather("enable-gather", cl::desc("Enable gather intrinsics for bitstream PDEP"), cl::init(false), cl::cat(lz4GrepDebugFlags));
55
56
57int main(int argc, char *argv[]) {
58    // This boilerplate provides convenient stack traces and clean LLVM exit
59    // handling. It also initializes the built in support for convenient
60    // command line option handling.
61    sys::PrintStackTraceOnErrorSignal(argv[0]);
62    llvm::PrettyStackTraceProgram X(argc, argv);
63    llvm_shutdown_obj shutdown;
64    codegen::ParseCommandLineOptions(argc, argv, {&lz4GrepFlags, &lz4GrepDebugFlags, codegen::codegen_flags()});
65    std::string fileName = inputFile;
66    LZ4FrameDecoder lz4Frame(fileName);
67    if (!lz4Frame.isValid()) {
68        errs() << "Invalid LZ4 file.\n";
69        return -1;
70    }
71
72    boost::iostreams::mapped_file_source mappedFile;
73    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
74    mappedFile.open(fileName , lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
75    //char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
76    char *fileBuffer = const_cast<char *>(mappedFile.data());
77    re::RE * re_ast = re::RE_Parser::parse(regexString, re::MULTILINE_MODE_FLAG);
78    LZ4GrepGenerator g(enableMultiplexing);
79    if (aio) {
80        if (parallelDecompression) {
81            g.generateParallelAioPipeline(re_ast);
82        } else if (enableMultiplexing) {
83            g.generateMultiplexingSwizzledAioPipeline2(re_ast);
84        } else if (swizzledDecompression) {
85            g.generateSwizzledAioPipeline(re_ast);
86        } else {
87            g.generateAioPipeline(re_ast);
88        }
89
90        auto main = g.getCountOnlyGrepMainFunction();
91        uint64_t countResult = main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
92        llvm::outs() << countResult << "\n";
93    } else if (countOnly) {
94        if (swizzledDecompression) {
95            g.generateSwizzledCountOnlyGrepPipeline(re_ast);
96        } else {
97            g.generateCountOnlyGrepPipeline(re_ast, enableGather);
98        }
99
100        auto main = g.getCountOnlyGrepMainFunction();
101        uint64_t countResult = main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
102        llvm::outs() << countResult << "\n";
103    } else {
104        g.generateScanMatchGrepPipeline(re_ast);
105        g.invokeScanMatchGrep(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
106
107    }
108
109
110    mappedFile.close();
111    return 0;
112}
Note: See TracBrowser for help on using the repository browser.