source: icGREP/icgrep-devel/icgrep/lz4d_ext_dep.cpp @ 5923

Last change on this file since 5923 was 5921, checked in by xwa163, 16 months ago
  1. Initial checkin for new approach for lz4 index decoder that always use 4MB buffer
  2. Add test case for new approach (for now test cases will fail when test file is larger than 4MB)
File size: 3.8 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/Linker/Linker.h>
10#include <llvm/Support/PrettyStackTrace.h>
11#include <llvm/Support/Signals.h>
12#include <llvm/Support/ManagedStatic.h>
13#include <IR_Gen/idisa_target.h>
14#include <boost/filesystem.hpp>
15#include <boost/iostreams/device/mapped_file.hpp>
16
17#include <lz4FrameDecoder.h>
18#include <cc/cc_compiler.h>
19#include <toolchain/toolchain.h>
20#include <kernels/cc_kernel.h>
21#include <kernels/streamset.h>
22#include <kernels/s2p_kernel.h>
23#include <kernels/kernel_builder.h>
24#include <toolchain/cpudriver.h>
25
26#include <iostream>
27#include <lz4/LZ4Generator.h>
28#include <lz4/LZ4GeneratorNew.h>
29
30namespace re { class CC; }
31
32using namespace llvm;
33using namespace parabix;
34using namespace kernel;
35
36static cl::OptionCategory lz4dFlags("Command Flags", "lz4d options");
37static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4dFlags));
38static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(lz4dFlags));
39static cl::opt<bool> overwriteOutput("f", cl::desc("Overwrite existing output file."), cl::init(false), cl::cat(lz4dFlags));
40
41
42static cl::OptionCategory lz4dDebugFlags("LZ4D Debug Flags", "lz4d debug options");
43static cl::opt<bool> extractOnly("extract-only", cl::desc("Only extract literal data to output file"), cl::init(false), cl::cat(lz4dDebugFlags));
44static cl::opt<bool> extractAndDepositOnly("extract-and-deposit-only", cl::desc("Only extract and deposit literal data to output file"), cl::init(false), cl::cat(lz4dDebugFlags));
45static cl::opt<bool> newApproach("new-approach", cl::desc("Use new approach"), cl::init(false), cl::cat(lz4dDebugFlags));
46
47
48int main(int argc, char *argv[]) {
49    // This boilerplate provides convenient stack traces and clean LLVM exit
50    // handling. It also initializes the built in support for convenient
51    // command line option handling.
52    sys::PrintStackTraceOnErrorSignal(argv[0]);
53    llvm::PrettyStackTraceProgram X(argc, argv);
54    llvm_shutdown_obj shutdown;
55    codegen::ParseCommandLineOptions(argc, argv, {&lz4dFlags, &lz4dDebugFlags, codegen::codegen_flags()});
56    std::string fileName = inputFile;
57    LZ4FrameDecoder lz4Frame(fileName);
58    if (!lz4Frame.isValid()) {
59        errs() << "Invalid LZ4 file.\n";
60        return -1;
61    }
62
63    if (boost::filesystem::exists(outputFile)) {
64        if (overwriteOutput) {
65            boost::filesystem::remove(outputFile);
66        } else {
67            errs() << outputFile + " existed. Use -f argument to overwrite.\n";
68            return -1;
69        }
70    }
71
72    boost::iostreams::mapped_file_source mappedFile;
73    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
74    mappedFile.open(fileName , lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
75    size_t headerLength = lz4Frame.getBlocksStart();
76    //char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
77    char *fileBuffer = const_cast<char *>(mappedFile.data());
78
79    if (codegen::SegmentSize < 2) {
80        codegen::SegmentSize = 2;
81    }
82
83    std::unique_ptr<LZ4Generator> g = newApproach? llvm::make_unique<LZ4GeneratorNew>(): llvm::make_unique<LZ4Generator>();
84
85    if (extractOnly) {
86        g->generateExtractOnlyPipeline(outputFile);
87    } else if (extractAndDepositOnly) {
88        g->generateExtractAndDepositOnlyPipeline(outputFile);
89    } else {
90        g->generatePipeline(outputFile);
91    }
92
93    auto main = g->getMainFunc();
94
95    main(fileBuffer, lz4Frame.getBlocksStart(), lz4Frame.getBlocksStart() + lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
96
97    mappedFile.close();
98    return 0;
99}
Note: See TracBrowser for help on using the repository browser.