source: icGREP/icgrep-devel/icgrep/lz4d.cpp @ 6184

Last change on this file since 6184 was 6184, checked in by nmedfort, 8 months ago

Initial version of PipelineKernel? + revised StreamSet? model.

File size: 4.8 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/Linker/Linker.h>
10#include <llvm/Support/PrettyStackTrace.h>
11#include <llvm/Support/Signals.h>
12#include <llvm/Support/ManagedStatic.h>
13#include <toolchain/toolchain.h>
14
15#include <IR_Gen/idisa_target.h>
16#include <boost/filesystem.hpp>
17#include <boost/iostreams/device/mapped_file.hpp>
18
19#include <lz4/lz4_frame_decoder.h>
20#include <cc/alphabet.h>
21#include <cc/cc_compiler.h>
22#include <kernels/cc_kernel.h>
23#include <kernels/streamset.h>
24#include <kernels/s2p_kernel.h>
25#include <kernels/source_kernel.h>
26#include <kernels/stdout_kernel.h>
27#include <kernels/lz4/lz4_index_decoder.h>
28#include <kernels/lz4/lz4_bytestream_decoder.h>
29
30#include <kernels/kernel_builder.h>
31#include <kernels/pipeline_builder.h>
32#include <toolchain/cpudriver.h>
33#include <iostream>
34#include <llvm/Support/raw_ostream.h>
35namespace re { class CC; }
36
37using namespace llvm;
38using namespace kernel;
39
40static cl::OptionCategory lz4dFlags("Command Flags", "lz4d options");
41static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4dFlags));
42static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(lz4dFlags));
43static cl::opt<bool> overwriteOutput("f", cl::desc("Overwrite existing output file."), cl::init(false), cl::cat(lz4dFlags));
44
45typedef void (*MainFunctionType)(char * byte_data, size_t filesize, bool hasBlockChecksum, const char * outputFileName);
46
47MainFunctionType generatePipeline(CPUDriver & pxDriver) {
48    auto & b = pxDriver.getBuilder();
49
50    Type * const sizeTy = b->getSizeTy();
51    Type * const boolTy = b->getIntNTy(sizeof(bool) * 8);
52    Type * const int8PtrTy = b->getInt8PtrTy();
53   
54
55    auto P = pxDriver.makePipeline({Binding{int8PtrTy, "input"}, Binding{sizeTy, "fileSize"}, Binding{boolTy, "hasBlockChecksum"}, Binding{int8PtrTy, "outputFileName"}});
56
57    StreamSet * const ByteStream = P->CreateStreamSet(1, 8);
58    StreamSet * const BasisBits = P->CreateStreamSet(8, 1);
59    StreamSet * const Extenders = P->CreateStreamSet(1, 1);
60    StreamSet * const LiteralIndexes = P->CreateStreamSet(2, 32);
61    StreamSet * const MatchIndexes = P->CreateStreamSet(2, 32);
62    StreamSet * const DecompressedByteStream = P->CreateStreamSet(1, 8);
63
64
65    Scalar * const inputStream = P->getInputScalar("input");
66    Scalar * const fileSize = P->getInputScalar("fileSize");
67
68    P->CreateKernelCall<MemorySourceKernel>(inputStream, fileSize, ByteStream);
69
70    // Input stream is not aligned due to the offset.
71    P->CreateKernelCall<S2PKernel>(ByteStream, BasisBits, cc::BitNumbering::LittleEndian, /*aligned = */ false);
72
73    P->CreateKernelCall<ParabixCharacterClassKernelBuilder>("extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, BasisBits, Extenders);
74
75    Scalar * const hasBlockChecksum = P->getInputScalar("hasBlockChecksum");
76
77    P->CreateKernelCall<LZ4IndexDecoderKernel>(hasBlockChecksum, ByteStream, Extenders, LiteralIndexes, MatchIndexes);
78
79    P->CreateKernelCall<LZ4ByteStreamDecoderKernel>(LiteralIndexes, MatchIndexes, ByteStream, DecompressedByteStream);
80
81    Scalar * outputFileName = P->getInputScalar("outputFileName");
82    P->CreateKernelCall<FileSink>(outputFileName, DecompressedByteStream);
83
84    return reinterpret_cast<MainFunctionType>(P->compile());
85}
86
87int main(int argc, char *argv[]) {
88    // This boilerplate provides convenient stack traces and clean LLVM exit
89    // handling. It also initializes the built in support for convenient
90    // command line option handling.
91    sys::PrintStackTraceOnErrorSignal(argv[0]);
92    llvm::PrettyStackTraceProgram X(argc, argv);
93    llvm_shutdown_obj shutdown;
94    codegen::ParseCommandLineOptions(argc, argv, {&lz4dFlags, codegen::codegen_flags()});
95    std::string fileName = inputFile;
96    LZ4FrameDecoder lz4Frame(fileName);
97    if (!lz4Frame.isValid()) {
98        errs() << "Invalid LZ4 file.\n";
99        return -1;
100    }
101
102    if (boost::filesystem::exists(outputFile)) {
103        if (overwriteOutput) {
104            boost::filesystem::remove(outputFile);
105        } else {
106            errs() << outputFile + " existed. Use -f argument to overwrite.\n";
107            return -1;
108        }
109    }
110
111    boost::iostreams::mapped_file_source mappedFile;
112    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
113    mappedFile.open(fileName, lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
114    char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
115    CPUDriver pxDriver("lz4d");
116    auto main = generatePipeline(pxDriver);
117
118    main(fileBuffer, lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum(), outputFile.c_str());
119
120    mappedFile.close();
121    return 0;
122}
Note: See TracBrowser for help on using the repository browser.