source: icGREP/icgrep-devel/icgrep/lz4d.cpp @ 5422

Last change on this file since 5422 was 5422, checked in by cameron, 2 years ago

lz4d - LZ4 decompressor - initial check-in

File size: 7.3 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/IR/Function.h>
10#include <llvm/Linker/Linker.h>
11#include <llvm/Support/CommandLine.h>
12#include <llvm/Support/PrettyStackTrace.h>
13#include <llvm/Support/Signals.h>
14#include <llvm/Support/ManagedStatic.h>
15#include <IR_Gen/idisa_builder.h>
16#include <IR_Gen/idisa_target.h>
17#include <boost/filesystem.hpp>
18#include <boost/iostreams/device/mapped_file.hpp>
19
20#include <lz4FrameDecoder.h>
21#include <cc/cc_compiler.h>
22#include <kernels/toolchain.h>
23#include <kernels/cc_kernel.h>
24#include <kernels/streamset.h>
25#include <kernels/s2p_kernel.h>
26#include <kernels/stdin_kernel.h>
27#include <kernels/stdout_kernel.h>
28#include <kernels/mmap_kernel.h>
29#include <kernels/lz4_index_decoder.h>
30#include <kernels/lz4_bytestream_decoder.h>
31#include <kernels/pipeline.h>
32
33#include <string>
34#include <iostream>
35namespace re { class CC; }
36
37using namespace llvm;
38using namespace parabix;
39using namespace kernel;
40
41static cl::OptionCategory lz4dFlags("Command Flags", "lz4d options");
42static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4dFlags));
43static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(lz4dFlags));
44static cl::opt<bool> overwriteOutput("f", cl::desc("Overwrite existing output file."), cl::init(false), cl::cat(lz4dFlags));
45
46
47typedef void (*MainFunctionType)(char * byte_data, size_t filesize, bool hasBlockChecksum);
48
49ParabixDriver * pxDriver = nullptr;
50
51
52void generatePipeline(ParabixDriver & pxDriver) {
53    IDISA::IDISA_Builder * iBuilder = pxDriver.getIDISA_Builder();
54    Module * M = iBuilder->getModule();
55
56    Type * const mBitBlockType = iBuilder->getBitBlockType();
57    Type * const size_ty = iBuilder->getSizeTy();
58    Type * const bool_ty = iBuilder->getIntNTy(sizeof(bool) * 8);
59    Type * const voidTy = iBuilder->getVoidTy();
60    Type * const inputType = iBuilder->getInt8PtrTy();
61   
62    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, size_ty, bool_ty, nullptr));
63    main->setCallingConv(CallingConv::C);
64    Function::arg_iterator args = main->arg_begin();
65    Value * const inputStream = &*(args++);
66    inputStream->setName("input");
67    Value * const fileSize = &*(args++);
68    fileSize->setName("fileSize");
69    Value * const hasBlockChecksum = &*(args++);
70    hasBlockChecksum->setName("hasBlockChecksum");
71
72    const unsigned segmentSize = codegen::SegmentSize;
73    const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
74    // Output buffer should be at least one whole LZ4 block (4MB) large in case of uncompressed blocks.
75    // And the size (in bytes) also needs to be a power of two.
76    const unsigned decompressBufBlocks = 4U * 1024 * 1024 / codegen::BlockSize;
77
78    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
79
80    StreamSetBuffer * const ByteStream = pxDriver.addBuffer(make_unique<SourceFileBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)));
81    StreamSetBuffer * const BasisBits = pxDriver.addBuffer(make_unique<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), segmentSize * bufferSegments));
82    StreamSetBuffer * const Extenders = pxDriver.addBuffer(make_unique<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), segmentSize * bufferSegments));
83    StreamSetBuffer * const LiteralIndexes = pxDriver.addBuffer(make_unique<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(2, 32), segmentSize * bufferSegments));
84    StreamSetBuffer * const MatchIndexes = pxDriver.addBuffer(make_unique<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(2, 32), segmentSize * bufferSegments));
85    StreamSetBuffer * const DecompressedByteStream = pxDriver.addBuffer(make_unique<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), decompressBufBlocks));
86
87   
88    kernel::KernelBuilder * sourceK = pxDriver.addKernelInstance(make_unique<kernel::FileSourceKernel>(iBuilder, iBuilder->getInt8PtrTy(), segmentSize));
89    sourceK->setInitialArguments({inputStream, fileSize});
90    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
91
92    // Input stream is not aligned due to the offset.
93    KernelBuilder * s2pk = pxDriver.addKernelInstance(make_unique<S2PKernel>(iBuilder, /*aligned = */ false));
94    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
95   
96    KernelBuilder * extenderK = pxDriver.addKernelInstance(make_unique<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8));
97    pxDriver.makeKernelCall(extenderK, {BasisBits}, {Extenders});
98
99    KernelBuilder * lz4iK = pxDriver.addKernelInstance(make_unique<LZ4IndexDecoderKernel>(iBuilder));
100    lz4iK->setInitialArguments({iBuilder->CreateTrunc(hasBlockChecksum, iBuilder->getInt1Ty())});
101    pxDriver.makeKernelCall(lz4iK, {ByteStream, Extenders}, {LiteralIndexes, MatchIndexes});
102
103    KernelBuilder * lz4bK = pxDriver.addKernelInstance(make_unique<LZ4ByteStreamDecoderKernel>(iBuilder, decompressBufBlocks * codegen::BlockSize));
104    pxDriver.makeKernelCall(lz4bK, {LiteralIndexes, MatchIndexes, ByteStream}, {DecompressedByteStream});
105
106    KernelBuilder * outK = pxDriver.addKernelInstance(make_unique<FileSink>(iBuilder, 8));
107    outK->setInitialArguments({iBuilder->CreatePointerCast(iBuilder->CreateGlobalString(outputFile), iBuilder->getInt8PtrTy())});
108    pxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
109 
110    pxDriver.generatePipelineIR();
111
112    iBuilder->CreateRetVoid();
113 
114    pxDriver.linkAndFinalize();
115}
116
117
118MainFunctionType codeGen() {
119    Module * M = new Module("lz4d", getGlobalContext());
120    IDISA::IDISA_Builder * idb = IDISA::GetIDISA_Builder(M);
121    pxDriver = new ParabixDriver(idb);
122
123    generatePipeline(*pxDriver);
124
125    delete idb;
126    return reinterpret_cast<MainFunctionType>(pxDriver->getPointerToMain());
127}
128
129
130int main(int argc, char *argv[]) {
131    // This boilerplate provides convenient stack traces and clean LLVM exit
132    // handling. It also initializes the built in support for convenient
133    // command line option handling.
134    sys::PrintStackTraceOnErrorSignal(argv[0]);
135    llvm::PrettyStackTraceProgram X(argc, argv);
136    llvm_shutdown_obj shutdown;
137    cl::HideUnrelatedOptions(ArrayRef<const cl::OptionCategory *>{&lz4dFlags, codegen::codegen_flags()});
138    cl::ParseCommandLineOptions(argc, argv);
139    std::string fileName = inputFile;
140    LZ4FrameDecoder lz4Frame(fileName);
141    if (!lz4Frame.isValid()) {
142        errs() << "Invalid LZ4 file.\n";
143        return -1;
144    }
145
146    if (boost::filesystem::exists(outputFile)) {
147        if (overwriteOutput) {
148            boost::filesystem::remove(outputFile);
149        } else {
150            errs() << outputFile + " existed. Use -f argument to overwrite.\n";
151            return -1;
152        }
153    }
154
155    boost::iostreams::mapped_file_source mappedFile;
156    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
157    mappedFile.open(fileName, lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
158    char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
159
160    MainFunctionType fn_ptr = codeGen();
161    fn_ptr(fileBuffer, lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
162
163    mappedFile.close();
164    return 0;
165}
Note: See TracBrowser for help on using the repository browser.