source: icGREP/icgrep-devel/icgrep/lz4d.cpp @ 6008

Last change on this file since 6008 was 6008, checked in by nmedfort, 17 months ago

Removed temporary buffers from pipeline and placed them in the source kernels.

File size: 6.6 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/IR/Function.h>
10#include <llvm/Linker/Linker.h>
11#include <llvm/Support/CommandLine.h>
12#include <llvm/Support/PrettyStackTrace.h>
13#include <llvm/Support/Signals.h>
14#include <llvm/Support/ManagedStatic.h>
15#include <toolchain/toolchain.h>
16
17#include <IR_Gen/idisa_target.h>
18#include <boost/filesystem.hpp>
19#include <boost/iostreams/device/mapped_file.hpp>
20
21#include <lz4FrameDecoder.h>
22#include <cc/cc_compiler.h>
23#include <kernels/cc_kernel.h>
24#include <kernels/streamset.h>
25#include <kernels/s2p_kernel.h>
26#include <kernels/source_kernel.h>
27#include <kernels/stdout_kernel.h>
28#include <kernels/lz4/lz4_index_decoder.h>
29#include <kernels/lz4/lz4_bytestream_decoder.h>
30
31#include <kernels/kernel_builder.h>
32#include <toolchain/cpudriver.h>
33#include <llvm/Support/raw_ostream.h>
34#include <string>
35#include <iostream>
36namespace re { class CC; }
37
38using namespace llvm;
39using namespace parabix;
40using namespace kernel;
41
42static cl::OptionCategory lz4dFlags("Command Flags", "lz4d options");
43static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4dFlags));
44static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(lz4dFlags));
45static cl::opt<bool> overwriteOutput("f", cl::desc("Overwrite existing output file."), cl::init(false), cl::cat(lz4dFlags));
46
47typedef void (*MainFunctionType)(char * byte_data, size_t filesize, bool hasBlockChecksum);
48
49void generatePipeline(ParabixDriver & pxDriver) {
50    auto & iBuilder = pxDriver.getBuilder();
51    Module * M = iBuilder->getModule();
52
53    Type * const sizeTy = iBuilder->getSizeTy();
54    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
55    Type * const voidTy = iBuilder->getVoidTy();
56    Type * const inputType = iBuilder->getInt8PtrTy();
57   
58    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, boolTy, nullptr));
59    main->setCallingConv(CallingConv::C);
60    Function::arg_iterator args = main->arg_begin();
61    Value * const inputStream = &*(args++);
62    inputStream->setName("input");
63    Value * const fileSize = &*(args++);
64    fileSize->setName("fileSize");
65    Value * const hasBlockChecksum = &*(args++);
66    hasBlockChecksum->setName("hasBlockChecksum");
67
68    const unsigned segmentSize = codegen::SegmentSize;
69    const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
70    // Output buffer should be at least one whole LZ4 block (4MB) large in case of uncompressed blocks.
71    // And the size (in bytes) also needs to be a power of two.
72    const unsigned decompressBufBlocks = (4 * 1024 * 1024) / codegen::BlockSize;
73
74    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
75
76    StreamSetBuffer * const ByteStream = pxDriver.addBuffer<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
77    StreamSetBuffer * const BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), segmentSize * bufferSegments);
78    StreamSetBuffer * const Extenders = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), segmentSize * bufferSegments);
79    StreamSetBuffer * const LiteralIndexes = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(2, 32), segmentSize * bufferSegments);
80    StreamSetBuffer * const MatchIndexes = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(2, 32), segmentSize * bufferSegments);
81    StreamSetBuffer * const DecompressedByteStream = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), decompressBufBlocks);
82
83   
84    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
85    sourceK->setInitialArguments({inputStream, fileSize});
86    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
87
88    // Input stream is not aligned due to the offset.
89    Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder, /*aligned = */ false);
90    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
91   
92    Kernel * extenderK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8);
93    pxDriver.makeKernelCall(extenderK, {BasisBits}, {Extenders});
94
95    Kernel * lz4iK = pxDriver.addKernelInstance<LZ4IndexDecoderKernel>(iBuilder);
96    lz4iK->setInitialArguments({iBuilder->CreateTrunc(hasBlockChecksum, iBuilder->getInt1Ty())});
97    pxDriver.makeKernelCall(lz4iK, {ByteStream, Extenders}, {LiteralIndexes, MatchIndexes});
98
99    Kernel * lz4bK = pxDriver.addKernelInstance<LZ4ByteStreamDecoderKernel>(iBuilder, decompressBufBlocks * codegen::BlockSize);
100    pxDriver.makeKernelCall(lz4bK, {LiteralIndexes, MatchIndexes, ByteStream}, {DecompressedByteStream});
101
102    Kernel * outK = pxDriver.addKernelInstance<FileSink>(iBuilder, 8);
103    outK->setInitialArguments({iBuilder->GetString(outputFile)});
104    pxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
105 
106    pxDriver.generatePipelineIR();
107
108    pxDriver.deallocateBuffers();
109
110    iBuilder->CreateRetVoid();
111 
112    pxDriver.finalizeObject();
113}
114
115int main(int argc, char *argv[]) {
116    // This boilerplate provides convenient stack traces and clean LLVM exit
117    // handling. It also initializes the built in support for convenient
118    // command line option handling.
119    sys::PrintStackTraceOnErrorSignal(argv[0]);
120    llvm::PrettyStackTraceProgram X(argc, argv);
121    llvm_shutdown_obj shutdown;
122    codegen::ParseCommandLineOptions(argc, argv, {&lz4dFlags, codegen::codegen_flags()});
123    std::string fileName = inputFile;
124    LZ4FrameDecoder lz4Frame(fileName);
125    if (!lz4Frame.isValid()) {
126        errs() << "Invalid LZ4 file.\n";
127        return -1;
128    }
129
130    if (boost::filesystem::exists(outputFile)) {
131        if (overwriteOutput) {
132            boost::filesystem::remove(outputFile);
133        } else {
134            errs() << outputFile + " existed. Use -f argument to overwrite.\n";
135            return -1;
136        }
137    }
138
139    boost::iostreams::mapped_file_source mappedFile;
140    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
141    mappedFile.open(fileName, lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
142    char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
143    ParabixDriver pxDriver("lz4d");
144    generatePipeline(pxDriver);
145    auto main = reinterpret_cast<MainFunctionType>(pxDriver.getMain());
146
147    main(fileBuffer, lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
148
149    mappedFile.close();
150    return 0;
151}
Note: See TracBrowser for help on using the repository browser.