source: icGREP/icgrep-devel/icgrep/lz4d.cpp @ 6079

Last change on this file since 6079 was 6047, checked in by nmedfort, 17 months ago

Major refactoring of buffer types. Static buffers replace Circular and CircularCopyback?. External buffers unify Source/External?.

File size: 6.6 KB
Line 
1/*
2 *  Copyright (c) 2017 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7
8#include <llvm/IR/Module.h>
9#include <llvm/Linker/Linker.h>
10#include <llvm/Support/PrettyStackTrace.h>
11#include <llvm/Support/Signals.h>
12#include <llvm/Support/ManagedStatic.h>
13#include <toolchain/toolchain.h>
14
15#include <IR_Gen/idisa_target.h>
16#include <boost/filesystem.hpp>
17#include <boost/iostreams/device/mapped_file.hpp>
18
19#include <lz4FrameDecoder.h>
20#include <cc/cc_compiler.h>
21#include <kernels/cc_kernel.h>
22#include <kernels/streamset.h>
23#include <kernels/s2p_kernel.h>
24#include <kernels/source_kernel.h>
25#include <kernels/stdout_kernel.h>
26#include <kernels/lz4/lz4_index_decoder.h>
27#include <kernels/lz4/lz4_bytestream_decoder.h>
28
29#include <kernels/kernel_builder.h>
30#include <toolchain/cpudriver.h>
31#include <iostream>
32#include <llvm/Support/raw_ostream.h>
33namespace re { class CC; }
34
35using namespace llvm;
36using namespace parabix;
37using namespace kernel;
38
39static cl::OptionCategory lz4dFlags("Command Flags", "lz4d options");
40static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4dFlags));
41static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(lz4dFlags));
42static cl::opt<bool> overwriteOutput("f", cl::desc("Overwrite existing output file."), cl::init(false), cl::cat(lz4dFlags));
43
44typedef void (*MainFunctionType)(char * byte_data, size_t filesize, bool hasBlockChecksum);
45
46void generatePipeline(ParabixDriver & pxDriver) {
47    auto & iBuilder = pxDriver.getBuilder();
48    Module * M = iBuilder->getModule();
49
50    Type * const sizeTy = iBuilder->getSizeTy();
51    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
52    Type * const voidTy = iBuilder->getVoidTy();
53    Type * const inputType = iBuilder->getInt8PtrTy();
54   
55    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, boolTy, nullptr));
56    main->setCallingConv(CallingConv::C);
57    Function::arg_iterator args = main->arg_begin();
58    Value * const inputStream = &*(args++);
59    inputStream->setName("input");
60    Value * const fileSize = &*(args++);
61    fileSize->setName("fileSize");
62    Value * const hasBlockChecksum = &*(args++);
63    hasBlockChecksum->setName("hasBlockChecksum");
64
65    const unsigned segmentSize = codegen::SegmentSize;
66    const unsigned bufferSegments = codegen::BufferSegments * codegen::ThreadNum;
67    // Output buffer should be at least one whole LZ4 block (4MB) large in case of uncompressed blocks.
68    // And the size (in bytes) also needs to be a power of two.
69    const unsigned decompressBufBlocks = (4 * 1024 * 1024) / codegen::BlockSize;
70
71    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
72
73    StreamSetBuffer * const ByteStream = pxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
74    StreamSetBuffer * const BasisBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), segmentSize * bufferSegments);
75    StreamSetBuffer * const Extenders = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), segmentSize * bufferSegments);
76    StreamSetBuffer * const LiteralIndexes = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(2, 32), segmentSize * bufferSegments);
77    StreamSetBuffer * const MatchIndexes = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(2, 32), segmentSize * bufferSegments);
78    StreamSetBuffer * const DecompressedByteStream = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), decompressBufBlocks);
79
80   
81    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
82    sourceK->setInitialArguments({inputStream, fileSize});
83    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
84
85    // Input stream is not aligned due to the offset.
86    Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder, /*aligned = */ false);
87    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
88   
89    Kernel * extenderK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8);
90    pxDriver.makeKernelCall(extenderK, {BasisBits}, {Extenders});
91
92    Kernel * lz4iK = pxDriver.addKernelInstance<LZ4IndexDecoderKernel>(iBuilder);
93    lz4iK->setInitialArguments({iBuilder->CreateTrunc(hasBlockChecksum, iBuilder->getInt1Ty())});
94    pxDriver.makeKernelCall(lz4iK, {ByteStream, Extenders}, {LiteralIndexes, MatchIndexes});
95
96    Kernel * lz4bK = pxDriver.addKernelInstance<LZ4ByteStreamDecoderKernel>(iBuilder, decompressBufBlocks * codegen::BlockSize);
97    pxDriver.makeKernelCall(lz4bK, {LiteralIndexes, MatchIndexes, ByteStream}, {DecompressedByteStream});
98
99    Kernel * outK = pxDriver.addKernelInstance<FileSink>(iBuilder, 8);
100    outK->setInitialArguments({iBuilder->GetString(outputFile)});
101    pxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
102 
103    pxDriver.generatePipelineIR();
104
105    pxDriver.deallocateBuffers();
106
107    iBuilder->CreateRetVoid();
108 
109    pxDriver.finalizeObject();
110}
111
112int main(int argc, char *argv[]) {
113    // This boilerplate provides convenient stack traces and clean LLVM exit
114    // handling. It also initializes the built in support for convenient
115    // command line option handling.
116    sys::PrintStackTraceOnErrorSignal(argv[0]);
117    llvm::PrettyStackTraceProgram X(argc, argv);
118    llvm_shutdown_obj shutdown;
119    codegen::ParseCommandLineOptions(argc, argv, {&lz4dFlags, codegen::codegen_flags()});
120    std::string fileName = inputFile;
121    LZ4FrameDecoder lz4Frame(fileName);
122    if (!lz4Frame.isValid()) {
123        errs() << "Invalid LZ4 file.\n";
124        return -1;
125    }
126
127    if (boost::filesystem::exists(outputFile)) {
128        if (overwriteOutput) {
129            boost::filesystem::remove(outputFile);
130        } else {
131            errs() << outputFile + " existed. Use -f argument to overwrite.\n";
132            return -1;
133        }
134    }
135
136    boost::iostreams::mapped_file_source mappedFile;
137    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
138    mappedFile.open(fileName, lz4Frame.getBlocksLength() + lz4Frame.getBlocksStart());
139    char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
140    ParabixDriver pxDriver("lz4d");
141    generatePipeline(pxDriver);
142    auto main = reinterpret_cast<MainFunctionType>(pxDriver.getMain());
143
144    main(fileBuffer, lz4Frame.getBlocksLength(), lz4Frame.hasBlockChecksum());
145
146    mappedFile.close();
147    return 0;
148}
Note: See TracBrowser for help on using the repository browser.