source: icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.cpp @ 6114

Last change on this file since 6114 was 6114, checked in by xwa163, 11 months ago

Init check in for new compression format (lzparabix) related kernels and pipelines, including compressor, decoder and grep

File size: 5.8 KB
RevLine 
[6114]1//
2// Created by wxy325 on 2018/6/18.
3//
4
5#include "LZParabixGenerator.h"
6
7#include <boost/filesystem.hpp>
8#include <boost/iostreams/device/mapped_file.hpp>
9
10
11#include <cc/cc_compiler.h>
12
13#include <kernels/cc_kernel.h>
14#include <kernels/s2p_kernel.h>
15#include <kernels/p2s_kernel.h>
16#include <kernels/source_kernel.h>
17#include <kernels/stdout_kernel.h>
18#include <kernels/kernel_builder.h>
19#include <kernels/deletion.h>
20#include <kernels/swizzle.h>
21#include <kernels/pdep_kernel.h>
22#include <kernels/swizzled_multiple_pdep_kernel.h>
23#include <kernels/lzparabix/LZParabixBlockDecoder.h>
24#include <kernels/lzparabix/LZParabixAioKernel.h>
25
26namespace re { class CC; }
27
28using namespace llvm;
29using namespace parabix;
30using namespace kernel;
31
32LZParabixGenerator::LZParabixGenerator(): mPxDriver("lzParabixDecoder"), mLzParabixBlockSize(4 * 1024 * 1024) {
33
34}
35
36MainFunctionType LZParabixGenerator::getMainFunc() {
37    return reinterpret_cast<MainFunctionType>(mPxDriver.getMain());
38}
39
40
41
42void LZParabixGenerator::generatePipeline(const std::string &outputFile) {
43    auto & iBuilder = mPxDriver.getBuilder();
44    this->generateMainFunc(iBuilder);
45
46    this->generateLoadByteStreamAndBitStream(iBuilder);
47    auto decompressedBitStream = this->generateAioBitStreamDecompressoin(iBuilder, {mCompressedBasisBits})[0];
48
49    auto decompressedByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
50    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder);
51    mPxDriver.makeKernelCall(p2sK, {decompressedBitStream}, {decompressedByteStream});
52
53    // --------------------------------------------------------
54    // End
55    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
56    outK->setInitialArguments({iBuilder->GetString(outputFile)});
57    mPxDriver.makeKernelCall(outK, {decompressedByteStream}, {});
58
59    mPxDriver.generatePipelineIR();
60    mPxDriver.deallocateBuffers();
61
62    iBuilder->CreateRetVoid();
63
64    mPxDriver.finalizeObject();
65}
66
67std::vector<parabix::StreamSetBuffer*> LZParabixGenerator::generateAioBitStreamDecompressoin(
68        const std::unique_ptr<kernel::KernelBuilder> & iBuilder,
69        std::vector<parabix::StreamSetBuffer*> bitStreamSets
70) {
71    StreamSetBuffer * const BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
72    StreamSetBuffer * const BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder), 1);
73    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZParabixBlockDecoderKernel>(iBuilder);
74    blockDecoderK->setInitialArguments({mFileSize});
75    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_BlockStart, BlockData_BlockEnd});
76
77    std::vector<parabix::StreamSetBuffer*> inputStreamSetParams = {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd};
78
79    std::vector<unsigned> numsOfStreams;
80    std::vector<parabix::StreamSetBuffer*> outputStreamSets;
81
82    for (unsigned i = 0; i < bitStreamSets.size(); i++) {
83        unsigned numOfStreams = bitStreamSets[i]->getNumOfStreams();
84        inputStreamSetParams.push_back(bitStreamSets[i]);
85        numsOfStreams.push_back(numOfStreams);
86        outputStreamSets.push_back(mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(numOfStreams, 1), this->getInputBufferBlocks(iBuilder)));
87    }
88
89    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixAioKernel>(iBuilder, numsOfStreams);
90    aioK->setInitialArguments({mFileSize});
91
92    mPxDriver.makeKernelCall(aioK, inputStreamSetParams, outputStreamSets);
93    return outputStreamSets;
94}
95
96
97void LZParabixGenerator::generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
98    mCompressedByteStream = mPxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
99    mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
100
101    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
102    sourceK->setInitialArguments({mInputStream, mFileSize});
103    mPxDriver.makeKernelCall(sourceK, {}, {mCompressedByteStream});
104    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::LittleEndian);
105    mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
106}
107
108void LZParabixGenerator::generateMainFunc(const std::unique_ptr<kernel::KernelBuilder> &iBuilder) {
109    Module * M = iBuilder->getModule();
110    Type * const sizeTy = iBuilder->getSizeTy();
111    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
112    Type * const voidTy = iBuilder->getVoidTy();
113    Type * const inputType = iBuilder->getInt8PtrTy();
114
115    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, sizeTy, boolTy, nullptr));
116    main->setCallingConv(CallingConv::C);
117    Function::arg_iterator args = main->arg_begin();
118    mInputStream = &*(args++);
119    mInputStream->setName("input");
120
121    mHeaderSize = &*(args++);
122    mHeaderSize->setName("mHeaderSize");
123
124    mFileSize = &*(args++);
125    mFileSize->setName("mFileSize");
126
127    mHasBlockChecksum = &*(args++);
128    mHasBlockChecksum->setName("mHasBlockChecksum");
129    // TODO for now, we do not handle blockCheckSum
130    mHasBlockChecksum = iBuilder->getInt1(false);
131
132    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
133}
134
135
136int LZParabixGenerator::get4MbBufferBlocks() {
137    return mLzParabixBlockSize / codegen::BlockSize;
138}
139
140int LZParabixGenerator::getInputBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & b) {
141    return this->get4MbBufferBlocks() * 2;
142}
143int LZParabixGenerator::getDecompressedBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & b) {
144    return this->get4MbBufferBlocks() * 2;
145}
146
Note: See TracBrowser for help on using the repository browser.