source: icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.cpp @ 6137

Last change on this file since 6137 was 6137, checked in by xwa163, 11 months ago
  1. LZ4 ScanMatch? pipeline
  2. Refactor LZ4 Generator
  3. Adjust some naming
File size: 10.1 KB
Line 
1
2#include "LZParabixGenerator.h"
3
4#include <boost/filesystem.hpp>
5#include <boost/iostreams/device/mapped_file.hpp>
6
7
8#include <cc/cc_compiler.h>
9
10#include <kernels/cc_kernel.h>
11#include <kernels/s2p_kernel.h>
12#include <kernels/p2s_kernel.h>
13#include <kernels/source_kernel.h>
14#include <kernels/stdout_kernel.h>
15#include <kernels/kernel_builder.h>
16#include <kernels/deletion.h>
17#include <kernels/swizzle.h>
18#include <kernels/pdep_kernel.h>
19#include <kernels/swizzled_multiple_pdep_kernel.h>
20#include <kernels/lzparabix/decoder/LZParabixBlockDecoder.h>
21#include <kernels/lzparabix/decoder/LZParabixBitStreamAioKernel.h>
22#include <kernels/lzparabix/decoder/LZParabixSwizzledAioKernel.h>
23#include <kernels/lzparabix/decoder/LZParabixLiteralDecoderKernel.h>
24
25namespace re { class CC; }
26
27using namespace llvm;
28using namespace parabix;
29using namespace kernel;
30
31LZParabixGenerator::LZParabixGenerator(): mPxDriver("lzParabixDecoder"), mLzParabixBlockSize(4 * 1024 * 1024) {
32
33}
34
35MainFunctionType LZParabixGenerator::getMainFunc() {
36    return reinterpret_cast<MainFunctionType>(mPxDriver.getMain());
37}
38
39
40
41void LZParabixGenerator::generatePipeline(const std::string &outputFile) {
42    auto & iBuilder = mPxDriver.getBuilder();
43    this->generateMainFunc(iBuilder);
44
45    this->generateLoadByteStreamAndBitStream(iBuilder);
46
47    auto uncompressedBitStream = this->generateFullBitStreamDecompression(iBuilder);
48
49    auto uncompressedByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
50    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder, cc::BitNumbering::BigEndian);
51    mPxDriver.makeKernelCall(p2sK, {uncompressedBitStream}, {uncompressedByteStream});
52
53    // --------------------------------------------------------
54    // End
55    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
56    outK->setInitialArguments({iBuilder->GetString(outputFile)});
57    mPxDriver.makeKernelCall(outK, {uncompressedByteStream}, {});
58
59    mPxDriver.generatePipelineIR();
60    mPxDriver.deallocateBuffers();
61
62    iBuilder->CreateRetVoid();
63
64    mPxDriver.finalizeObject();
65}
66
67
68void LZParabixGenerator::generateBlockData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
69    BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder));
70    BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder));
71    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZParabixBlockDecoderKernel>(iBuilder);
72    blockDecoderK->setInitialArguments({mFileSize});
73    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_BlockStart, BlockData_BlockEnd});
74}
75
76parabix::StreamSetBuffer* LZParabixGenerator::extractLiteralBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
77    StreamSetBuffer * const LiteralBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
78    Kernel* literalDecoderK = mPxDriver.addKernelInstance<LZParabixLiteralDecoderKernel>(iBuilder);
79    literalDecoderK->setInitialArguments({mFileSize});
80    mPxDriver.makeKernelCall(literalDecoderK, {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd}, {LiteralBitStream});
81    return LiteralBitStream;
82}
83
84
85
86parabix::StreamSetBuffer* LZParabixGenerator::generateSwizzledBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, parabix::StreamSetBuffer* inputBitStreams) {
87
88    Kernel * swizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 2, 1);
89
90    StreamSetBuffer* inputSwizzled1 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
91    StreamSetBuffer* inputSwizzled2 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
92    mPxDriver.makeKernelCall(swizzleK, {inputBitStreams}, {inputSwizzled1, inputSwizzled2});
93
94
95    StreamSetBuffer* outputSwizzled1 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
96    StreamSetBuffer* outputSwizzled2 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
97    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixSwizzledAioKernel>(iBuilder);
98    aioK->setInitialArguments({mFileSize});
99    mPxDriver.makeKernelCall(aioK, {
100            mCompressedByteStream,
101            BlockData_BlockStart,
102            BlockData_BlockEnd,
103
104//            inputBitStreams,
105            inputSwizzled1,
106            inputSwizzled2
107    }, {
108            outputSwizzled1,
109            outputSwizzled2
110    });
111
112    StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1),
113                                                                                       this->getUncompressedBufferBlocks(
114                                                                                               iBuilder));
115
116    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
117//    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleByGather>(iBuilder);
118    mPxDriver.makeKernelCall(unSwizzleK2, {outputSwizzled1, outputSwizzled2}, {decompressionBitStream});
119    return decompressionBitStream;
120}
121
122std::vector<parabix::StreamSetBuffer*> LZParabixGenerator::generateBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, std::vector<parabix::StreamSetBuffer*> inputBitStreams) {
123    std::vector<unsigned> numbersOfStreams;
124    std::vector<StreamSetBuffer*> inputStreams = {
125            mCompressedByteStream,
126            BlockData_BlockStart,
127            BlockData_BlockEnd,
128    };;
129    std::vector<StreamSetBuffer*> outputStreams;
130    for (unsigned i = 0; i < inputBitStreams.size(); i++) {
131        unsigned numOfStream = inputBitStreams[i]->getNumOfStreams();
132
133        numbersOfStreams.push_back(numOfStream);
134        inputStreams.push_back(inputBitStreams[i]);
135        outputStreams.push_back(mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(numOfStream, 1), this->getInputBufferBlocks(iBuilder)));
136    }
137
138    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixBitStreamAioKernel>(iBuilder, numbersOfStreams);
139    aioK->setInitialArguments({mFileSize});
140    mPxDriver.makeKernelCall(aioK, inputStreams, outputStreams);
141    return outputStreams;
142
143}
144
145parabix::StreamSetBuffer* LZParabixGenerator::generateFullBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & b) {
146    this->generateBlockData(b);
147    StreamSetBuffer * const LiteralBitStream = this->extractLiteralBitStream(b);
148    return this->generateBitStreamDecompression(b, {LiteralBitStream})[0];
149}
150
151
152std::vector<parabix::StreamSetBuffer*> LZParabixGenerator::generateAioBitStreamDecompressoin(
153        const std::unique_ptr<kernel::KernelBuilder> & iBuilder,
154        std::vector<parabix::StreamSetBuffer*> bitStreamSets
155) {
156    this->generateBlockData(iBuilder);
157
158    std::vector<parabix::StreamSetBuffer*> inputStreamSetParams = {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd};
159
160    std::vector<unsigned> numsOfStreams;
161    std::vector<parabix::StreamSetBuffer*> outputStreamSets;
162
163    for (unsigned i = 0; i < bitStreamSets.size(); i++) {
164        unsigned numOfStreams = bitStreamSets[i]->getNumOfStreams();
165        inputStreamSetParams.push_back(bitStreamSets[i]);
166        numsOfStreams.push_back(numOfStreams);
167        outputStreamSets.push_back(mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(numOfStreams, 1), this->getInputBufferBlocks(iBuilder)));
168    }
169
170    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixBitStreamAioKernel>(iBuilder, numsOfStreams);
171    aioK->setInitialArguments({mFileSize});
172
173    mPxDriver.makeKernelCall(aioK, inputStreamSetParams, outputStreamSets);
174    return outputStreamSets;
175}
176
177
178void LZParabixGenerator::generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
179    mCompressedByteStream = mPxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
180//    mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getDefaultBufferBlocks(iBuilder));
181
182    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
183    sourceK->setInitialArguments({mInputStream, mFileSize});
184    mPxDriver.makeKernelCall(sourceK, {}, {mCompressedByteStream});
185//    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::BigEndian);
186//    mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
187}
188
189void LZParabixGenerator::generateMainFunc(const std::unique_ptr<kernel::KernelBuilder> &iBuilder) {
190    Module * M = iBuilder->getModule();
191    Type * const sizeTy = iBuilder->getSizeTy();
192    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
193    Type * const voidTy = iBuilder->getVoidTy();
194    Type * const inputType = iBuilder->getInt8PtrTy();
195
196    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, sizeTy, boolTy, nullptr));
197    main->setCallingConv(CallingConv::C);
198    Function::arg_iterator args = main->arg_begin();
199    mInputStream = &*(args++);
200    mInputStream->setName("input");
201
202    mHeaderSize = &*(args++);
203    mHeaderSize->setName("mHeaderSize");
204
205    mFileSize = &*(args++);
206    mFileSize->setName("mFileSize");
207
208    mHasBlockChecksum = &*(args++);
209    mHasBlockChecksum->setName("mHasBlockChecksum");
210    // TODO for now, we do not handle blockCheckSum
211    mHasBlockChecksum = iBuilder->getInt1(false);
212
213    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
214}
215
216
217int LZParabixGenerator::get4MbBufferBlocks() {
218    return mLzParabixBlockSize / codegen::BlockSize;
219}
220
221int LZParabixGenerator::getInputBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & b) {
222    return this->get4MbBufferBlocks() * 2;
223}
224int LZParabixGenerator::getUncompressedBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> &b) {
225    return this->get4MbBufferBlocks() * 2;
226}
227
Note: See TracBrowser for help on using the repository browser.