source: icGREP/icgrep-devel/icgrep/lzparabix/LZParabixGenerator.cpp @ 6133

Last change on this file since 6133 was 6133, checked in by xwa163, 11 months ago
  1. Add sourceCC in multiplexed CC
  2. Remove workaround FakeBasisBits? from ICGrep
  3. Implement Swizzled version of LZParabix
  4. Init checkin for SwizzleByGather? Kernel
File size: 10.0 KB
Line 
1
2#include "LZParabixGenerator.h"
3
4#include <boost/filesystem.hpp>
5#include <boost/iostreams/device/mapped_file.hpp>
6
7
8#include <cc/cc_compiler.h>
9
10#include <kernels/cc_kernel.h>
11#include <kernels/s2p_kernel.h>
12#include <kernels/p2s_kernel.h>
13#include <kernels/source_kernel.h>
14#include <kernels/stdout_kernel.h>
15#include <kernels/kernel_builder.h>
16#include <kernels/deletion.h>
17#include <kernels/swizzle.h>
18#include <kernels/pdep_kernel.h>
19#include <kernels/swizzled_multiple_pdep_kernel.h>
20#include <kernels/lzparabix/decoder/LZParabixBlockDecoder.h>
21#include <kernels/lzparabix/decoder/LZParabixBitStreamAioKernel.h>
22#include <kernels/lzparabix/decoder/LZParabixSwizzledAioKernel.h>
23#include <kernels/lzparabix/decoder/LZParabixLiteralDecoderKernel.h>
24
25namespace re { class CC; }
26
27using namespace llvm;
28using namespace parabix;
29using namespace kernel;
30
31LZParabixGenerator::LZParabixGenerator(): mPxDriver("lzParabixDecoder"), mLzParabixBlockSize(4 * 1024 * 1024) {
32
33}
34
35MainFunctionType LZParabixGenerator::getMainFunc() {
36    return reinterpret_cast<MainFunctionType>(mPxDriver.getMain());
37}
38
39
40
41void LZParabixGenerator::generatePipeline(const std::string &outputFile) {
42    auto & iBuilder = mPxDriver.getBuilder();
43    this->generateMainFunc(iBuilder);
44
45    this->generateLoadByteStreamAndBitStream(iBuilder);
46
47    auto decompressedBitStream = this->generateFullBitStreamDecompression(iBuilder);
48//    auto decompressedBitStream = this->generateAioBitStreamDecompressoin(iBuilder, {mCompressedBasisBits})[0];
49
50    auto decompressedByteStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
51    Kernel * p2sK = mPxDriver.addKernelInstance<P2SKernel>(iBuilder, cc::BitNumbering::BigEndian);
52    mPxDriver.makeKernelCall(p2sK, {decompressedBitStream}, {decompressedByteStream});
53
54    // --------------------------------------------------------
55    // End
56    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
57    outK->setInitialArguments({iBuilder->GetString(outputFile)});
58    mPxDriver.makeKernelCall(outK, {decompressedByteStream}, {});
59
60    mPxDriver.generatePipelineIR();
61    mPxDriver.deallocateBuffers();
62
63    iBuilder->CreateRetVoid();
64
65    mPxDriver.finalizeObject();
66}
67
68
69void LZParabixGenerator::generateBlockData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
70    BlockData_BlockStart = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder));
71    BlockData_BlockEnd = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks(iBuilder));
72    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZParabixBlockDecoderKernel>(iBuilder);
73    blockDecoderK->setInitialArguments({mFileSize});
74    mPxDriver.makeKernelCall(blockDecoderK, {mCompressedByteStream}, {BlockData_BlockStart, BlockData_BlockEnd});
75}
76
77parabix::StreamSetBuffer* LZParabixGenerator::extractLiteralBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
78    StreamSetBuffer * const LiteralBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
79    Kernel* literalDecoderK = mPxDriver.addKernelInstance<LZParabixLiteralDecoderKernel>(iBuilder);
80    literalDecoderK->setInitialArguments({mFileSize});
81    mPxDriver.makeKernelCall(literalDecoderK, {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd}, {LiteralBitStream});
82    return LiteralBitStream;
83}
84
85
86
87parabix::StreamSetBuffer* LZParabixGenerator::generateSwizzledBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, parabix::StreamSetBuffer* inputBitStreams) {
88
89    Kernel * swizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 2, 1);
90
91    StreamSetBuffer* inputSwizzled1 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
92    StreamSetBuffer* inputSwizzled2 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
93    mPxDriver.makeKernelCall(swizzleK, {inputBitStreams}, {inputSwizzled1, inputSwizzled2});
94
95
96    StreamSetBuffer* outputSwizzled1 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
97    StreamSetBuffer* outputSwizzled2 = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4, 1), this->getInputBufferBlocks(iBuilder));
98    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixSwizzledAioKernel>(iBuilder);
99    aioK->setInitialArguments({mFileSize});
100    mPxDriver.makeKernelCall(aioK, {
101            mCompressedByteStream,
102            BlockData_BlockStart,
103            BlockData_BlockEnd,
104
105//            inputBitStreams,
106            inputSwizzled1,
107            inputSwizzled2
108    }, {
109            outputSwizzled1,
110            outputSwizzled2
111    });
112
113    StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getDecompressedBufferBlocks(iBuilder));
114
115    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
116//    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleByGather>(iBuilder);
117    mPxDriver.makeKernelCall(unSwizzleK2, {outputSwizzled1, outputSwizzled2}, {decompressionBitStream});
118    return decompressionBitStream;
119}
120
121std::vector<parabix::StreamSetBuffer*> LZParabixGenerator::generateBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, std::vector<parabix::StreamSetBuffer*> inputBitStreams) {
122    std::vector<unsigned> numbersOfStreams;
123    std::vector<StreamSetBuffer*> inputStreams = {
124            mCompressedByteStream,
125            BlockData_BlockStart,
126            BlockData_BlockEnd,
127    };;
128    std::vector<StreamSetBuffer*> outputStreams;
129    for (unsigned i = 0; i < inputBitStreams.size(); i++) {
130        unsigned numOfStream = inputBitStreams[i]->getNumOfStreams();
131
132        numbersOfStreams.push_back(numOfStream);
133        inputStreams.push_back(inputBitStreams[i]);
134        outputStreams.push_back(mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(numOfStream, 1), this->getInputBufferBlocks(iBuilder)));
135    }
136
137    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixBitStreamAioKernel>(iBuilder, numbersOfStreams);
138    aioK->setInitialArguments({mFileSize});
139    mPxDriver.makeKernelCall(aioK, inputStreams, outputStreams);
140    return outputStreams;
141
142}
143
144parabix::StreamSetBuffer* LZParabixGenerator::generateFullBitStreamDecompression(const std::unique_ptr<kernel::KernelBuilder> & b) {
145    this->generateBlockData(b);
146    StreamSetBuffer * const LiteralBitStream = this->extractLiteralBitStream(b);
147    return this->generateBitStreamDecompression(b, {LiteralBitStream})[0];
148}
149
150
151std::vector<parabix::StreamSetBuffer*> LZParabixGenerator::generateAioBitStreamDecompressoin(
152        const std::unique_ptr<kernel::KernelBuilder> & iBuilder,
153        std::vector<parabix::StreamSetBuffer*> bitStreamSets
154) {
155    this->generateBlockData(iBuilder);
156
157    std::vector<parabix::StreamSetBuffer*> inputStreamSetParams = {mCompressedByteStream, BlockData_BlockStart, BlockData_BlockEnd};
158
159    std::vector<unsigned> numsOfStreams;
160    std::vector<parabix::StreamSetBuffer*> outputStreamSets;
161
162    for (unsigned i = 0; i < bitStreamSets.size(); i++) {
163        unsigned numOfStreams = bitStreamSets[i]->getNumOfStreams();
164        inputStreamSetParams.push_back(bitStreamSets[i]);
165        numsOfStreams.push_back(numOfStreams);
166        outputStreamSets.push_back(mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(numOfStreams, 1), this->getInputBufferBlocks(iBuilder)));
167    }
168
169    Kernel * aioK = mPxDriver.addKernelInstance<LZParabixBitStreamAioKernel>(iBuilder, numsOfStreams);
170    aioK->setInitialArguments({mFileSize});
171
172    mPxDriver.makeKernelCall(aioK, inputStreamSetParams, outputStreamSets);
173    return outputStreamSets;
174}
175
176
177void LZParabixGenerator::generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
178    mCompressedByteStream = mPxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
179//    mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
180
181    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
182    sourceK->setInitialArguments({mInputStream, mFileSize});
183    mPxDriver.makeKernelCall(sourceK, {}, {mCompressedByteStream});
184//    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::BigEndian);
185//    mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
186}
187
188void LZParabixGenerator::generateMainFunc(const std::unique_ptr<kernel::KernelBuilder> &iBuilder) {
189    Module * M = iBuilder->getModule();
190    Type * const sizeTy = iBuilder->getSizeTy();
191    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
192    Type * const voidTy = iBuilder->getVoidTy();
193    Type * const inputType = iBuilder->getInt8PtrTy();
194
195    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, sizeTy, boolTy, nullptr));
196    main->setCallingConv(CallingConv::C);
197    Function::arg_iterator args = main->arg_begin();
198    mInputStream = &*(args++);
199    mInputStream->setName("input");
200
201    mHeaderSize = &*(args++);
202    mHeaderSize->setName("mHeaderSize");
203
204    mFileSize = &*(args++);
205    mFileSize->setName("mFileSize");
206
207    mHasBlockChecksum = &*(args++);
208    mHasBlockChecksum->setName("mHasBlockChecksum");
209    // TODO for now, we do not handle blockCheckSum
210    mHasBlockChecksum = iBuilder->getInt1(false);
211
212    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
213}
214
215
216int LZParabixGenerator::get4MbBufferBlocks() {
217    return mLzParabixBlockSize / codegen::BlockSize;
218}
219
220int LZParabixGenerator::getInputBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & b) {
221    return this->get4MbBufferBlocks() * 2;
222}
223int LZParabixGenerator::getDecompressedBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & b) {
224    return this->get4MbBufferBlocks() * 2;
225}
226
Note: See TracBrowser for help on using the repository browser.