source: icGREP/icgrep-devel/icgrep/lz4/lz4_base_generator.cpp @ 6143

Last change on this file since 6143 was 6143, checked in by xwa163, 11 months ago

lz4_grep: some bug fixing and refactor

File size: 7.9 KB
Line 
1
2
3#include "lz4_base_generator.h"
4#include <kernels/lz4/lz4_block_decoder.h>
5#include <kernels/kernel_builder.h>
6#include <kernels/source_kernel.h>
7#include <kernels/s2p_kernel.h>
8#include <kernels/swizzle.h>
9#include <kernels/lz4/twist_kernel.h>
10#include <kernels/lz4/untwist_kernel.h>
11#include <kernels/lz4/decompression/lz4_bytestream_decompression.h>
12#include <kernels/lz4/decompression/lz4_swizzled_decompression.h>
13#include <kernels/lz4/decompression/lz4_twist_decompression.h>
14#include <kernels/lz4/decompression/lz4_bitstream_decompression.h>
15
16
17using namespace llvm;
18using namespace parabix;
19using namespace kernel;
20
21LZ4BaseGenerator::LZ4BaseGenerator():mPxDriver("lz4"), mLz4BlockSize(4 * 1024 * 1024) {
22
23}
24
25StreamSetBuffer* LZ4BaseGenerator::loadByteStream() {
26    auto & b = mPxDriver.getBuilder();
27    StreamSetBuffer* byteStream = mPxDriver.addBuffer<ExternalBuffer>(b, b->getStreamSetTy(1, 8));
28    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(b);
29    sourceK->setInitialArguments({mInputStream, mFileSize});
30    mPxDriver.makeKernelCall(sourceK, {}, {byteStream});
31    return byteStream;
32}
33
34StreamSetBuffer* LZ4BaseGenerator::s2p(parabix::StreamSetBuffer* byteStream) {
35    auto & b = mPxDriver.getBuilder();
36    StreamSetBuffer* basisBits = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(8, 1),
37                                                             this->getDefaultBufferBlocks());
38    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(b, cc::BitNumbering::BigEndian);
39    mPxDriver.makeKernelCall(s2pk, {byteStream}, {basisBits});
40    return basisBits;
41}
42
43std::pair<parabix::StreamSetBuffer*, parabix::StreamSetBuffer*>  LZ4BaseGenerator::loadByteStreamAndBitStream() {
44    StreamSetBuffer* byteStream = this->loadByteStream();
45    StreamSetBuffer*  basisBits = s2p(byteStream);
46    return std::make_pair(byteStream, basisBits);
47}
48
49
50LZ4BlockInfo LZ4BaseGenerator::getBlockInfo(StreamSetBuffer* compressedByteStream) {
51    auto & b = mPxDriver.getBuilder();
52    LZ4BlockInfo blockInfo;
53    blockInfo.isCompress = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 8), this->getDefaultBufferBlocks(), 1);
54    blockInfo.blockStart = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 64),
55                                                             this->getDefaultBufferBlocks(), 1);
56    blockInfo.blockEnd = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 64), this->getDefaultBufferBlocks(), 1);
57
58    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZ4BlockDecoderKernel>(b);
59    blockDecoderK->setInitialArguments({b->CreateTrunc(mHasBlockChecksum, b->getInt1Ty()), mHeaderSize, mFileSize});
60    mPxDriver.makeKernelCall(blockDecoderK, {compressedByteStream}, {blockInfo.isCompress, blockInfo.blockStart, blockInfo.blockEnd});
61
62    return blockInfo;
63}
64
65StreamSetBuffer * LZ4BaseGenerator::byteStreamDecompression(StreamSetBuffer* compressedByteStream) {
66    auto & b = mPxDriver.getBuilder();
67    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
68
69    StreamSetBuffer *const decompressionByteStream =
70            mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 8),
71                                              this->getDefaultBufferBlocks(), 1);
72    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4ByteStreamDecompressionKernel>(b);
73    lz4AioK->setInitialArguments({mFileSize});
74    mPxDriver.makeKernelCall(
75            lz4AioK,
76            {
77                    compressedByteStream,
78
79                    // Block Data
80                    blockInfo.isCompress,
81                    blockInfo.blockStart,
82                    blockInfo.blockEnd
83            }, {
84                    decompressionByteStream
85            });
86
87    return decompressionByteStream;
88}
89
90StreamSetBuffer * LZ4BaseGenerator::swizzledDecompression(
91        StreamSetBuffer* compressedByteStream,
92        StreamSetBuffer* compressedBasisBits
93) {
94    auto & b = mPxDriver.getBuilder();
95    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
96
97    // Produce unswizzled bit streams
98    StreamSetBuffer * u16Swizzle0 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
99                                                                      this->getDefaultBufferBlocks(), 1);
100    StreamSetBuffer * u16Swizzle1 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
101                                                                      this->getDefaultBufferBlocks(), 1);
102    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(b, 8, 2, 1, 64, "source");
103    mPxDriver.makeKernelCall(unSwizzleK, {compressedBasisBits}, {u16Swizzle0, u16Swizzle1});
104
105
106
107    StreamSetBuffer * uncompressedSwizzled0 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
108                                                                                this->getDefaultBufferBlocks(), 1);
109    StreamSetBuffer * uncompressedSwizzled1 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
110                                                                                this->getDefaultBufferBlocks(), 1);
111
112
113    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4SwizzledDecompressionKernel>(b, 4, 2, 4);
114    lz4AioK->setInitialArguments({mFileSize});
115    mPxDriver.makeKernelCall(
116            lz4AioK,
117            {
118                    compressedByteStream,
119
120                    blockInfo.isCompress,
121                    blockInfo.blockStart,
122                    blockInfo.blockEnd,
123
124                    u16Swizzle0,
125                    u16Swizzle1
126            }, {
127                    uncompressedSwizzled0,
128                    uncompressedSwizzled1
129            });
130
131
132    StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(8, 1),
133                                                                                       this->getDefaultBufferBlocks(), 1);
134
135    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleGenerator>(b, 8, 1, 2);
136    mPxDriver.makeKernelCall(unSwizzleK2, {uncompressedSwizzled0, uncompressedSwizzled1}, {decompressionBitStream});
137
138    return decompressionBitStream;
139}
140
141StreamSetBuffer * LZ4BaseGenerator::bitStreamDecompression(
142        parabix::StreamSetBuffer* compressedByteStream,
143        parabix::StreamSetBuffer* compressedBasisBits
144) {
145    return this->convertCompressedBitsStreamWithBitStreamAioApproach(compressedByteStream, {compressedBasisBits})[0];
146}
147
148std::vector<StreamSetBuffer*> LZ4BaseGenerator::convertCompressedBitsStreamWithBitStreamAioApproach(
149        parabix::StreamSetBuffer* compressedByteStream,
150        std::vector<StreamSetBuffer*> compressedBitStreams
151) {
152    auto & b = mPxDriver.getBuilder();
153
154    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
155
156    std::vector<StreamSetBuffer *> inputStreams = {
157            compressedByteStream,
158
159            blockInfo.isCompress,
160            blockInfo.blockStart,
161            blockInfo.blockEnd,
162    };
163
164    std::vector<StreamSetBuffer *> outputStream;
165    std::vector<unsigned> numbersOfStreams;
166
167    for (unsigned i = 0; i < compressedBitStreams.size(); i++) {
168        unsigned numOfStreams = compressedBitStreams[i]->getNumOfStreams();
169        numbersOfStreams.push_back(numOfStreams);
170        inputStreams.push_back(compressedBitStreams[i]);
171        outputStream.push_back(mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(numOfStreams, 1),
172                                                                 this->getDefaultBufferBlocks(), 1));
173    }
174
175    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4BitStreamDecompressionKernel>(b, numbersOfStreams);
176    lz4AioK->setInitialArguments({mFileSize});
177    mPxDriver.makeKernelCall(lz4AioK, inputStreams, outputStream);
178
179    return outputStream;
180}
181
182unsigned LZ4BaseGenerator::getBlockSizeBufferBlocks() {
183    return mLz4BlockSize / codegen::BlockSize;
184}
185
186unsigned LZ4BaseGenerator::getDefaultBufferBlocks() {
187    return this->getBlockSizeBufferBlocks() * 2; // buffer 2 LZ4 Block By Default
188}
Note: See TracBrowser for help on using the repository browser.