source: icGREP/icgrep-devel/icgrep/lz4/lz4_base_generator.cpp @ 6150

Last change on this file since 6150 was 6150, checked in by xwa163, 9 months ago
  1. Remove LZParabix related codes
  2. Enable multiplexing for LZ4 ScanMatch? pipeline
  3. Some minor bug fixing
File size: 8.5 KB
Line 
1
2
3#include "lz4_base_generator.h"
4#include <kernels/lz4/lz4_block_decoder.h>
5#include <kernels/kernel_builder.h>
6#include <kernels/source_kernel.h>
7#include <kernels/s2p_kernel.h>
8#include <kernels/p2s_kernel.h>
9#include <kernels/swizzle.h>
10#include <kernels/lz4/twist_kernel.h>
11#include <kernels/lz4/untwist_kernel.h>
12#include <kernels/lz4/decompression/lz4_bytestream_decompression.h>
13#include <kernels/lz4/decompression/lz4_swizzled_decompression.h>
14#include <kernels/lz4/decompression/lz4_twist_decompression.h>
15#include <kernels/lz4/decompression/lz4_bitstream_decompression.h>
16
17
18using namespace llvm;
19using namespace parabix;
20using namespace kernel;
21
22LZ4BaseGenerator::LZ4BaseGenerator():mPxDriver("lz4"), mLz4BlockSize(4 * 1024 * 1024), mInitBlockInfo(false) {
23
24}
25
26StreamSetBuffer* LZ4BaseGenerator::loadByteStream() {
27    auto & b = mPxDriver.getBuilder();
28    StreamSetBuffer* byteStream = mPxDriver.addBuffer<ExternalBuffer>(b, b->getStreamSetTy(1, 8));
29    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(b);
30    sourceK->setInitialArguments({mInputStream, mFileSize});
31    mPxDriver.makeKernelCall(sourceK, {}, {byteStream});
32    return byteStream;
33}
34
35StreamSetBuffer* LZ4BaseGenerator::s2p(parabix::StreamSetBuffer* byteStream) {
36    auto & b = mPxDriver.getBuilder();
37    StreamSetBuffer* basisBits = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(8, 1),
38                                                             this->getDefaultBufferBlocks());
39    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(b, cc::BitNumbering::BigEndian);
40    mPxDriver.makeKernelCall(s2pk, {byteStream}, {basisBits});
41    return basisBits;
42}
43parabix::StreamSetBuffer* LZ4BaseGenerator::p2s(parabix::StreamSetBuffer* bitStream) {
44    auto & b = mPxDriver.getBuilder();
45    StreamSetBuffer* byteStream = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 8),
46                                                                   this->getDefaultBufferBlocks());
47    Kernel * p2sk = mPxDriver.addKernelInstance<P2SKernel>(b, cc::BitNumbering::BigEndian);
48    mPxDriver.makeKernelCall(p2sk, {bitStream}, {byteStream});
49    return byteStream;
50}
51
52std::pair<parabix::StreamSetBuffer*, parabix::StreamSetBuffer*>  LZ4BaseGenerator::loadByteStreamAndBitStream() {
53    StreamSetBuffer* byteStream = this->loadByteStream();
54    StreamSetBuffer*  basisBits = s2p(byteStream);
55    return std::make_pair(byteStream, basisBits);
56}
57
58LZ4BlockInfo LZ4BaseGenerator::getBlockInfo(StreamSetBuffer* compressedByteStream) {
59    if (mInitBlockInfo) {
60        return mBlockInfo;
61    }
62
63    auto & b = mPxDriver.getBuilder();
64    LZ4BlockInfo blockInfo;
65    blockInfo.isCompress = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 8), this->getDefaultBufferBlocks(), 1);
66    blockInfo.blockStart = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 64),
67                                                             this->getDefaultBufferBlocks(), 1);
68    blockInfo.blockEnd = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 64), this->getDefaultBufferBlocks(), 1);
69
70    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZ4BlockDecoderKernel>(b);
71    blockDecoderK->setInitialArguments({b->CreateTrunc(mHasBlockChecksum, b->getInt1Ty()), mHeaderSize, mFileSize});
72    mPxDriver.makeKernelCall(blockDecoderK, {compressedByteStream}, {blockInfo.isCompress, blockInfo.blockStart, blockInfo.blockEnd});
73
74    mInitBlockInfo = true;
75    mBlockInfo = blockInfo;
76    return blockInfo;
77}
78
79StreamSetBuffer * LZ4BaseGenerator::byteStreamDecompression(StreamSetBuffer* compressedByteStream) {
80    auto & b = mPxDriver.getBuilder();
81    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
82
83    StreamSetBuffer *const decompressionByteStream =
84            mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 8),
85                                              this->getDefaultBufferBlocks(), 1);
86    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4ByteStreamDecompressionKernel>(b);
87    lz4AioK->setInitialArguments({mFileSize});
88    mPxDriver.makeKernelCall(
89            lz4AioK,
90            {
91                    compressedByteStream,
92
93                    // Block Data
94                    blockInfo.isCompress,
95                    blockInfo.blockStart,
96                    blockInfo.blockEnd
97            }, {
98                    decompressionByteStream
99            });
100
101    return decompressionByteStream;
102}
103
104StreamSetBuffer * LZ4BaseGenerator::swizzledDecompression(
105        StreamSetBuffer* compressedByteStream,
106        StreamSetBuffer* compressedBasisBits
107) {
108    auto & b = mPxDriver.getBuilder();
109    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
110
111    // Produce unswizzled bit streams
112    StreamSetBuffer * u16Swizzle0 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
113                                                                      this->getDefaultBufferBlocks(), 1);
114    StreamSetBuffer * u16Swizzle1 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
115                                                                      this->getDefaultBufferBlocks(), 1);
116    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(b, 8, 2, 1, 64, "source");
117    mPxDriver.makeKernelCall(unSwizzleK, {compressedBasisBits}, {u16Swizzle0, u16Swizzle1});
118
119
120
121    StreamSetBuffer * uncompressedSwizzled0 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
122                                                                                this->getDefaultBufferBlocks(), 1);
123    StreamSetBuffer * uncompressedSwizzled1 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
124                                                                                this->getDefaultBufferBlocks(), 1);
125
126
127    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4SwizzledDecompressionKernel>(b, 4, 2, 4);
128    lz4AioK->setInitialArguments({mFileSize});
129    mPxDriver.makeKernelCall(
130            lz4AioK,
131            {
132                    compressedByteStream,
133
134                    blockInfo.isCompress,
135                    blockInfo.blockStart,
136                    blockInfo.blockEnd,
137
138                    u16Swizzle0,
139                    u16Swizzle1
140            }, {
141                    uncompressedSwizzled0,
142                    uncompressedSwizzled1
143            });
144
145
146    StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(8, 1),
147                                                                                       this->getDefaultBufferBlocks(), 1);
148
149    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleGenerator>(b, 8, 1, 2);
150    mPxDriver.makeKernelCall(unSwizzleK2, {uncompressedSwizzled0, uncompressedSwizzled1}, {decompressionBitStream});
151
152    return decompressionBitStream;
153}
154
155StreamSetBuffer * LZ4BaseGenerator::bitStreamDecompression(
156        parabix::StreamSetBuffer* compressedByteStream,
157        parabix::StreamSetBuffer* compressedBasisBits
158) {
159    return this->convertCompressedBitsStreamWithBitStreamAioApproach(compressedByteStream, {compressedBasisBits})[0];
160}
161
162std::vector<StreamSetBuffer*> LZ4BaseGenerator::convertCompressedBitsStreamWithBitStreamAioApproach(
163        parabix::StreamSetBuffer* compressedByteStream,
164        std::vector<StreamSetBuffer*> compressedBitStreams
165) {
166    auto & b = mPxDriver.getBuilder();
167
168    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
169
170    std::vector<StreamSetBuffer *> inputStreams = {
171            compressedByteStream,
172
173            blockInfo.isCompress,
174            blockInfo.blockStart,
175            blockInfo.blockEnd,
176    };
177
178    std::vector<StreamSetBuffer *> outputStream;
179    std::vector<unsigned> numbersOfStreams;
180
181    for (unsigned i = 0; i < compressedBitStreams.size(); i++) {
182        unsigned numOfStreams = compressedBitStreams[i]->getNumOfStreams();
183        numbersOfStreams.push_back(numOfStreams);
184        inputStreams.push_back(compressedBitStreams[i]);
185        outputStream.push_back(mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(numOfStreams, 1),
186                                                                 this->getDefaultBufferBlocks(), 1));
187    }
188
189    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4BitStreamDecompressionKernel>(b, numbersOfStreams);
190    lz4AioK->setInitialArguments({mFileSize});
191    mPxDriver.makeKernelCall(lz4AioK, inputStreams, outputStream);
192
193    return outputStream;
194}
195
196unsigned LZ4BaseGenerator::getBlockSizeBufferBlocks() {
197    return mLz4BlockSize / codegen::BlockSize;
198}
199
200unsigned LZ4BaseGenerator::getDefaultBufferBlocks() {
201    return this->getBlockSizeBufferBlocks() * 2; // buffer 2 LZ4 Block By Default
202}
Note: See TracBrowser for help on using the repository browser.