source: icGREP/icgrep-devel/icgrep/lz4/lz4_base_generator.cpp @ 6173

Last change on this file since 6173 was 6173, checked in by nmedfort, 11 months ago

Added RE_Inspector.

Migrated RE passes to RE_Transformer.

Incorporated Memoizer functionality into RE_Transformer/Inspector.

Removed Memoizer.

Bug fix for unicode_set.

File size: 8.5 KB
Line 
1
2
3#include "lz4_base_generator.h"
4#include <kernels/lz4/lz4_block_decoder.h>
5#include <kernels/kernel_builder.h>
6#include <kernels/source_kernel.h>
7#include <kernels/s2p_kernel.h>
8#include <kernels/p2s_kernel.h>
9#include <kernels/swizzle.h>
10#include <kernels/lz4/twist_kernel.h>
11#include <kernels/lz4/untwist_kernel.h>
12#include <kernels/lz4/decompression/lz4_bytestream_decompression.h>
13#include <kernels/lz4/decompression/lz4_swizzled_decompression.h>
14#include <kernels/lz4/decompression/lz4_twist_decompression.h>
15#include <kernels/lz4/decompression/lz4_bitstream_decompression.h>
16
17using namespace llvm;
18using namespace parabix;
19using namespace kernel;
20
21LZ4BaseGenerator::LZ4BaseGenerator():mPxDriver("lz4"), mLz4BlockSize(4 * 1024 * 1024), mInitBlockInfo(false) {
22
23}
24
25StreamSetBuffer* LZ4BaseGenerator::loadByteStream() {
26    auto & b = mPxDriver.getBuilder();
27    StreamSetBuffer* byteStream = mPxDriver.addBuffer<ExternalBuffer>(b, b->getStreamSetTy(1, 8));
28    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(b);
29    sourceK->setInitialArguments({mInputStream, mFileSize});
30    mPxDriver.makeKernelCall(sourceK, {}, {byteStream});
31    return byteStream;
32}
33
34StreamSetBuffer* LZ4BaseGenerator::s2p(parabix::StreamSetBuffer* byteStream) {
35    auto & b = mPxDriver.getBuilder();
36    StreamSetBuffer* basisBits = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(8, 1),
37                                                             this->getDefaultBufferBlocks());
38    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(b, cc::BitNumbering::BigEndian);
39    mPxDriver.makeKernelCall(s2pk, {byteStream}, {basisBits});
40    return basisBits;
41}
42parabix::StreamSetBuffer* LZ4BaseGenerator::p2s(parabix::StreamSetBuffer* bitStream) {
43    auto & b = mPxDriver.getBuilder();
44    StreamSetBuffer* byteStream = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 8),
45                                                                   this->getDefaultBufferBlocks());
46    Kernel * p2sk = mPxDriver.addKernelInstance<P2SKernel>(b, cc::BitNumbering::BigEndian);
47    mPxDriver.makeKernelCall(p2sk, {bitStream}, {byteStream});
48    return byteStream;
49}
50
51std::pair<parabix::StreamSetBuffer*, parabix::StreamSetBuffer*>  LZ4BaseGenerator::loadByteStreamAndBitStream() {
52    StreamSetBuffer* byteStream = this->loadByteStream();
53    StreamSetBuffer*  basisBits = s2p(byteStream);
54    return std::make_pair(byteStream, basisBits);
55}
56
57LZ4BlockInfo LZ4BaseGenerator::getBlockInfo(StreamSetBuffer* compressedByteStream) {
58    if (mInitBlockInfo) {
59        return mBlockInfo;
60    }
61
62    auto & b = mPxDriver.getBuilder();
63    LZ4BlockInfo blockInfo;
64    blockInfo.isCompress = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 8), this->getDefaultBufferBlocks(), 1);
65    blockInfo.blockStart = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 64),
66                                                             this->getDefaultBufferBlocks(), 1);
67    blockInfo.blockEnd = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 64), this->getDefaultBufferBlocks(), 1);
68
69    Kernel * blockDecoderK = mPxDriver.addKernelInstance<LZ4BlockDecoderKernel>(b);
70    blockDecoderK->setInitialArguments({b->CreateTrunc(mHasBlockChecksum, b->getInt1Ty()), mHeaderSize, mFileSize});
71    mPxDriver.makeKernelCall(blockDecoderK, {compressedByteStream}, {blockInfo.isCompress, blockInfo.blockStart, blockInfo.blockEnd});
72
73    mInitBlockInfo = true;
74    mBlockInfo = blockInfo;
75    return blockInfo;
76}
77
78StreamSetBuffer * LZ4BaseGenerator::byteStreamDecompression(StreamSetBuffer* compressedByteStream) {
79    auto & b = mPxDriver.getBuilder();
80    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
81
82    StreamSetBuffer *const decompressionByteStream =
83            mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(1, 8),
84                                              this->getDefaultBufferBlocks(), 1);
85    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4ByteStreamDecompressionKernel>(b);
86    lz4AioK->setInitialArguments({mFileSize});
87    mPxDriver.makeKernelCall(
88            lz4AioK,
89            {
90                    compressedByteStream,
91
92                    // Block Data
93                    blockInfo.isCompress,
94                    blockInfo.blockStart,
95                    blockInfo.blockEnd
96            }, {
97                    decompressionByteStream
98            });
99
100    return decompressionByteStream;
101}
102
103StreamSetBuffer * LZ4BaseGenerator::swizzledDecompression(
104        StreamSetBuffer* compressedByteStream,
105        StreamSetBuffer* compressedBasisBits
106) {
107    auto & b = mPxDriver.getBuilder();
108    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
109
110    // Produce unswizzled bit streams
111    StreamSetBuffer * u16Swizzle0 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
112                                                                      this->getDefaultBufferBlocks(), 1);
113    StreamSetBuffer * u16Swizzle1 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
114                                                                      this->getDefaultBufferBlocks(), 1);
115    Kernel * unSwizzleK = mPxDriver.addKernelInstance<SwizzleGenerator>(b, 8, 2, 1, 64, "source");
116    mPxDriver.makeKernelCall(unSwizzleK, {compressedBasisBits}, {u16Swizzle0, u16Swizzle1});
117
118
119
120    StreamSetBuffer * uncompressedSwizzled0 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
121                                                                                this->getDefaultBufferBlocks(), 1);
122    StreamSetBuffer * uncompressedSwizzled1 = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(4),
123                                                                                this->getDefaultBufferBlocks(), 1);
124
125
126    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4SwizzledDecompressionKernel>(b, 4, 2, 4);
127    lz4AioK->setInitialArguments({mFileSize});
128    mPxDriver.makeKernelCall(
129            lz4AioK,
130            {
131                    compressedByteStream,
132
133                    blockInfo.isCompress,
134                    blockInfo.blockStart,
135                    blockInfo.blockEnd,
136
137                    u16Swizzle0,
138                    u16Swizzle1
139            }, {
140                    uncompressedSwizzled0,
141                    uncompressedSwizzled1
142            });
143
144
145    StreamSetBuffer * const decompressionBitStream = mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(8, 1),
146                                                                                       this->getDefaultBufferBlocks(), 1);
147
148    Kernel * unSwizzleK2 = mPxDriver.addKernelInstance<SwizzleGenerator>(b, 8, 1, 2);
149    mPxDriver.makeKernelCall(unSwizzleK2, {uncompressedSwizzled0, uncompressedSwizzled1}, {decompressionBitStream});
150
151    return decompressionBitStream;
152}
153
154StreamSetBuffer * LZ4BaseGenerator::bitStreamDecompression(
155        parabix::StreamSetBuffer* compressedByteStream,
156        parabix::StreamSetBuffer* compressedBasisBits
157) {
158    return this->convertCompressedBitsStreamWithBitStreamAioApproach(compressedByteStream, {compressedBasisBits})[0];
159}
160
161std::vector<StreamSetBuffer*> LZ4BaseGenerator::convertCompressedBitsStreamWithBitStreamAioApproach(
162        parabix::StreamSetBuffer* compressedByteStream,
163        std::vector<StreamSetBuffer*> compressedBitStreams
164) {
165    auto & b = mPxDriver.getBuilder();
166
167    LZ4BlockInfo blockInfo = this->getBlockInfo(compressedByteStream);
168
169    std::vector<StreamSetBuffer *> inputStreams = {
170            compressedByteStream,
171
172            blockInfo.isCompress,
173            blockInfo.blockStart,
174            blockInfo.blockEnd,
175    };
176
177    std::vector<StreamSetBuffer *> outputStream;
178    std::vector<unsigned> numbersOfStreams;
179
180    for (unsigned i = 0; i < compressedBitStreams.size(); i++) {
181        unsigned numOfStreams = compressedBitStreams[i]->getNumOfStreams();
182        numbersOfStreams.push_back(numOfStreams);
183        inputStreams.push_back(compressedBitStreams[i]);
184        outputStream.push_back(mPxDriver.addBuffer<StaticBuffer>(b, b->getStreamSetTy(numOfStreams, 1),
185                                                                 this->getDefaultBufferBlocks(), 1));
186    }
187
188    Kernel* lz4AioK = mPxDriver.addKernelInstance<LZ4BitStreamDecompressionKernel>(b, numbersOfStreams);
189    lz4AioK->setInitialArguments({mFileSize});
190    mPxDriver.makeKernelCall(lz4AioK, inputStreams, outputStream);
191
192    return outputStream;
193}
194
195unsigned LZ4BaseGenerator::getBlockSizeBufferBlocks() {
196    return mLz4BlockSize / codegen::BlockSize;
197}
198
199unsigned LZ4BaseGenerator::getDefaultBufferBlocks() {
200    return this->getBlockSizeBufferBlocks() * 2; // buffer 2 LZ4 Block By Default
201}
Note: See TracBrowser for help on using the repository browser.