source: icGREP/icgrep-devel/icgrep/lz4/LZ4GeneratorNew.cpp @ 5921

Last change on this file since 5921 was 5921, checked in by xwa163, 13 months ago
  1. Initial checkin for new approach for lz4 index decoder that always use 4MB buffer
  2. Add test case for new approach (for now test cases will fail when test file is larger than 4MB)
File size: 6.3 KB
Line 
1
2
3#include "LZ4GeneratorNew.h"
4
5#include <boost/filesystem.hpp>
6#include <boost/iostreams/device/mapped_file.hpp>
7
8#include <llvm/Support/CommandLine.h>
9#include <llvm/Support/PrettyStackTrace.h>
10
11#include <cc/cc_compiler.h>
12
13#include <lz4FrameDecoder.h>
14#include <kernels/streamset.h>
15#include <kernels/cc_kernel.h>
16#include <kernels/s2p_kernel.h>
17#include <kernels/p2s_kernel.h>
18#include <kernels/source_kernel.h>
19#include <kernels/stdout_kernel.h>
20#include <kernels/lz4/lz4_extract_e_m0.h>
21#include <kernels/lz4/lz4_generate_deposit_stream.h>
22#include <kernels/lz4/lz4_numbers_to_bitstream_kernel.h>
23#include <kernels/lz4/lz4_bitstream_not_kernel.h>
24#include <kernels/kernel_builder.h>
25#include <kernels/lz4/lz4_block_decoder.h>
26#include <kernels/deletion.h>
27#include <kernels/swizzle.h>
28#include <kernels/pdep_kernel.h>
29#include <kernels/lz4/lz4_multiple_pdep_kernel.h>
30#include <kernels/lz4/lz4_match_copy_kernel.h>
31#include <kernels/lz4/lz4_swizzled_match_copy_kernel.h>
32#include <kernels/lz4/lz4_block_decoder_new.h>
33#include <kernels/lz4/lz4_index_builder.h>
34
35namespace re { class CC; }
36
37using namespace llvm;
38using namespace parabix;
39using namespace kernel;
40
41LZ4GeneratorNew::LZ4GeneratorNew():LZ4Generator() {
42
43}
44
45int LZ4GeneratorNew::get4MbBufferBlocks() {
46    return 4 * 1024 * 1024 / codegen::BlockSize;
47}
48
49int LZ4GeneratorNew::getInputBufferBlocks() {
50    return this->get4MbBufferBlocks();
51}
52int LZ4GeneratorNew::getDecompressedBufferBlocks() {
53    return this->get4MbBufferBlocks();
54}
55
56
57void LZ4GeneratorNew::generateExtractAndDepositMarkers(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
58    //// Decode Block Information
59    StreamSetBuffer * const BlockData_IsCompressed = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->get4MbBufferBlocks());
60    StreamSetBuffer * const BlockData_BlockStart = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->get4MbBufferBlocks());
61    StreamSetBuffer * const BlockData_BlockEnd = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->get4MbBufferBlocks());
62
63    //// Generate Helper Markers Extenders, FX, XF
64    StreamSetBuffer * const Extenders = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->get4MbBufferBlocks());
65    StreamSetBuffer * const CC_0xFX = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->get4MbBufferBlocks());
66    StreamSetBuffer * const CC_0xXF = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), this->get4MbBufferBlocks());
67
68
69    Kernel * extenderK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(0xFF)}, 8);
70    pxDriver.makeKernelCall(extenderK, {BasisBits}, {Extenders});
71
72    re::CC* xfCC = re::makeCC(0x0f);
73    re::CC* fxCC = re::makeCC(0xf0);
74    for (re::codepoint_t i = 1; i <= 0xf; i++) {
75        xfCC = re::makeCC(xfCC, re::makeCC(i * 0x10 + 0x0f));
76        fxCC = re::makeCC(fxCC, re::makeCC(0xf0 + i));
77    }
78
79    Kernel * CC_0xFXKernel = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "CC_0xFX", std::vector<re::CC *>{fxCC}, 8);
80    pxDriver.makeKernelCall(CC_0xFXKernel, {BasisBits}, {CC_0xFX});
81
82    Kernel * CC_0xXFKernel = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "CC_0xXF", std::vector<re::CC *>{xfCC}, 8);
83    pxDriver.makeKernelCall(CC_0xXFKernel, {BasisBits}, {CC_0xXF});
84
85
86    //// Generate Extract/Deposit Markers, M0_Start, M0_End, MatchOffset
87
88    size_t m0BufferSize = this->getDecompressedBufferBlocks() * 2;
89    size_t e1BufferSize = this->getInputBufferBlocks();
90
91    M0_Start = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks());
92    M0_End = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks());
93
94    //TODO handle uncompressed part
95    StreamSetBuffer * const UncompressedStartPos = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->get4MbBufferBlocks());
96    StreamSetBuffer * const UncompressedLength = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->get4MbBufferBlocks());
97    StreamSetBuffer * const UncompressedOutputPos = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->get4MbBufferBlocks());
98
99    EMarker = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), e1BufferSize);
100    StreamSetBuffer * const M0Marker = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), m0BufferSize);
101    DepositMarker = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), m0BufferSize);
102    Match_Offset = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), this->getInputBufferBlocks());
103
104
105    Kernel * blockDecoderK = pxDriver.addKernelInstance<LZ4BlockDecoderNewKernel>(iBuilder);
106    blockDecoderK->setInitialArguments({iBuilder->CreateTrunc(hasBlockChecksum, iBuilder->getInt1Ty()), headerSize});
107    pxDriver.makeKernelCall(blockDecoderK, {ByteStream, Extenders}, {BlockData_IsCompressed, BlockData_BlockStart, BlockData_BlockEnd});
108
109
110
111    Kernel* Lz4IndexBuilderK = pxDriver.addKernelInstance<LZ4IndexBuilderKernel>(iBuilder);
112    pxDriver.makeKernelCall(
113            Lz4IndexBuilderK,
114            {
115                    ByteStream,
116                    Extenders,
117                    CC_0xFX,
118                    CC_0xXF,
119
120                    // Block Data
121                    BlockData_IsCompressed,
122                    BlockData_BlockStart,
123                    BlockData_BlockEnd
124            }, {
125                    //Uncompressed Data
126                    UncompressedStartPos,
127                    UncompressedLength,
128                    UncompressedOutputPos,
129
130                    EMarker,
131                    M0_Start,
132                    M0_End,
133                    Match_Offset
134            });
135
136
137    Kernel * buildM0StartMarkerK = pxDriver.addKernelInstance<LZ4NumbersToBitstreamKernel>("buildM0Marker", iBuilder);
138    pxDriver.makeKernelCall(buildM0StartMarkerK, {M0_Start, M0_End}, {M0Marker});
139
140
141    Kernel * generateDepositK = pxDriver.addKernelInstance<LZ4GenerateDepositStreamKernel>(iBuilder);
142    pxDriver.makeKernelCall(generateDepositK, {M0Marker}, {DepositMarker}); // TODO deposit
143
144}
Note: See TracBrowser for help on using the repository browser.