source: icGREP/icgrep-devel/icgrep/lz4/LZ4Generator.h @ 6132

Last change on this file since 6132 was 6132, checked in by xwa163, 12 months ago
  1. More experiment on lz4 grep
  2. Improve performance of lzparabix grep
File size: 3.9 KB
Line 
1
2#ifndef ICGREP_LZ4GENERATOR_H
3#define ICGREP_LZ4GENERATOR_H
4#include <llvm/IR/Module.h>
5#include <llvm/IR/Function.h>
6
7#include <llvm/Support/Signals.h>
8#include <llvm/Support/ManagedStatic.h>
9#include <IR_Gen/idisa_target.h>
10
11#include <toolchain/toolchain.h>
12
13#include <toolchain/cpudriver.h>
14#include <string>
15
16namespace re { class CC; }
17
18
19typedef void (*MainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum);
20
21struct LZ4BlockInfo {
22    parabix::StreamSetBuffer* blockStart;
23    parabix::StreamSetBuffer* blockEnd;
24    parabix::StreamSetBuffer* isCompress;
25};
26
27class LZ4Generator {
28
29public:
30    LZ4Generator();
31
32    MainFunctionType getMainFunc();
33
34    void generatePipeline(const std::string &outputFile);
35    void generateSwizzledPipeline(const std::string &outputFile);
36    void generateNewExtractOnlyPipeline(const std::string &outputFile);
37    void generateExtractOnlyPipeline(const std::string &outputFile);
38    void generateSwizzledExtractOnlyPipeline(const std::string &outputFile);
39    void generateExtractAndDepositOnlyPipeline(const std::string &outputFile);
40    void generateSwizzledExtractAndDepositOnlyPipeline(const std::string &outputFile);
41
42protected:
43    //// Protected Method
44    std::vector<parabix::StreamSetBuffer*> convertCompressedBitsStreamWithBitStreamAioApproach(
45            std::vector<parabix::StreamSetBuffer*> compressedBitStreams, std::string prefix);
46
47    void generateMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
48    // Pipeline
49    virtual void generateLoadByteStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
50    virtual void generateLoadByteStreamAndBitStream(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
51    virtual void generateExtractAndDepositMarkers(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
52    virtual parabix::StreamSetBuffer * generateParallelAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, bool enableGather, bool enableScatter, int minParallelLevel);
53    virtual parabix::StreamSetBuffer * generateAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
54    virtual parabix::StreamSetBuffer * generateSwizzledAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
55    virtual parabix::StreamSetBuffer * generateBitStreamAIODecompression(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
56
57    virtual std::pair<parabix::StreamSetBuffer*, parabix::StreamSetBuffer*> generateSwizzleExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
58    virtual parabix::StreamSetBuffer* generateBitStreamExtractData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
59
60    void generateCompressionMarker(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
61
62    // BufferSize related Helper Function
63    virtual int getInputBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
64    virtual int getDecompressedBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
65    int get4MbBufferBlocks();
66
67
68    //// Data Member
69    // Driver
70    ParabixDriver mPxDriver;
71
72    // Runtime Arguments
73    llvm::Value * mInputStream;
74    llvm::Value * mHeaderSize;
75    llvm::Value * mFileSize;
76    llvm::Value * mHasBlockChecksum;
77
78
79    // StreamSetBuffers
80    parabix::StreamSetBuffer * mCompressedByteStream;
81    parabix::StreamSetBuffer * mCompressedBasisBits;
82    parabix::StreamSetBuffer * mDeletionMarker;
83    parabix::StreamSetBuffer * mCompressionMarker;
84    parabix::StreamSetBuffer * mDepositMarker;
85    parabix::StreamSetBuffer * mMatchOffsetMarker;
86
87    // M0CountMarker will not contain anything, it will only be used to pass producedItemCount and manage processedItemCount between different kernel
88    parabix::StreamSetBuffer * mM0Marker;
89
90    unsigned mLz4BlockSize;
91
92    LZ4BlockInfo getBlockInfo(const std::unique_ptr<kernel::KernelBuilder> & b);
93};
94
95
96#endif //ICGREP_LZ4GENERATOR_H
Note: See TracBrowser for help on using the repository browser.