source: icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.h @ 6111

Last change on this file since 6111 was 6111, checked in by xwa163, 10 months ago
  1. Cleanup LZ4 AIO related kernels
  2. Improve LZ4ParallelByteStreamAIOKernel
  3. Implement simd_cttz
File size: 3.1 KB
Line 
1
2#ifndef ICGREP_LZ4GREPGENERATOR_H
3#define ICGREP_LZ4GREPGENERATOR_H
4
5#include "LZ4Generator.h"
6#include <grep_interface.h>
7#include <kernels/streamset.h>
8#include <cc/multiplex_CCs.h>
9#include <string>
10#include <vector>
11#include <sstream>
12#include <atomic>
13#include <grep/grep_engine.h>
14
15typedef void (*ScanMatchGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum, intptr_t match_accumulator);
16typedef uint64_t (*CountOnlyGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum);
17
18class LZ4GrepGenerator : public LZ4Generator{
19public:
20
21    LZ4GrepGenerator(bool enableMultiplexing = false);
22    void generateSwizzledCountOnlyGrepPipeline(re::RE *regex);
23    void generateCountOnlyGrepPipeline(re::RE *regex, bool enableGather = true);
24
25    void generateScanMatchGrepPipeline(re::RE* regex);
26    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(std::vector<re::RE *> &REs,
27                                                                                   parabix::StreamSetBuffer *decompressedBasisBits);
28
29    void generateMultiplexingCompressedBitStream(std::vector<re::RE *> &REs);
30    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> multiplexingGrepPipeline(std::vector<re::RE *> &REs, bool useAio = false);
31
32
33    void invokeScanMatchGrep(char* fileBuffer, size_t blockStart, size_t blockEnd, bool hasBlockChecksum);
34
35    void generateMultiplexingSwizzledAioPipeline(re::RE *regex);
36
37    void generateSwizzledAioPipeline(re::RE* regex);
38
39    void generateAioPipeline(re::RE* regex);
40    void generateParallelAioPipeline(re::RE* regex, bool enableGather, bool enableScatter, int minParallelLevel);
41
42    ScanMatchGrepMainFunctionType getScanMatchGrepMainFunction();
43    CountOnlyGrepMainFunctionType getCountOnlyGrepMainFunction();
44
45private:
46    bool mEnableMultiplexing;
47
48    grep::GrepRecordBreakKind mGrepRecordBreak;
49    void initREs(std::vector<re::RE *> & REs);
50
51
52    re::CC * mBreakCC;
53    std::vector<re:: RE *> mREs;
54    std::set<re::Name *> mUnicodeProperties;
55    bool mMoveMatchesToEOL;
56
57
58    std::vector<std::ostringstream> mResultStrs;
59
60    void generateCountOnlyMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
61    void generateScanMatchMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
62
63
64    llvm::Value * match_accumulator;
65
66
67
68    std::unique_ptr<cc::MultiplexedAlphabet> mpx;
69
70    parabix::StreamSetBuffer * linefeedStreamFromDecompressedBits(parabix::StreamSetBuffer *decompressedBasisBits);
71    parabix::StreamSetBuffer * linefeedStreamFromCompressedBits();
72
73
74    parabix::StreamSetBuffer * convertCompressedBitsStreamWithSwizzledApproach(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
75    parabix::StreamSetBuffer * convertCompressedBitsStreamWithAioApproach(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
76    parabix::StreamSetBuffer * convertCompressedBitsStream(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
77};
78
79
80#endif //ICGREP_LZ4GREPGENERATOR_H
Note: See TracBrowser for help on using the repository browser.