source: icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.h @ 6132

Last change on this file since 6132 was 6132, checked in by xwa163, 11 months ago
  1. More experiment on lz4 grep
  2. Improve performance of lzparabix grep
File size: 3.5 KB
Line 
1
2#ifndef ICGREP_LZ4GREPGENERATOR_H
3#define ICGREP_LZ4GREPGENERATOR_H
4
5#include "LZ4Generator.h"
6#include <grep_interface.h>
7#include <kernels/streamset.h>
8#include <cc/multiplex_CCs.h>
9#include <string>
10#include <vector>
11#include <sstream>
12#include <atomic>
13#include <grep/grep_engine.h>
14
15typedef void (*ScanMatchGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum, intptr_t match_accumulator);
16typedef uint64_t (*CountOnlyGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum);
17
18class LZ4GrepGenerator : public LZ4Generator{
19public:
20
21    LZ4GrepGenerator(bool enableMultiplexing = false);
22    void generateSwizzledCountOnlyGrepPipeline(re::RE *regex);
23    void generateCountOnlyGrepPipeline(re::RE *regex, bool enableGather = true);
24
25    void generateScanMatchGrepPipeline(re::RE* regex);
26    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(std::vector<re::RE *> &REs,
27                                                                                   parabix::StreamSetBuffer *decompressedBasisBits);
28
29    void generateMultiplexingCompressedBitStream(std::vector<re::RE *> &REs);
30    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> multiplexingGrepPipeline(std::vector<re::RE *> &REs, bool useAio = false, bool useSwizzled = true, bool useByteStream = false);
31
32
33    void invokeScanMatchGrep(char* fileBuffer, size_t blockStart, size_t blockEnd, bool hasBlockChecksum);
34
35    void generateMultiplexingSwizzledAioPipeline(re::RE *regex);
36    void generateSwizzledAioPipeline(re::RE* regex);
37    void generateMultiplexingBitStreamAioPipeline(re::RE* regex);
38    void generateBitStreamAioPipeline(re::RE* regex);
39
40    void generateByteStreamMultiplexingAioPipeline(re::RE* regex);
41    void generateAioPipeline(re::RE* regex);
42    void generateParallelAioPipeline(re::RE* regex, bool enableGather, bool enableScatter, int minParallelLevel);
43
44    ScanMatchGrepMainFunctionType getScanMatchGrepMainFunction();
45    CountOnlyGrepMainFunctionType getCountOnlyGrepMainFunction();
46
47private:
48    bool mEnableMultiplexing;
49
50    grep::GrepRecordBreakKind mGrepRecordBreak;
51    void initREs(std::vector<re::RE *> & REs);
52
53
54    re::CC * mBreakCC;
55    std::vector<re:: RE *> mREs;
56    std::set<re::Name *> mUnicodeProperties;
57    bool mMoveMatchesToEOL;
58
59
60    std::vector<std::ostringstream> mResultStrs;
61
62    void generateCountOnlyMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
63    void generateScanMatchMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
64
65
66    llvm::Value * match_accumulator;
67
68
69
70    std::unique_ptr<cc::MultiplexedAlphabet> mpx;
71
72    parabix::StreamSetBuffer * linefeedStreamFromDecompressedBits(parabix::StreamSetBuffer *decompressedBasisBits);
73    parabix::StreamSetBuffer * linefeedStreamFromCompressedBits();
74
75
76    parabix::StreamSetBuffer * convertCompressedBitsStreamWithSwizzledApproach(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
77    parabix::StreamSetBuffer * convertCompressedBitsStreamWithSwizzledAioApproach(
78            parabix::StreamSetBuffer *compressedBitStream, int numberOfStream, std::string prefix);
79    parabix::StreamSetBuffer * convertCompressedBitsStreamWithByteStreamAioApproach(
80            parabix::StreamSetBuffer *compressedBitStream, int numberOfStream, std::string prefix);
81
82
83    parabix::StreamSetBuffer * convertCompressedBitsStream(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
84};
85
86
87#endif //ICGREP_LZ4GREPGENERATOR_H
Note: See TracBrowser for help on using the repository browser.