source: icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.h @ 6119

Last change on this file since 6119 was 6119, checked in by xwa163, 12 months ago
  1. Add some BasisSetNumbering? option to fix bug of multiplexing
  2. Use BiigEndian? BitNumbering? for lz4 and lzparabix related pipeline
  3. Support multiplexing in LZ4BitStreamAio pipeline
File size: 3.3 KB
Line 
1
2#ifndef ICGREP_LZ4GREPGENERATOR_H
3#define ICGREP_LZ4GREPGENERATOR_H
4
5#include "LZ4Generator.h"
6#include <grep_interface.h>
7#include <kernels/streamset.h>
8#include <cc/multiplex_CCs.h>
9#include <string>
10#include <vector>
11#include <sstream>
12#include <atomic>
13#include <grep/grep_engine.h>
14
15typedef void (*ScanMatchGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum, intptr_t match_accumulator);
16typedef uint64_t (*CountOnlyGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum);
17
18class LZ4GrepGenerator : public LZ4Generator{
19public:
20
21    LZ4GrepGenerator(bool enableMultiplexing = false);
22    void generateSwizzledCountOnlyGrepPipeline(re::RE *regex);
23    void generateCountOnlyGrepPipeline(re::RE *regex, bool enableGather = true);
24
25    void generateScanMatchGrepPipeline(re::RE* regex);
26    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(std::vector<re::RE *> &REs,
27                                                                                   parabix::StreamSetBuffer *decompressedBasisBits);
28
29    void generateMultiplexingCompressedBitStream(std::vector<re::RE *> &REs);
30    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> multiplexingGrepPipeline(std::vector<re::RE *> &REs, bool useAio = false, bool useSwizzled = true);
31
32
33    void invokeScanMatchGrep(char* fileBuffer, size_t blockStart, size_t blockEnd, bool hasBlockChecksum);
34
35    void generateMultiplexingSwizzledAioPipeline(re::RE *regex);
36    void generateSwizzledAioPipeline(re::RE* regex);
37    void generateMultiplexingBitStreamAioPipeline(re::RE* regex);
38    void generateBitStreamAioPipeline(re::RE* regex);
39
40    void generateAioPipeline(re::RE* regex);
41    void generateParallelAioPipeline(re::RE* regex, bool enableGather, bool enableScatter, int minParallelLevel);
42
43    ScanMatchGrepMainFunctionType getScanMatchGrepMainFunction();
44    CountOnlyGrepMainFunctionType getCountOnlyGrepMainFunction();
45
46private:
47    bool mEnableMultiplexing;
48
49    grep::GrepRecordBreakKind mGrepRecordBreak;
50    void initREs(std::vector<re::RE *> & REs);
51
52
53    re::CC * mBreakCC;
54    std::vector<re:: RE *> mREs;
55    std::set<re::Name *> mUnicodeProperties;
56    bool mMoveMatchesToEOL;
57
58
59    std::vector<std::ostringstream> mResultStrs;
60
61    void generateCountOnlyMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
62    void generateScanMatchMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
63
64
65    llvm::Value * match_accumulator;
66
67
68
69    std::unique_ptr<cc::MultiplexedAlphabet> mpx;
70
71    parabix::StreamSetBuffer * linefeedStreamFromDecompressedBits(parabix::StreamSetBuffer *decompressedBasisBits);
72    parabix::StreamSetBuffer * linefeedStreamFromCompressedBits();
73
74
75    parabix::StreamSetBuffer * convertCompressedBitsStreamWithSwizzledApproach(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
76    parabix::StreamSetBuffer * convertCompressedBitsStreamWithSwizzledAioApproach(
77            parabix::StreamSetBuffer *compressedBitStream, int numberOfStream, std::string prefix);
78
79
80    parabix::StreamSetBuffer * convertCompressedBitsStream(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
81};
82
83
84#endif //ICGREP_LZ4GREPGENERATOR_H
Note: See TracBrowser for help on using the repository browser.