source: icGREP/icgrep-devel/icgrep/lz4/LZ4GrepGenerator.h @ 6081

Last change on this file since 6081 was 6081, checked in by xwa163, 11 months ago
  1. Add command line parameter -enable-gather and -enable-scatter in lz4 parallel grep
  2. fix some bugs for AVX512 machine
File size: 3.1 KB
Line 
1
2#ifndef ICGREP_LZ4GREPGENERATOR_H
3#define ICGREP_LZ4GREPGENERATOR_H
4
5#include "LZ4Generator.h"
6#include <grep_interface.h>
7#include <kernels/streamset.h>
8#include <cc/multiplex_CCs.h>
9#include <string>
10#include <vector>
11#include <sstream>
12#include <atomic>
13#include <grep/grep_engine.h>
14
15typedef void (*ScanMatchGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum, intptr_t match_accumulator);
16typedef uint64_t (*CountOnlyGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum);
17
18class LZ4GrepGenerator : public LZ4Generator{
19public:
20
21    LZ4GrepGenerator(bool enableMultiplexing = false);
22    void generateSwizzledCountOnlyGrepPipeline(re::RE *regex);
23    void generateCountOnlyGrepPipeline(re::RE *regex, bool enableGather = true);
24
25    void generateScanMatchGrepPipeline(re::RE* regex);
26    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(std::vector<re::RE *> &REs,
27                                                                                   parabix::StreamSetBuffer *decompressedBasisBits);
28
29    void generateMultiplexingCompressedBitStream(std::vector<re::RE *> &REs);
30    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> multiplexingGrepPipeline(std::vector<re::RE *> &REs, bool useAio = false);
31
32
33    void invokeScanMatchGrep(char* fileBuffer, size_t blockStart, size_t blockEnd, bool hasBlockChecksum);
34
35    void generateMultiplexingSwizzledAioPipeline(re::RE* regex);
36    void generateMultiplexingSwizzledAioPipeline2(re::RE* regex);
37
38    void generateSwizzledAioPipeline(re::RE* regex);
39
40    void generateAioPipeline(re::RE* regex);
41    void generateParallelAioPipeline(re::RE* regex, bool enableGather, bool enableScatter);
42
43    ScanMatchGrepMainFunctionType getScanMatchGrepMainFunction();
44    CountOnlyGrepMainFunctionType getCountOnlyGrepMainFunction();
45
46private:
47    bool mEnableMultiplexing;
48
49    grep::GrepRecordBreakKind mGrepRecordBreak;
50    void initREs(std::vector<re::RE *> & REs);
51
52
53    re::CC * mBreakCC;
54    std::vector<re:: RE *> mREs;
55    std::set<re::Name *> mUnicodeProperties;
56    bool mMoveMatchesToEOL;
57
58
59    std::vector<std::ostringstream> mResultStrs;
60
61    void generateCountOnlyMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
62    void generateScanMatchMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
63
64
65    llvm::Value * match_accumulator;
66
67
68
69    std::unique_ptr<cc::MultiplexedAlphabet> mpx;
70
71    parabix::StreamSetBuffer * linefeedStreamFromDecompressedBits(parabix::StreamSetBuffer *decompressedBasisBits);
72    parabix::StreamSetBuffer * linefeedStreamFromCompressedBits();
73
74
75    parabix::StreamSetBuffer * convertCompressedBitsStreamWithSwizzledApproach(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
76    parabix::StreamSetBuffer * convertCompressedBitsStreamWithAioApproach(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
77    parabix::StreamSetBuffer * convertCompressedBitsStream(parabix::StreamSetBuffer* compressedBitStream, int numberOfStream, std::string prefix);
78};
79
80
81#endif //ICGREP_LZ4GREPGENERATOR_H
Note: See TracBrowser for help on using the repository browser.