source: icGREP/icgrep-devel/icgrep/lz4/lz4_grep_generator.h @ 6140

Last change on this file since 6140 was 6140, checked in by xwa163, 10 months ago

UTF-8 support for Multiplexing LZ4 Grep

File size: 3.2 KB
Line 
1
2#ifndef ICGREP_LZ4GREPGENERATOR_H
3#define ICGREP_LZ4GREPGENERATOR_H
4
5#include "lz4_base_generator.h"
6
7#include <grep_interface.h>
8#include <kernels/streamset.h>
9#include <cc/multiplex_CCs.h>
10#include <string>
11#include <vector>
12#include <sstream>
13#include <atomic>
14#include <grep/grep_engine.h>
15
16typedef void (*ScanMatchGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum, intptr_t match_accumulator);
17typedef uint64_t (*CountOnlyGrepMainFunctionType)(char * byte_data, size_t headerSize, size_t filesize, bool hasBlockChecksum);
18
19
20class LZ4GrepGenerator : public LZ4BaseGenerator {
21public:
22
23    LZ4GrepGenerator(bool enableMultiplexing = false);
24
25    void generateScanMatchGrepPipeline(re::RE* regex);
26    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> grepPipeline(std::vector<re::RE *> &REs,
27                                                                                   parabix::StreamSetBuffer *uncompressedBasisBits);
28
29    std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> multiplexingGrepPipeline(
30            std::vector<re::RE *> &REs,
31            parabix::StreamSetBuffer *compressedByteStream,
32            parabix::StreamSetBuffer *compressedBitStream,
33            bool useSwizzled = true,
34            bool useByteStream = false
35    );
36
37
38    void invokeScanMatchGrep(char* fileBuffer, size_t blockStart, size_t blockEnd, bool hasBlockChecksum);
39
40    void generateMultiplexingSwizzledAioPipeline(re::RE *regex);
41    void generateSwizzledAioPipeline(re::RE* regex);
42    void generateMultiplexingBitStreamAioPipeline(re::RE* regex);
43    void generateBitStreamAioPipeline(re::RE* regex);
44
45    void generateByteStreamMultiplexingAioPipeline(re::RE* regex);
46    void generateAioPipeline(re::RE* regex);
47    void generateParallelAioPipeline(re::RE* regex, bool enableGather, bool enableScatter, int minParallelLevel);
48
49    ScanMatchGrepMainFunctionType getScanMatchGrepMainFunction();
50    CountOnlyGrepMainFunctionType getCountOnlyGrepMainFunction();
51
52private:
53    bool mEnableMultiplexing;
54
55    grep::GrepRecordBreakKind mGrepRecordBreak;
56    void initREs(std::vector<re::RE *> & REs);
57
58
59    re::CC * mBreakCC;
60    std::vector<re:: RE *> mREs;
61    std::set<re::Name *> mUnicodeProperties;
62    bool mMoveMatchesToEOL;
63
64
65    std::vector<std::ostringstream> mResultStrs;
66
67    void generateCountOnlyMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
68    void generateScanMatchMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
69
70
71    llvm::Value * match_accumulator;
72
73
74
75    std::unique_ptr<cc::MultiplexedAlphabet> mpx;
76
77    parabix::StreamSetBuffer * linefeedStreamFromUncompressedBits(parabix::StreamSetBuffer *uncompressedBasisBits);
78
79
80
81    parabix::StreamSetBuffer * convertCompressedBitsStreamWithSwizzledAioApproach(
82            parabix::StreamSetBuffer *compressedByteStream,
83            parabix::StreamSetBuffer *compressedBitStream,
84            std::string prefix
85    );
86
87    parabix::StreamSetBuffer *convertCompressedBitsStreamWithTwistApproach(
88            parabix::StreamSetBuffer *compressedByteStream,
89            parabix::StreamSetBuffer *compressedBitStream,
90            std::string prefix
91    );
92
93};
94
95
96#endif //ICGREP_LZ4GREPGENERATOR_H
Note: See TracBrowser for help on using the repository browser.