source: icGREP/icgrep-devel/icgrep/lz4/grep/lz4_grep_base_generator.cpp @ 6145

Last change on this file since 6145 was 6145, checked in by xwa163, 3 months ago
  1. LZ4 Grep: complete utf8 character classes for multiplexing pipeline
  2. Implement multiple streams version of S2P and P2S
File size: 21.7 KB
Line 
1
2#include "lz4_grep_base_generator.h"
3
4#include <boost/iostreams/device/mapped_file.hpp>
5
6#include <llvm/Support/PrettyStackTrace.h>
7
8#include <cc/cc_compiler.h>
9
10#include <kernels/cc_kernel.h>
11#include <kernels/s2p_kernel.h>
12#include <kernels/p2s_kernel.h>
13#include <kernels/source_kernel.h>
14#include <kernels/stdout_kernel.h>
15#include <kernels/fake_stream_generating_kernel.h>
16#include <re/re_toolchain.h>
17
18#include <re/collect_ccs.h>
19#include <re/replaceCC.h>
20
21#include <re/casing.h>
22#include <re/exclude_CC.h>
23#include <re/to_utf8.h>
24#include <re/re_analysis.h>
25#include <re/re_name_resolve.h>
26#include <re/re_name_gather.h>
27#include <re/re_multiplex.h>
28#include <re/re_utility.h>
29
30#include <UCD/resolve_properties.h>
31#include <kernels/charclasses.h>
32#include <kernels/grep_kernel.h>
33#include <kernels/UCD_property_kernel.h>
34#include <kernels/grapheme_kernel.h>
35#include <kernels/linebreak_kernel.h>
36#include <kernels/streams_merge.h>
37#include <kernels/scanmatchgen.h>
38#include <kernels/until_n.h>
39#include <re/grapheme_clusters.h>
40#include <re/printer_re.h>
41#include <llvm/Support/raw_ostream.h>
42#include <llvm/Support/Debug.h>
43#include <kernels/lz4/lz4_block_decoder.h>
44
45#include <re/re_seq.h>
46#include <kernels/kernel_builder.h>
47#include <re/re_alt.h>
48
49namespace re { class CC; }
50
51using namespace llvm;
52using namespace parabix;
53using namespace kernel;
54using namespace grep;
55using namespace re;
56
57LZ4GrepBaseGenerator::LZ4GrepBaseGenerator()
58        : LZ4BaseGenerator(),
59          u8NonFinalRe(makeAlt({makeByte(0xC2, 0xF4),
60                                makeSeq({makeByte(0xE0, 0xF4), makeByte(0x80, 0xBF)}),
61                                makeSeq({makeByte(0xF0, 0xF4), makeByte(0x80, 0xBF), makeByte(0x80, 0xBF)})}))
62{
63    mGrepRecordBreak = grep::GrepRecordBreakKind::LF;
64    mMoveMatchesToEOL = true;
65}
66
67void LZ4GrepBaseGenerator::generateScanMatchGrepPipeline(re::RE* regex) {
68    auto & iBuilder = mPxDriver.getBuilder();
69    this->generateScanMatchMainFunc(iBuilder);
70
71    StreamSetBuffer* compressedByteStream = this->loadByteStream();
72
73    StreamSetBuffer * const uncompressedByteStream = this->byteStreamDecompression(compressedByteStream);
74    StreamSetBuffer * uncompressedBitStream = this->s2p(uncompressedByteStream);
75
76    StreamSetBuffer * LineBreakStream;
77    StreamSetBuffer * Matches;
78    std::tie(LineBreakStream, Matches) = grep(regex, uncompressedBitStream);
79
80    kernel::Kernel * scanMatchK = mPxDriver.addKernelInstance<kernel::ScanMatchKernel>(iBuilder);
81    scanMatchK->setInitialArguments({match_accumulator});
82    mPxDriver.makeKernelCall(scanMatchK, {Matches, LineBreakStream, uncompressedByteStream}, {});
83    mPxDriver.LinkFunction(*scanMatchK, "accumulate_match_wrapper", &accumulate_match_wrapper);
84    mPxDriver.LinkFunction(*scanMatchK, "finalize_match_wrapper", &finalize_match_wrapper);
85
86    mPxDriver.generatePipelineIR();
87    mPxDriver.deallocateBuffers();
88
89    iBuilder->CreateRetVoid();
90
91    mPxDriver.finalizeObject();
92}
93
94void LZ4GrepBaseGenerator::generateCountOnlyGrepPipeline(re::RE* regex, bool enableMultiplexing, bool utf8CC){
95    if (enableMultiplexing) {
96        this->generateMultiplexingCountOnlyGrepPipeline(regex, utf8CC);
97    } else {
98        this->generateFullyDecompressionCountOnlyGrepPipeline(regex);
99    }
100}
101
102void LZ4GrepBaseGenerator::initREs(re::RE * RE) {
103    if (mGrepRecordBreak == GrepRecordBreakKind::Unicode) {
104        mBreakCC = re::makeCC(re::makeCC(0x0A, 0x0D), re::makeCC(re::makeCC(0x85), re::makeCC(0x2028, 0x2029)));
105    } else if (mGrepRecordBreak == GrepRecordBreakKind::Null) {
106        mBreakCC = re::makeByte(0);  // Null
107    } else {
108        mBreakCC = re::makeByte(0x0A); // LF
109    }
110    re::RE * anchorRE = mBreakCC;
111    if (mGrepRecordBreak == GrepRecordBreakKind::Unicode) {
112        re::Name * anchorName = re::makeName("UTF8_LB", re::Name::Type::Unicode);
113        anchorName->setDefinition(re::makeUnicodeBreak());
114        anchorRE = anchorName;
115    }
116
117    mRE = RE;
118    bool allAnchored = true;
119
120    if (!hasEndAnchor(mRE)) allAnchored = false;
121    mRE = resolveModesAndExternalSymbols(mRE);
122    mRE = re::exclude_CC(mRE, mBreakCC);
123    mRE = resolveAnchors(mRE, anchorRE);
124    re::gatherUnicodeProperties(mRE, mUnicodeProperties);
125    mRE = regular_expression_passes(mRE);
126
127    if (allAnchored && (mGrepRecordBreak != GrepRecordBreakKind::Unicode)) mMoveMatchesToEOL = false;
128
129}
130
131parabix::StreamSetBuffer * LZ4GrepBaseGenerator::linefeedStreamFromUncompressedBits(
132        parabix::StreamSetBuffer *uncompressedBasisBits) {
133    auto & idb = mPxDriver.getBuilder();
134    const unsigned baseBufferSize = this->getDefaultBufferBlocks();
135    StreamSetBuffer * LineFeedStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
136    kernel::Kernel * linefeedK = mPxDriver.addKernelInstance<kernel::LineFeedKernelBuilder>(idb, Binding{idb->getStreamSetTy(8), "basis", FixedRate(), Principal()}, cc::BitNumbering::BigEndian);
137    mPxDriver.makeKernelCall(linefeedK, {uncompressedBasisBits}, {LineFeedStream});
138    return LineFeedStream;
139}
140
141std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> LZ4GrepBaseGenerator::multiplexingGrep(
142        re::RE *RE,
143        parabix::StreamSetBuffer *compressedByteStream,
144        parabix::StreamSetBuffer *compressedBitStream,
145        bool utf8CC
146) {
147
148    this->initREs(RE);
149    auto mGrepDriver = &mPxDriver;
150
151    auto & idb = mGrepDriver->getBuilder();
152    const unsigned baseBufferSize = this->getDefaultBufferBlocks();
153    int MaxCountFlag = 0;
154
155    //  Regular Expression Processing and Analysis Phase
156    const auto nREs = 1;
157
158    std::vector<StreamSetBuffer *> MatchResultsBufs(nREs);
159
160
161    std::map<std::string, StreamSetBuffer *> propertyStream;
162
163    std::vector<std::string> externalStreamNames;
164    std::set<re::Name *> UnicodeProperties;
165
166    StreamSetBuffer* fakeMatchCopiedBits = nullptr;
167    StreamSetBuffer* u8NoFinalStream = nullptr;
168    StreamSetBuffer * uncompressedCharClasses = nullptr;
169
170    re::CC* linefeedCC = nullptr;
171
172
173    if (utf8CC) {
174        re::Seq* seq = re::makeSeq();
175        re::RE* targetRe = mRE;
176
177        linefeedCC = re::makeCC(0x0A);
178
179        seq->push_back(targetRe);
180        seq->push_back(std::move(linefeedCC));
181
182        std::vector<re::CC*> UnicodeSets = re::collectCCs(seq, &cc::Unicode, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));;
183
184        mpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSets);
185        mRE = transformCCs(mpx.get(), targetRe);
186
187
188        std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
189        auto numOfCharacterClasses = mpx_basis.size();
190        StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize, 1);
191
192        kernel::Kernel * ccK = mGrepDriver->addKernelInstance<kernel::CharClassesKernel>(idb, std::move(mpx_basis), false, cc::BitNumbering::BigEndian);
193        mGrepDriver->makeKernelCall(ccK, {compressedBitStream}, {CharClasses});
194
195
196
197        StreamSetBuffer* compressedNonFinalStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
198        kernel::Kernel * nonFinalK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, u8NonFinalRe, externalStreamNames, std::vector<cc::Alphabet *>(), cc::BitNumbering::BigEndian);
199        mGrepDriver->makeKernelCall(nonFinalK, {compressedBitStream}, {compressedNonFinalStream});
200
201
202
203        auto decompressedStreams = this->decompressBitStreams(compressedByteStream, {CharClasses, compressedNonFinalStream});
204
205//        uncompressedCharClasses = this->decompressBitStream(compressedByteStream, CharClasses);
206        uncompressedCharClasses = decompressedStreams[0];
207        u8NoFinalStream = decompressedStreams[1];
208
209
210
211        auto fakeStreams = this->generateFakeStreams(idb, uncompressedCharClasses, std::vector<unsigned>{8});
212        fakeMatchCopiedBits = fakeStreams[0];
213
214    } else {
215        re::Seq* seq = re::makeSeq();
216        re::RE* targetRe = mRE;
217        targetRe = re::toUTF8(targetRe, true);
218
219        linefeedCC = re::makeByte(0x0A);
220
221        seq->push_back(targetRe);
222        seq->push_back(std::move(linefeedCC));
223
224        std::vector<re::CC*> UnicodeSets = re::collectCCs(seq, &cc::Byte, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));
225
226        mpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSets);
227        mRE = transformCCs(mpx.get(), targetRe);
228
229        std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
230        auto numOfCharacterClasses = mpx_basis.size();
231        StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize, 1);
232
233        kernel::Kernel * ccK = mGrepDriver->addKernelInstance<kernel::ByteClassesKernel>(idb, std::move(mpx_basis), false, cc::BitNumbering::BigEndian);
234        mGrepDriver->makeKernelCall(ccK, {compressedBitStream}, {CharClasses});
235
236        uncompressedCharClasses = this->decompressBitStream(compressedByteStream, CharClasses);
237        auto fakeStreams = this->generateFakeStreams(idb, uncompressedCharClasses, std::vector<unsigned>{8, 1});
238        fakeMatchCopiedBits = fakeStreams[0];
239        u8NoFinalStream = fakeStreams[1];
240    }
241
242    StreamSetBuffer * const MatchResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
243
244
245    StreamSetBuffer * LineBreakStream = mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), this->getDefaultBufferBlocks(), 1);
246    kernel::Kernel * lineFeedGrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, transformCCs(mpx.get(), linefeedCC), externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
247    mGrepDriver->makeKernelCall(lineFeedGrepK, {fakeMatchCopiedBits, uncompressedCharClasses}, {LineBreakStream});
248
249
250    externalStreamNames.push_back("UTF8_nonfinal");
251
252    kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mRE, externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
253    mGrepDriver->makeKernelCall(icgrepK, {fakeMatchCopiedBits, u8NoFinalStream, uncompressedCharClasses}, {MatchResults});
254    MatchResultsBufs[0] = MatchResults;
255
256    StreamSetBuffer * MergedResults = MatchResultsBufs[0];
257
258    StreamSetBuffer * Matches = MergedResults;
259    if (mMoveMatchesToEOL) {
260        StreamSetBuffer * OriginalMatches = Matches;
261        kernel::Kernel * matchedLinesK = mGrepDriver->addKernelInstance<kernel::MatchedLinesKernel>(idb);
262        Matches = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
263        mGrepDriver->makeKernelCall(matchedLinesK, {OriginalMatches, LineBreakStream}, {Matches});
264    }
265
266    if (MaxCountFlag > 0) {
267        kernel::Kernel * untilK = mGrepDriver->addKernelInstance<kernel::UntilNkernel>(idb);
268        untilK->setInitialArguments({idb->getSize(MaxCountFlag)});
269        StreamSetBuffer * const AllMatches = Matches;
270        Matches = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
271        mGrepDriver->makeKernelCall(untilK, {AllMatches}, {Matches});
272    }
273
274    return std::pair<StreamSetBuffer *, StreamSetBuffer *>(LineBreakStream, Matches);
275};
276
277std::pair<parabix::StreamSetBuffer *, parabix::StreamSetBuffer *> LZ4GrepBaseGenerator::grep(
278        re::RE *RE, parabix::StreamSetBuffer *uncompressedBasisBits, bool ccMultiplexing) {
279
280    this->initREs(RE);
281    auto mGrepDriver = &mPxDriver;
282
283    auto & idb = mGrepDriver->getBuilder();
284    // TODO: until we automate stream buffer sizing, use this calculation to determine how large our matches buffer needs to be.
285    const unsigned baseBufferSize = this->getDefaultBufferBlocks();
286    int MaxCountFlag = 0;
287
288    //  Regular Expression Processing and Analysis Phase
289    const auto nREs = 1;
290
291    std::vector<StreamSetBuffer *> MatchResultsBufs(nREs);
292
293    StreamSetBuffer * LineBreakStream = this->linefeedStreamFromUncompressedBits(uncompressedBasisBits);
294
295
296    std::map<std::string, StreamSetBuffer *> propertyStream;
297
298    for(unsigned i = 0; i < nREs; ++i) {
299        std::vector<std::string> externalStreamNames;
300        std::vector<StreamSetBuffer *> icgrepInputSets = {uncompressedBasisBits};
301
302        if (ccMultiplexing) {
303            const auto UnicodeSets = re::collectCCs(mRE, &cc::Unicode, std::set<re::Name *>({re::makeZeroWidth("\\b{g}")}));
304            StreamSetBuffer * const MatchResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
305
306            std::unique_ptr<cc::MultiplexedAlphabet> mpx = make_unique<cc::MultiplexedAlphabet>("mpx", UnicodeSets);
307            mRE = transformCCs(mpx.get(), mRE);
308            std::vector<re::CC *> mpx_basis = mpx->getMultiplexedCCs();
309            auto numOfCharacterClasses = mpx_basis.size();
310            StreamSetBuffer * CharClasses = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfCharacterClasses), baseBufferSize, 1);
311            kernel::Kernel * ccK = mGrepDriver->addKernelInstance<kernel::CharClassesKernel>(idb, std::move(mpx_basis), false, cc::BitNumbering::BigEndian);
312            mGrepDriver->makeKernelCall(ccK, {uncompressedBasisBits}, {CharClasses});
313
314            kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mRE, externalStreamNames, std::vector<cc::Alphabet *>{mpx.get()}, cc::BitNumbering::BigEndian);
315            icgrepInputSets.push_back(CharClasses);
316            mGrepDriver->makeKernelCall(icgrepK, icgrepInputSets, {MatchResults});
317            MatchResultsBufs[i] = MatchResults;
318        } else {
319            std::set<re::Name *> UnicodeProperties;
320
321
322
323
324            StreamSetBuffer* nonFinalStream = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
325            kernel::Kernel * nonFinalK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, u8NonFinalRe, externalStreamNames, std::vector<cc::Alphabet *>(), cc::BitNumbering::BigEndian);
326            mGrepDriver->makeKernelCall(nonFinalK, icgrepInputSets, {nonFinalStream});
327            icgrepInputSets.push_back(nonFinalStream);
328            externalStreamNames.push_back("UTF8_nonfinal");
329
330
331
332
333
334            StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
335            kernel::Kernel * icgrepK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, mRE, externalStreamNames, std::vector<cc::Alphabet *>(), cc::BitNumbering::BigEndian);
336            mGrepDriver->makeKernelCall(icgrepK, icgrepInputSets, {MatchResults});
337            MatchResultsBufs[i] = MatchResults;
338        }
339    }
340
341    StreamSetBuffer * MergedResults = MatchResultsBufs[0];
342
343    StreamSetBuffer * Matches = MergedResults;
344    if (mMoveMatchesToEOL) {
345        StreamSetBuffer * OriginalMatches = Matches;
346        kernel::Kernel * matchedLinesK = mGrepDriver->addKernelInstance<kernel::MatchedLinesKernel>(idb);
347        Matches = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
348        mGrepDriver->makeKernelCall(matchedLinesK, {OriginalMatches, LineBreakStream}, {Matches});
349    }
350
351    if (MaxCountFlag > 0) {
352        kernel::Kernel * untilK = mGrepDriver->addKernelInstance<kernel::UntilNkernel>(idb);
353        untilK->setInitialArguments({idb->getSize(MaxCountFlag)});
354        StreamSetBuffer * const AllMatches = Matches;
355        Matches = mGrepDriver->addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(1, 1), baseBufferSize, 1);
356        mGrepDriver->makeKernelCall(untilK, {AllMatches}, {Matches});
357    }
358
359    return std::pair<StreamSetBuffer *, StreamSetBuffer *>(LineBreakStream, Matches);
360
361}
362
363void LZ4GrepBaseGenerator::invokeScanMatchGrep(char* fileBuffer, size_t blockStart, size_t blockEnd, bool hasBlockChecksum) {
364    auto main = this->getScanMatchGrepMainFunction();
365    std::ostringstream s;
366    EmitMatch accum("", false, false, s);
367
368    main(fileBuffer, blockStart, blockEnd, hasBlockChecksum, reinterpret_cast<intptr_t>(&accum));
369    llvm::outs() << s.str();
370}
371
372
373
374void LZ4GrepBaseGenerator::generateMultiplexingCountOnlyGrepPipeline(re::RE *regex, bool utf8CC) {
375    auto & iBuilder = mPxDriver.getBuilder();
376    this->generateCountOnlyMainFunc(iBuilder);
377
378    StreamSetBuffer *compressedByteStream = nullptr, *compressedBasisBits = nullptr;
379    std::tie(compressedByteStream, compressedBasisBits) = this->loadByteStreamAndBitStream();
380
381    StreamSetBuffer * LineBreakStream;
382    StreamSetBuffer * Matches;
383    std::tie(LineBreakStream, Matches) = multiplexingGrep(regex, compressedByteStream, compressedBasisBits, utf8CC);
384
385    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
386    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
387    mPxDriver.generatePipelineIR();
388
389    iBuilder->setKernel(matchCountK);
390    Value * matchedLineCount = iBuilder->getAccumulator("countResult");
391    matchedLineCount = iBuilder->CreateZExt(matchedLineCount, iBuilder->getInt64Ty());
392
393    mPxDriver.deallocateBuffers();
394
395    iBuilder->CreateRet(matchedLineCount);
396
397    mPxDriver.finalizeObject();
398}
399
400
401void LZ4GrepBaseGenerator::generateFullyDecompressionCountOnlyGrepPipeline(re::RE *regex) {
402    auto & iBuilder = mPxDriver.getBuilder();
403    this->generateCountOnlyMainFunc(iBuilder);
404
405    StreamSetBuffer * const uncompressedBitStream = this->generateUncompressedBitStreams();
406
407    StreamSetBuffer * LineBreakStream;
408    StreamSetBuffer * Matches;
409
410    std::tie(LineBreakStream, Matches) = grep(regex, uncompressedBitStream);
411
412    kernel::Kernel * matchCountK = mPxDriver.addKernelInstance<kernel::PopcountKernel>(iBuilder);
413    mPxDriver.makeKernelCall(matchCountK, {Matches}, {});
414    mPxDriver.generatePipelineIR();
415
416    iBuilder->setKernel(matchCountK);
417    Value * matchedLineCount = iBuilder->getAccumulator("countResult");
418    matchedLineCount = iBuilder->CreateZExt(matchedLineCount, iBuilder->getInt64Ty());
419
420    mPxDriver.deallocateBuffers();
421
422    iBuilder->CreateRet(matchedLineCount);
423
424    mPxDriver.finalizeObject();
425}
426
427
428ScanMatchGrepMainFunctionType LZ4GrepBaseGenerator::getScanMatchGrepMainFunction() {
429    return reinterpret_cast<ScanMatchGrepMainFunctionType>(mPxDriver.getMain());
430}
431CountOnlyGrepMainFunctionType LZ4GrepBaseGenerator::getCountOnlyGrepMainFunction() {
432    return reinterpret_cast<CountOnlyGrepMainFunctionType>(mPxDriver.getMain());
433}
434
435void LZ4GrepBaseGenerator::generateCountOnlyMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
436    Module * M = iBuilder->getModule();
437    Type * const int64Ty = iBuilder->getInt64Ty();
438    Type * const sizeTy = iBuilder->getSizeTy();
439    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
440//    Type * const voidTy = iBuilder->getVoidTy();
441    Type * const inputType = iBuilder->getInt8PtrTy();
442
443    Function * const main = cast<Function>(M->getOrInsertFunction("Main", int64Ty, inputType, sizeTy, sizeTy, boolTy, nullptr));
444    main->setCallingConv(CallingConv::C);
445    Function::arg_iterator args = main->arg_begin();
446    mInputStream = &*(args++);
447    mInputStream->setName("input");
448
449    mHeaderSize = &*(args++);
450    mHeaderSize->setName("mHeaderSize");
451
452    mFileSize = &*(args++);
453    mFileSize->setName("mFileSize");
454
455    mHasBlockChecksum = &*(args++);
456    mHasBlockChecksum->setName("mHasBlockChecksum");
457    // TODO for now, we do not handle blockCheckSum
458    mHasBlockChecksum = iBuilder->getInt1(false);
459
460    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
461}
462
463void LZ4GrepBaseGenerator::generateScanMatchMainFunc(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
464    Module * M = iBuilder->getModule();
465    Type * const sizeTy = iBuilder->getSizeTy();
466    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
467    Type * const voidTy = iBuilder->getVoidTy();
468    Type * const inputType = iBuilder->getInt8PtrTy();
469    Type * const intAddrTy = iBuilder->getIntAddrTy();
470
471    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, sizeTy, boolTy, intAddrTy, nullptr));
472    main->setCallingConv(CallingConv::C);
473    Function::arg_iterator args = main->arg_begin();
474    mInputStream = &*(args++);
475    mInputStream->setName("input");
476
477    mHeaderSize = &*(args++);
478    mHeaderSize->setName("mHeaderSize");
479
480    mFileSize = &*(args++);
481    mFileSize->setName("mFileSize");
482
483    mHasBlockChecksum = &*(args++);
484    mHasBlockChecksum->setName("mHasBlockChecksum");
485
486    match_accumulator = &*(args++);
487    match_accumulator->setName("match_accumulator");
488
489    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
490}
491
492std::vector<parabix::StreamSetBuffer *>
493LZ4GrepBaseGenerator::generateFakeStreams(const std::unique_ptr<kernel::KernelBuilder> &idb,
494                                          parabix::StreamSetBuffer *refStream, std::vector<unsigned> numOfStreams) {
495
496    if (!numOfStreams.size()) {
497        return std::vector<StreamSetBuffer *>();
498    }
499    std::vector<StreamSetBuffer *> outputStreams;
500    for (unsigned i = 0; i < numOfStreams.size(); i++) {
501        outputStreams.push_back(mPxDriver.addBuffer<StaticBuffer>(idb, idb->getStreamSetTy(numOfStreams[i]),
502                                                                  this->getDefaultBufferBlocks(), 1));
503    }
504    Kernel* fakeStreamGeneratorK = mPxDriver.addKernelInstance<FakeStreamGeneratingKernel>(idb, refStream->getNumOfStreams(), numOfStreams);
505    mPxDriver.makeKernelCall(fakeStreamGeneratorK, {refStream}, outputStreams);
506    return outputStreams;
507}
508
509std::vector<parabix::StreamSetBuffer *>
510LZ4GrepBaseGenerator::decompressBitStreams(parabix::StreamSetBuffer *compressedByteStream,
511                                           std::vector<parabix::StreamSetBuffer *> compressedBitStreams) {
512    // Default implementation here will be slow
513    std::vector<parabix::StreamSetBuffer *> retVec;
514    for (unsigned i = 0; i < compressedBitStreams.size(); i++) {
515        retVec.push_back(this->decompressBitStream(compressedByteStream, compressedBitStreams[i]));
516    }
517    return retVec;
518}
Note: See TracBrowser for help on using the repository browser.