source: icGREP/icgrep-devel/icgrep/lzparabix/LZParabixCompressorGenerator.cpp @ 6123

Last change on this file since 6123 was 6123, checked in by xwa163, 13 months ago

Encode BitStream? directly in LZParabix compressed file

File size: 4.2 KB
Line 
1//
2// Created by wxy325 on 2018/6/24.
3//
4
5#include "LZParabixCompressorGenerator.h"
6
7#include <boost/filesystem.hpp>
8#include <boost/iostreams/device/mapped_file.hpp>
9
10
11#include <cc/cc_compiler.h>
12
13#include <kernels/cc_kernel.h>
14#include <kernels/s2p_kernel.h>
15#include <kernels/p2s_kernel.h>
16#include <kernels/source_kernel.h>
17#include <kernels/stdout_kernel.h>
18#include <kernels/kernel_builder.h>
19#include <kernels/lzparabix/encoder/LZParabixCompressionKernel.h>
20
21
22namespace re { class CC; }
23
24using namespace llvm;
25using namespace parabix;
26using namespace kernel;
27
28
29LZParabixCompressorGenerator::LZParabixCompressorGenerator(): mPxDriver("lzParabixCompressor"), mLzParabixBlockSize(4 * 1024 * 1024) {
30
31}
32
33CompressorMainFunctionType LZParabixCompressorGenerator::getMainFunc() {
34    return reinterpret_cast<CompressorMainFunctionType>(mPxDriver.getMain());
35}
36
37void LZParabixCompressorGenerator::generatePipeline(const std::string &outputFile) {
38    auto & iBuilder = mPxDriver.getBuilder();
39    this->generateMainFunc(iBuilder);
40
41    this->generateLoadByteStreamAndBitStream(iBuilder);
42
43
44    parabix::StreamSetBuffer* outputStream = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
45
46    parabix::StreamSetBuffer* literalByteBuffer = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
47    parabix::StreamSetBuffer* tokenByteBuffer = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), this->getInputBufferBlocks(iBuilder));
48
49    parabix::StreamSetBuffer* strToBlockIndex = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 64), HASHNBCELLS4);
50    parabix::StreamSetBuffer* strToMatchPos = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 32), HASHNBCELLS4);
51
52    Kernel * compressionKernel = mPxDriver.addKernelInstance<LZParabixCompressionKernel>(iBuilder);
53    compressionKernel->setInitialArguments({mFileSize});
54    mPxDriver.makeKernelCall(compressionKernel, {mCompressedByteStream, mCompressedBasisBits}, {outputStream, strToBlockIndex, strToMatchPos, literalByteBuffer, tokenByteBuffer});
55
56    // --------------------------------------------------------
57    // End
58    Kernel * outK = mPxDriver.addKernelInstance<FileSink>(iBuilder, 8);
59    outK->setInitialArguments({iBuilder->GetString(outputFile)});
60    mPxDriver.makeKernelCall(outK, {outputStream}, {});
61
62
63    mPxDriver.generatePipelineIR();
64    mPxDriver.deallocateBuffers();
65
66    iBuilder->CreateRetVoid();
67
68    mPxDriver.finalizeObject();
69}
70
71void LZParabixCompressorGenerator::generateLoadByteStreamAndBitStream(
72        const std::unique_ptr<kernel::KernelBuilder> &iBuilder) {
73    mCompressedByteStream = mPxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
74    mCompressedBasisBits = mPxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), this->getInputBufferBlocks(iBuilder));
75
76    kernel::Kernel * sourceK = mPxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
77    sourceK->setInitialArguments({mInputStream, mFileSize});
78    mPxDriver.makeKernelCall(sourceK, {}, {mCompressedByteStream});
79    Kernel * s2pk = mPxDriver.addKernelInstance<S2PKernel>(iBuilder, cc::BitNumbering::BigEndian);
80    mPxDriver.makeKernelCall(s2pk, {mCompressedByteStream}, {mCompressedBasisBits});
81
82}
83
84void LZParabixCompressorGenerator::generateMainFunc(const std::unique_ptr<kernel::KernelBuilder> &iBuilder) {
85    Module * M = iBuilder->getModule();
86    Type * const sizeTy = iBuilder->getSizeTy();
87    Type * const voidTy = iBuilder->getVoidTy();
88    Type * const inputType = iBuilder->getInt8PtrTy();
89
90    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, nullptr));
91    main->setCallingConv(CallingConv::C);
92    Function::arg_iterator args = main->arg_begin();
93    mInputStream = &*(args++);
94    mInputStream->setName("input");
95
96    mFileSize = &*(args++);
97    mFileSize->setName("mFileSize");
98
99
100    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
101}
102
103size_t LZParabixCompressorGenerator::getInputBufferBlocks(const std::unique_ptr<kernel::KernelBuilder> &iBuilder) {
104    return mLzParabixBlockSize / codegen::BlockSize * 2;
105}
106
107
108
109
110
111
Note: See TracBrowser for help on using the repository browser.