source: icGREP/icgrep-devel/icgrep/character_deletion.cpp @ 6034

Last change on this file since 6034 was 6032, checked in by xwa163, 17 months ago

Add compression pipeline in character_deletion

File size: 9.7 KB
Line 
1
2/*
3 *  Copyright (c) 2017 International Characters.
4 *  This software is licensed to the public under the Open Software License 3.0.
5 *  icgrep is a trademark of International Characters.
6 */
7
8#include <llvm/IR/Module.h>
9#include <llvm/IR/Function.h>
10#include <llvm/Linker/Linker.h>
11#include <llvm/Support/CommandLine.h>
12#include <llvm/Support/PrettyStackTrace.h>
13#include <llvm/Support/Signals.h>
14#include <llvm/Support/ManagedStatic.h>
15#include <toolchain/toolchain.h>
16
17#include <IR_Gen/idisa_target.h>
18#include <boost/filesystem.hpp>
19#include <boost/iostreams/device/mapped_file.hpp>
20
21#include <cc/cc_compiler.h>
22#include <kernels/cc_kernel.h>
23#include <kernels/streamset.h>
24#include <kernels/s2p_kernel.h>
25#include <kernels/source_kernel.h>
26#include <kernels/stdout_kernel.h>
27#include <kernels/swizzle.h>
28
29#include <kernels/kernel_builder.h>
30#include <kernels/p2s_kernel.h>
31#include <toolchain/cpudriver.h>
32#include <iostream>
33#include <fstream>
34#include <kernels/deletion.h>
35
36namespace re { class CC; }
37
38using namespace llvm;
39using namespace parabix;
40using namespace kernel;
41
42static cl::OptionCategory characterDeletionFlags("Command Flags", "deletion options");
43static cl::opt<char> characterToBeDeleted(cl::Positional, cl::desc("<character to be deletion>"), cl::Required, cl::cat(characterDeletionFlags));
44static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(characterDeletionFlags));
45static cl::opt<bool> useSwizzledDeletion("swizzle-deletion", cl::desc("Use swizzle deletion"), cl::init(false), cl::cat(characterDeletionFlags));
46
47
48typedef void (*MainFunctionType)(char * byte_data, size_t filesize);
49
50/*
51 * Usage:
52 *   character_deletion <character to be deleted> <input file name>
53 * It will delete the character from the input file and then print the output to stdout
54 * */
55
56StreamSetBuffer * loadBasisBits(ParabixDriver & pxDriver, Value* inputStream, Value* fileSize, int inputBufferBlocks) {
57    auto & iBuilder = pxDriver.getBuilder();
58
59    StreamSetBuffer * ByteStream = pxDriver.addBuffer<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
60    StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), inputBufferBlocks);
61
62    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
63    sourceK->setInitialArguments({inputStream, fileSize});
64    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
65
66    Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder, /*aligned = */ true);
67    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
68
69    return BasisBits;
70}
71
72StreamSetBuffer * generateSwizzledDeletion(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int inputBufferBlocks) {
73    auto & iBuilder = pxDriver.getBuilder();
74
75    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
76    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::makeCC(characterToBeDeleted)}, 8);
77
78    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
79
80    StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1);
81    StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1);
82    Kernel * delK = pxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(iBuilder, 8);
83    pxDriver.makeKernelCall(delK, {CharacterMarkerBuffer, BasisBits}, {u16Swizzle0, u16Swizzle1});
84
85    // Produce unswizzled bit streams
86    StreamSetBuffer * deletedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
87    Kernel * unSwizzleK = pxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
88
89    pxDriver.makeKernelCall(unSwizzleK, {u16Swizzle0, u16Swizzle1}, {deletedBits});
90    return deletedBits;
91}
92
93// TODO: It seems that there are still some bugs in DeleteByPEXTkernel
94StreamSetBuffer * generateDeletion(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int inputBufferBlocks) {
95    auto & iBuilder = pxDriver.getBuilder();
96
97    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
98    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::makeCC(characterToBeDeleted)}, 8);
99    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
100
101    StreamSetBuffer * deletedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
102    StreamSetBuffer * deletionCounts = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
103
104    Kernel * delK = pxDriver.addKernelInstance<DeleteByPEXTkernel>(iBuilder, 64, 8);
105    pxDriver.makeKernelCall(delK, {BasisBits, CharacterMarkerBuffer}, {deletedBits, deletionCounts});
106
107    StreamSetBuffer * compressedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
108    Kernel * streamCompressionK = pxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
109    pxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
110
111    return compressedBits;
112}
113
114StreamSetBuffer * generateDeletionByCompression(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int inputBufferBlocks) {
115    auto & iBuilder = pxDriver.getBuilder();
116
117    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
118    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::subtractCC(re::makeByte(0, 255), re::makeCC(characterToBeDeleted))}, 8);
119    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
120
121    StreamSetBuffer * deletedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
122    StreamSetBuffer * deletionCounts = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
123
124    Kernel * delK = pxDriver.addKernelInstance<PEXTFieldCompressKernel>(iBuilder, 64, 8);
125    pxDriver.makeKernelCall(delK, {BasisBits, CharacterMarkerBuffer}, {deletedBits, deletionCounts});
126
127    StreamSetBuffer * compressedBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
128    Kernel * streamCompressionK = pxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
129    pxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
130
131    return compressedBits;
132}
133
134
135int main(int argc, char *argv[]) {
136    // This boilerplate provides convenient stack traces and clean LLVM exit
137    // handling. It also initializes the built in support for convenient
138    // command line option handling.
139    sys::PrintStackTraceOnErrorSignal(argv[0]);
140    llvm::PrettyStackTraceProgram X(argc, argv);
141    llvm_shutdown_obj shutdown;
142    codegen::ParseCommandLineOptions(argc, argv, {&characterDeletionFlags, codegen::codegen_flags()});
143
144    const std::string fileName = inputFile;
145
146    std::ifstream f(fileName, std::ios::binary | std::ios::ate);
147    if (f.fail()) {
148        return -1;
149    }
150    size_t mFilesize = f.tellg();
151
152    boost::iostreams::mapped_file_source mappedFile;
153    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
154    mappedFile.open(fileName , mFilesize);
155
156    char *fileBuffer = const_cast<char *>(mappedFile.data());
157
158    const int inputBufferBlocks = codegen::BufferSegments * codegen::ThreadNum * 16;
159
160    ParabixDriver pxDriver("character_deletion");
161    auto & iBuilder = pxDriver.getBuilder();
162    Module * M = iBuilder->getModule();
163    Type * const sizeTy = iBuilder->getSizeTy();
164    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
165    Type * const voidTy = iBuilder->getVoidTy();
166    Type * const inputType = iBuilder->getInt8PtrTy();
167
168    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, sizeTy, boolTy, nullptr));
169    main->setCallingConv(CallingConv::C);
170    Function::arg_iterator args = main->arg_begin();
171    Value* inputStream = &*(args++);
172    inputStream->setName("input");
173
174    Value* fileSize = &*(args++);
175    fileSize->setName("fileSize");
176
177    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
178
179    // GeneratePipeline
180    StreamSetBuffer * BasisBits = loadBasisBits(pxDriver, inputStream, fileSize, inputBufferBlocks);
181    StreamSetBuffer * deletedBits = NULL;
182    if (useSwizzledDeletion) {
183        deletedBits = generateSwizzledDeletion(pxDriver, BasisBits, inputBufferBlocks);
184    } else {
185        deletedBits = generateDeletionByCompression(pxDriver, BasisBits, inputBufferBlocks);
186    }
187//    StreamSetBuffer * deletedBits = generateDeletion(pxDriver, BasisBits, inputBufferBlocks);
188
189    StreamSetBuffer * const deletedByteStream = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), inputBufferBlocks);
190    Kernel * p2sK = pxDriver.addKernelInstance<P2SKernel>(iBuilder);
191    pxDriver.makeKernelCall(p2sK, {deletedBits}, {deletedByteStream});
192
193    // --------------------------------------------------------
194    // End
195
196    Kernel * outK = pxDriver.addKernelInstance<StdOutKernel>(iBuilder, 8);
197    pxDriver.makeKernelCall(outK, {deletedByteStream}, {});
198
199    pxDriver.generatePipelineIR();
200
201    pxDriver.deallocateBuffers();
202
203    iBuilder->CreateRetVoid();
204
205    pxDriver.finalizeObject();
206
207
208    auto mainFunc = reinterpret_cast<MainFunctionType>(pxDriver.getMain());
209
210    mainFunc(fileBuffer, mFilesize);
211
212    mappedFile.close();
213    return 0;
214}
Note: See TracBrowser for help on using the repository browser.