source: icGREP/icgrep-devel/icgrep/character_deposit.cpp @ 6166

Last change on this file since 6166 was 6072, checked in by cameron, 14 months ago

character_deposit UseStreamDepositCompiler? option

File size: 10.1 KB
Line 
1
2/*
3 *  Copyright (c) 2017 International Characters.
4 *  This software is licensed to the public under the Open Software License 3.0.
5 *  icgrep is a trademark of International Characters.
6 */
7
8#include <llvm/IR/Module.h>
9#include <llvm/IR/Function.h>
10#include <llvm/Linker/Linker.h>
11#include <llvm/Support/PrettyStackTrace.h>
12#include <llvm/Support/Signals.h>
13#include <llvm/Support/ManagedStatic.h>
14#include <toolchain/toolchain.h>
15
16#include <IR_Gen/idisa_target.h>
17#include <boost/filesystem.hpp>
18#include <boost/iostreams/device/mapped_file.hpp>
19
20#include <cc/cc_compiler.h>
21#include <kernels/cc_kernel.h>
22#include <kernels/streamset.h>
23#include <kernels/s2p_kernel.h>
24#include <kernels/source_kernel.h>
25#include <kernels/stdout_kernel.h>
26#include <kernels/swizzle.h>
27
28#include <kernels/kernel_builder.h>
29#include <kernels/p2s_kernel.h>
30#include <toolchain/cpudriver.h>
31#include <iostream>
32#include <fstream>
33#include <kernels/deletion.h>
34#include <kernels/pdep_kernel.h>
35#include <kernels/bitstream_pdep_kernel.h>
36
37namespace re { class CC; }
38
39using namespace llvm;
40using namespace parabix;
41using namespace kernel;
42
43static cl::OptionCategory characterDepositFlags("Command Flags", "deletion options");
44static cl::opt<char> characterToBeDeposit(cl::Positional, cl::desc("<character to be depositted>"), cl::Required, cl::cat(characterDepositFlags));
45static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(characterDepositFlags));
46static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(characterDepositFlags));
47static cl::opt<bool> useSwizzledDeposit("swizzle-deposit", cl::desc("Use swizzle deletion"), cl::init(false), cl::cat(characterDepositFlags));
48static cl::opt<bool> UseStreamDepositCompiler("UseStreamDepositCompiler", cl::desc("Use the StreamDepositCompiler deletion"), cl::init(false), cl::cat(characterDepositFlags));
49
50
51typedef void (*MainFunctionType)(char * byte_data, size_t filesize);
52
53/*
54 * Usage:
55 *     character_deposit <character to be deposit> <input file name> <output file name>
56 *
57 * Assume we have a input file with 6 character:
58 *     qbwbeb
59 * Then, the pipeline will first deleted all of character 'b' from input:
60 *     qbwbeb => qwe
61 * And then it will deposit the deleted result (qwe) to the original position of b:
62 *     qwe => \0 q \0 w \0 e (spaces are not included)
63 * and store the result to output file
64 * */
65
66StreamSetBuffer * loadBasisBits(ParabixDriver & pxDriver, Value* inputStream, Value* fileSize, int bufferBlocks) {
67    auto & iBuilder = pxDriver.getBuilder();
68
69    StreamSetBuffer * ByteStream = pxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
70    StreamSetBuffer * BasisBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), bufferBlocks);
71
72    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
73    sourceK->setInitialArguments({inputStream, fileSize});
74    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
75    Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder);
76    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
77
78    return BasisBits;
79}
80
81StreamSetBuffer * generateSwizzledDeposit(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int bufferBlocks) {
82    auto & iBuilder = pxDriver.getBuilder();
83
84    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferBlocks);
85    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(characterToBeDeposit)}, 8);
86    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
87
88
89    StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
90    StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
91    Kernel * delK = pxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(iBuilder, 8);
92    pxDriver.makeKernelCall(delK, {CharacterMarkerBuffer, BasisBits}, {u16Swizzle0, u16Swizzle1});
93
94    StreamSetBuffer * depositedSwizzle0 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
95    Kernel * pdep0K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, "pdep0");
96    pxDriver.makeKernelCall(pdep0K, {CharacterMarkerBuffer, u16Swizzle0}, {depositedSwizzle0});
97
98
99    StreamSetBuffer * depositedSwizzle1 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
100    Kernel * pdep1K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, "pdep1");
101    pxDriver.makeKernelCall(pdep1K, {CharacterMarkerBuffer, u16Swizzle1}, {depositedSwizzle1});
102
103    // Produce unswizzled bit streams
104    StreamSetBuffer * resultbits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
105    Kernel * unSwizzleK = pxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
106
107    pxDriver.makeKernelCall(unSwizzleK, {depositedSwizzle0, depositedSwizzle1}, {resultbits});
108    return resultbits;
109}
110
111StreamSetBuffer * generateBitStreamDeposit(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int bufferBlocks) {
112    auto & iBuilder = pxDriver.getBuilder();
113
114    StreamSetBuffer * const deletionMarker = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferBlocks);
115    Kernel * ccK1 = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::subtractCC(re::makeByte(0, 255), re::makeCC(characterToBeDeposit))}, 8);
116    pxDriver.makeKernelCall(ccK1, {BasisBits}, {deletionMarker});
117
118    StreamSetBuffer * const depositMarker = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferBlocks);
119    Kernel * ccK2 = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(characterToBeDeposit)}, 8);
120    pxDriver.makeKernelCall(ccK2, {BasisBits}, {depositMarker});
121
122    // Deletion
123    StreamSetBuffer * deletedBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
124    StreamSetBuffer * deletionCounts = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
125
126    Kernel * delK = pxDriver.addKernelInstance<PEXTFieldCompressKernel>(iBuilder, 64, 8);
127    pxDriver.makeKernelCall(delK, {BasisBits, deletionMarker}, {deletedBits, deletionCounts});
128
129    StreamSetBuffer * compressedBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
130    Kernel * streamCompressionK = pxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
131    pxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
132
133    // Deposit
134    StreamSetBuffer * depositedBits = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks, 1);
135    if (UseStreamDepositCompiler) {
136        StreamDepositCompiler depositCompiler(pxDriver, 8, 0, 8, bufferBlocks);
137        depositCompiler.makeCall(depositMarker, compressedBits, depositedBits);
138    } else {
139        Kernel * pdepK = pxDriver.addKernelInstance<BitStreamPDEPKernel>(iBuilder, 8);
140        pxDriver.makeKernelCall(pdepK, {depositMarker, compressedBits}, {depositedBits});
141    }
142    return depositedBits;
143}
144
145int main(int argc, char *argv[]) {
146    // This boilerplate provides convenient stack traces and clean LLVM exit
147    // handling. It also initializes the built in support for convenient
148    // command line option handling.
149    sys::PrintStackTraceOnErrorSignal(argv[0]);
150    llvm::PrettyStackTraceProgram X(argc, argv);
151    llvm_shutdown_obj shutdown;
152    codegen::ParseCommandLineOptions(argc, argv, {&characterDepositFlags, codegen::codegen_flags()});
153
154    std::string fileName = inputFile;
155
156    std::ifstream f(fileName, std::ios::binary | std::ios::ate);
157    if (f.fail()) {
158        return -1;
159    }
160    size_t mFilesize = f.tellg();
161
162    boost::iostreams::mapped_file_source mappedFile;
163    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
164    mappedFile.open(fileName , mFilesize);
165
166    char *fileBuffer = const_cast<char *>(mappedFile.data());
167
168    const auto bufferBlocks = codegen::ThreadNum * codegen::SegmentSize;
169
170    ParabixDriver pxDriver("character_deletion");
171    auto & iBuilder = pxDriver.getBuilder();
172    Module * M = iBuilder->getModule();
173    Type * const sizeTy = iBuilder->getSizeTy();
174    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
175    Type * const voidTy = iBuilder->getVoidTy();
176    Type * const inputType = iBuilder->getInt8PtrTy();
177
178    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, sizeTy, boolTy, nullptr));
179    main->setCallingConv(CallingConv::C);
180    Function::arg_iterator args = main->arg_begin();
181    Value* inputStream = &*(args++);
182    inputStream->setName("input");
183
184    Value* fileSize = &*(args++);
185    fileSize->setName("fileSize");
186
187    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
188
189    StreamSetBuffer * BasisBits = loadBasisBits(pxDriver, inputStream, fileSize, bufferBlocks);
190    StreamSetBuffer * resultbits = NULL;
191    if (useSwizzledDeposit) {
192        resultbits = generateSwizzledDeposit(pxDriver, BasisBits, bufferBlocks);
193    } else {
194        resultbits = generateBitStreamDeposit(pxDriver, BasisBits, bufferBlocks);
195    }
196
197    StreamSetBuffer * const ResultBytes = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferBlocks);
198    Kernel * p2sK = pxDriver.addKernelInstance<P2SKernel>(iBuilder);
199    pxDriver.makeKernelCall(p2sK, {resultbits}, {ResultBytes});
200
201    Kernel * outK = pxDriver.addKernelInstance<FileSink>(iBuilder, 8);
202    outK->setInitialArguments({iBuilder->GetString(outputFile)});
203    pxDriver.makeKernelCall(outK, {ResultBytes}, {});
204
205    pxDriver.generatePipelineIR();
206
207    pxDriver.deallocateBuffers();
208
209    iBuilder->CreateRetVoid();
210
211    pxDriver.finalizeObject();
212
213
214    auto mainFunc = reinterpret_cast<MainFunctionType>(pxDriver.getMain());
215
216
217    mainFunc(fileBuffer, mFilesize);
218
219    mappedFile.close();
220    return 0;
221}
Note: See TracBrowser for help on using the repository browser.