source: icGREP/icgrep-devel/icgrep/character_deposit.cpp @ 6047

Last change on this file since 6047 was 6047, checked in by nmedfort, 14 months ago

Major refactoring of buffer types. Static buffers replace Circular and CircularCopyback?. External buffers unify Source/External?.

File size: 9.7 KB
Line 
1
2/*
3 *  Copyright (c) 2017 International Characters.
4 *  This software is licensed to the public under the Open Software License 3.0.
5 *  icgrep is a trademark of International Characters.
6 */
7
8#include <llvm/IR/Module.h>
9#include <llvm/IR/Function.h>
10#include <llvm/Linker/Linker.h>
11#include <llvm/Support/PrettyStackTrace.h>
12#include <llvm/Support/Signals.h>
13#include <llvm/Support/ManagedStatic.h>
14#include <toolchain/toolchain.h>
15
16#include <IR_Gen/idisa_target.h>
17#include <boost/filesystem.hpp>
18#include <boost/iostreams/device/mapped_file.hpp>
19
20#include <cc/cc_compiler.h>
21#include <kernels/cc_kernel.h>
22#include <kernels/streamset.h>
23#include <kernels/s2p_kernel.h>
24#include <kernels/source_kernel.h>
25#include <kernels/stdout_kernel.h>
26#include <kernels/swizzle.h>
27
28#include <kernels/kernel_builder.h>
29#include <kernels/p2s_kernel.h>
30#include <toolchain/cpudriver.h>
31#include <iostream>
32#include <kernels/deletion.h>
33#include <kernels/pdep_kernel.h>
34#include <kernels/bitstream_pdep_kernel.h>
35
36namespace re { class CC; }
37
38using namespace llvm;
39using namespace parabix;
40using namespace kernel;
41
42static cl::OptionCategory characterDepositFlags("Command Flags", "deletion options");
43static cl::opt<char> characterToBeDeposit(cl::Positional, cl::desc("<character to be depositted>"), cl::Required, cl::cat(characterDepositFlags));
44static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(characterDepositFlags));
45static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(characterDepositFlags));
46static cl::opt<bool> useSwizzledDeposit("swizzle-deposit", cl::desc("Use swizzle deletion"), cl::init(false), cl::cat(characterDepositFlags));
47
48
49typedef void (*MainFunctionType)(char * byte_data, size_t filesize);
50
51/*
52 * Usage:
53 *     character_deposit <character to be deposit> <input file name> <output file name>
54 *
55 * Assume we have a input file with 6 character:
56 *     qbwbeb
57 * Then, the pipeline will first deleted all of character 'b' from input:
58 *     qbwbeb => qwe
59 * And then it will deposit the deleted result (qwe) to the original position of b:
60 *     qwe => \0 q \0 w \0 e (spaces are not included)
61 * and store the result to output file
62 * */
63
64StreamSetBuffer * loadBasisBits(ParabixDriver & pxDriver, Value* inputStream, Value* fileSize, int bufferBlocks) {
65    auto & iBuilder = pxDriver.getBuilder();
66
67    StreamSetBuffer * ByteStream = pxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
68    StreamSetBuffer * BasisBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), bufferBlocks);
69
70    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
71    sourceK->setInitialArguments({inputStream, fileSize});
72    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
73    Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder);
74    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
75
76    return BasisBits;
77}
78
79StreamSetBuffer * generateSwizzledDeposit(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int bufferBlocks) {
80    auto & iBuilder = pxDriver.getBuilder();
81
82    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferBlocks);
83    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(characterToBeDeposit)}, 8);
84    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
85
86
87    StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
88    StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
89    Kernel * delK = pxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(iBuilder, 8);
90    pxDriver.makeKernelCall(delK, {CharacterMarkerBuffer, BasisBits}, {u16Swizzle0, u16Swizzle1});
91
92    StreamSetBuffer * depositedSwizzle0 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
93    Kernel * pdep0K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, "pdep0");
94    pxDriver.makeKernelCall(pdep0K, {CharacterMarkerBuffer, u16Swizzle0}, {depositedSwizzle0});
95
96
97    StreamSetBuffer * depositedSwizzle1 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), bufferBlocks, 1);
98    Kernel * pdep1K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, "pdep1");
99    pxDriver.makeKernelCall(pdep1K, {CharacterMarkerBuffer, u16Swizzle1}, {depositedSwizzle1});
100
101    // Produce unswizzled bit streams
102    StreamSetBuffer * resultbits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
103    Kernel * unSwizzleK = pxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
104
105    pxDriver.makeKernelCall(unSwizzleK, {depositedSwizzle0, depositedSwizzle1}, {resultbits});
106    return resultbits;
107}
108
109StreamSetBuffer * generateBitStreamDeposit(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int bufferBlocks) {
110    auto & iBuilder = pxDriver.getBuilder();
111
112    StreamSetBuffer * const deletionMarker = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferBlocks);
113    Kernel * ccK1 = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::subtractCC(re::makeByte(0, 255), re::makeCC(characterToBeDeposit))}, 8);
114    pxDriver.makeKernelCall(ccK1, {BasisBits}, {deletionMarker});
115
116    StreamSetBuffer * const depositMarker = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), bufferBlocks);
117    Kernel * ccK2 = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(characterToBeDeposit)}, 8);
118    pxDriver.makeKernelCall(ccK2, {BasisBits}, {depositMarker});
119
120    // Deletion
121    StreamSetBuffer * deletedBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
122    StreamSetBuffer * deletionCounts = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
123
124    Kernel * delK = pxDriver.addKernelInstance<PEXTFieldCompressKernel>(iBuilder, 64, 8);
125    pxDriver.makeKernelCall(delK, {BasisBits, deletionMarker}, {deletedBits, deletionCounts});
126
127    StreamSetBuffer * compressedBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks);
128    Kernel * streamCompressionK = pxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
129    pxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
130
131    // Deposit
132    StreamSetBuffer * depositedBits = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(8), bufferBlocks, 1);
133    Kernel * pdepK = pxDriver.addKernelInstance<BitStreamPDEPKernel>(iBuilder, 8);
134    pxDriver.makeKernelCall(pdepK, {depositMarker, compressedBits}, {depositedBits});
135
136    return depositedBits;
137}
138
139int main(int argc, char *argv[]) {
140    // This boilerplate provides convenient stack traces and clean LLVM exit
141    // handling. It also initializes the built in support for convenient
142    // command line option handling.
143    sys::PrintStackTraceOnErrorSignal(argv[0]);
144    llvm::PrettyStackTraceProgram X(argc, argv);
145    llvm_shutdown_obj shutdown;
146    codegen::ParseCommandLineOptions(argc, argv, {&characterDepositFlags, codegen::codegen_flags()});
147
148    std::string fileName = inputFile;
149
150    std::ifstream f(fileName, std::ios::binary | std::ios::ate);
151    if (f.fail()) {
152        return -1;
153    }
154    size_t mFilesize = f.tellg();
155
156    boost::iostreams::mapped_file_source mappedFile;
157    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
158    mappedFile.open(fileName , mFilesize);
159
160    char *fileBuffer = const_cast<char *>(mappedFile.data());
161
162    const auto bufferBlocks = codegen::ThreadNum * codegen::SegmentSize;
163
164    ParabixDriver pxDriver("character_deletion");
165    auto & iBuilder = pxDriver.getBuilder();
166    Module * M = iBuilder->getModule();
167    Type * const sizeTy = iBuilder->getSizeTy();
168    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
169    Type * const voidTy = iBuilder->getVoidTy();
170    Type * const inputType = iBuilder->getInt8PtrTy();
171
172    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, sizeTy, boolTy, nullptr));
173    main->setCallingConv(CallingConv::C);
174    Function::arg_iterator args = main->arg_begin();
175    Value* inputStream = &*(args++);
176    inputStream->setName("input");
177
178    Value* fileSize = &*(args++);
179    fileSize->setName("fileSize");
180
181    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
182
183    StreamSetBuffer * BasisBits = loadBasisBits(pxDriver, inputStream, fileSize, bufferBlocks);
184    StreamSetBuffer * resultbits = NULL;
185    if (useSwizzledDeposit) {
186        resultbits = generateSwizzledDeposit(pxDriver, BasisBits, bufferBlocks);
187    } else {
188        resultbits = generateBitStreamDeposit(pxDriver, BasisBits, bufferBlocks);
189    }
190
191    StreamSetBuffer * const ResultBytes = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferBlocks);
192    Kernel * p2sK = pxDriver.addKernelInstance<P2SKernel>(iBuilder);
193    pxDriver.makeKernelCall(p2sK, {resultbits}, {ResultBytes});
194
195    Kernel * outK = pxDriver.addKernelInstance<FileSink>(iBuilder, 8);
196    outK->setInitialArguments({iBuilder->GetString(outputFile)});
197    pxDriver.makeKernelCall(outK, {ResultBytes}, {});
198
199    pxDriver.generatePipelineIR();
200
201    pxDriver.deallocateBuffers();
202
203    iBuilder->CreateRetVoid();
204
205    pxDriver.finalizeObject();
206
207
208    auto mainFunc = reinterpret_cast<MainFunctionType>(pxDriver.getMain());
209
210
211    mainFunc(fileBuffer, mFilesize);
212
213    mappedFile.close();
214    return 0;
215}
Note: See TracBrowser for help on using the repository browser.