source: icGREP/icgrep-devel/icgrep/character_deletion.cpp @ 6079

Last change on this file since 6079 was 6047, checked in by nmedfort, 16 months ago

Major refactoring of buffer types. Static buffers replace Circular and CircularCopyback?. External buffers unify Source/External?.

File size: 9.2 KB
Line 
1
2/*
3 *  Copyright (c) 2017 International Characters.
4 *  This software is licensed to the public under the Open Software License 3.0.
5 *  icgrep is a trademark of International Characters.
6 */
7
8#include <llvm/IR/Module.h>
9#include <llvm/IR/Function.h>
10#include <llvm/Linker/Linker.h>
11#include <llvm/Support/CommandLine.h>
12#include <llvm/Support/PrettyStackTrace.h>
13#include <llvm/Support/Signals.h>
14#include <llvm/Support/ManagedStatic.h>
15#include <toolchain/toolchain.h>
16
17#include <IR_Gen/idisa_target.h>
18#include <boost/filesystem.hpp>
19#include <boost/iostreams/device/mapped_file.hpp>
20
21#include <cc/cc_compiler.h>
22#include <kernels/cc_kernel.h>
23#include <kernels/streamset.h>
24#include <kernels/s2p_kernel.h>
25#include <kernels/source_kernel.h>
26#include <kernels/stdout_kernel.h>
27#include <kernels/swizzle.h>
28
29#include <kernels/kernel_builder.h>
30#include <kernels/p2s_kernel.h>
31#include <toolchain/cpudriver.h>
32#include <iostream>
33#include <fstream>
34#include <kernels/deletion.h>
35
36namespace re { class CC; }
37
38using namespace llvm;
39using namespace parabix;
40using namespace kernel;
41
42static cl::OptionCategory characterDeletionFlags("Command Flags", "deletion options");
43static cl::opt<char> characterToBeDeleted(cl::Positional, cl::desc("<character to be deletion>"), cl::Required, cl::cat(characterDeletionFlags));
44static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(characterDeletionFlags));
45static cl::opt<bool> useSwizzledDeletion("swizzle-deletion", cl::desc("Use swizzle deletion"), cl::init(false), cl::cat(characterDeletionFlags));
46
47
48typedef void (*MainFunctionType)(char * byte_data, size_t filesize);
49
50/*
51 * Usage:
52 *   character_deletion <character to be deleted> <input file name>
53 * It will delete the character from the input file and then print the output to stdout
54 * */
55
56StreamSetBuffer * loadBasisBits(ParabixDriver & pxDriver, Value* inputStream, Value* fileSize, int inputBufferBlocks) {
57    auto & iBuilder = pxDriver.getBuilder();
58
59    StreamSetBuffer * ByteStream = pxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
60    StreamSetBuffer * BasisBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), inputBufferBlocks);
61
62    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder);
63    sourceK->setInitialArguments({inputStream, fileSize});
64    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
65
66    Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder, /*aligned = */ true);
67    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
68
69    return BasisBits;
70}
71
72StreamSetBuffer * generateSwizzledDeletion(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int inputBufferBlocks) {
73    auto & iBuilder = pxDriver.getBuilder();
74
75    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
76    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::makeCC(characterToBeDeleted)}, 8);
77
78    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
79
80    StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks);
81    StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks);
82    Kernel * delK = pxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(iBuilder, 8);
83    pxDriver.makeKernelCall(delK, {CharacterMarkerBuffer, BasisBits}, {u16Swizzle0, u16Swizzle1});
84
85    // Produce unswizzled bit streams
86    StreamSetBuffer * deletedBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
87    Kernel * unSwizzleK = pxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
88
89    pxDriver.makeKernelCall(unSwizzleK, {u16Swizzle0, u16Swizzle1}, {deletedBits});
90    return deletedBits;
91}
92
93// TODO: It seems that there are still some bugs in DeleteByPEXTkernel
94StreamSetBuffer * generateDeletion(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int inputBufferBlocks) {
95    auto & iBuilder = pxDriver.getBuilder();
96
97    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
98    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::makeCC(characterToBeDeleted)}, 8);
99    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
100
101    StreamSetBuffer * deletedBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
102    StreamSetBuffer * deletionCounts = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
103
104    Kernel * delK = pxDriver.addKernelInstance<DeleteByPEXTkernel>(iBuilder, 64, 8);
105    pxDriver.makeKernelCall(delK, {BasisBits, CharacterMarkerBuffer}, {deletedBits, deletionCounts});
106
107    StreamSetBuffer * compressedBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
108    Kernel * streamCompressionK = pxDriver.addKernelInstance<StreamCompressKernel>(iBuilder, 64, 8);
109    pxDriver.makeKernelCall(streamCompressionK, {deletedBits, deletionCounts}, {compressedBits});
110
111    return compressedBits;
112}
113
114StreamSetBuffer * generateDeletionByCompression(ParabixDriver & pxDriver, StreamSetBuffer * BasisBits, int inputBufferBlocks) {
115    auto & iBuilder = pxDriver.getBuilder();
116
117    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
118    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "deletionMarker", std::vector<re::CC *>{re::subtractCC(re::makeByte(0, 255), re::makeCC(characterToBeDeleted))}, 8);
119    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
120
121    StreamSetBuffer * compressedBits = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(8), inputBufferBlocks);
122    StreamFilterCompiler filterCompiler(pxDriver, iBuilder->getStreamSetTy(8), inputBufferBlocks);
123    filterCompiler.makeCall(CharacterMarkerBuffer, BasisBits, compressedBits);
124    return compressedBits;
125}
126
127
128int main(int argc, char *argv[]) {
129    // This boilerplate provides convenient stack traces and clean LLVM exit
130    // handling. It also initializes the built in support for convenient
131    // command line option handling.
132    sys::PrintStackTraceOnErrorSignal(argv[0]);
133    llvm::PrettyStackTraceProgram X(argc, argv);
134    llvm_shutdown_obj shutdown;
135    codegen::ParseCommandLineOptions(argc, argv, {&characterDeletionFlags, codegen::codegen_flags()});
136
137    const std::string fileName = inputFile;
138
139    std::ifstream f(fileName, std::ios::binary | std::ios::ate);
140    if (f.fail()) {
141        return -1;
142    }
143    size_t mFilesize = f.tellg();
144
145    boost::iostreams::mapped_file_source mappedFile;
146    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
147    mappedFile.open(fileName , mFilesize);
148
149    char *fileBuffer = const_cast<char *>(mappedFile.data());
150
151    const int inputBufferBlocks = codegen::BufferSegments * codegen::ThreadNum * 16;
152
153    ParabixDriver pxDriver("character_deletion");
154    auto & iBuilder = pxDriver.getBuilder();
155    Module * M = iBuilder->getModule();
156    Type * const sizeTy = iBuilder->getSizeTy();
157    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
158    Type * const voidTy = iBuilder->getVoidTy();
159    Type * const inputType = iBuilder->getInt8PtrTy();
160
161    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, sizeTy, boolTy, nullptr));
162    main->setCallingConv(CallingConv::C);
163    Function::arg_iterator args = main->arg_begin();
164    Value* inputStream = &*(args++);
165    inputStream->setName("input");
166
167    Value* fileSize = &*(args++);
168    fileSize->setName("fileSize");
169
170    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
171
172    // GeneratePipeline
173    StreamSetBuffer * BasisBits = loadBasisBits(pxDriver, inputStream, fileSize, inputBufferBlocks);
174    StreamSetBuffer * deletedBits = NULL;
175    if (useSwizzledDeletion) {
176        deletedBits = generateSwizzledDeletion(pxDriver, BasisBits, inputBufferBlocks);
177    } else {
178        deletedBits = generateDeletionByCompression(pxDriver, BasisBits, inputBufferBlocks);
179    }
180//    StreamSetBuffer * deletedBits = generateDeletion(pxDriver, BasisBits, inputBufferBlocks);
181
182    StreamSetBuffer * const deletedByteStream = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), inputBufferBlocks);
183    Kernel * p2sK = pxDriver.addKernelInstance<P2SKernel>(iBuilder);
184    pxDriver.makeKernelCall(p2sK, {deletedBits}, {deletedByteStream});
185
186    // --------------------------------------------------------
187    // End
188
189    Kernel * outK = pxDriver.addKernelInstance<StdOutKernel>(iBuilder, 8);
190    pxDriver.makeKernelCall(outK, {deletedByteStream}, {});
191
192    pxDriver.generatePipelineIR();
193
194    pxDriver.deallocateBuffers();
195
196    iBuilder->CreateRetVoid();
197
198    pxDriver.finalizeObject();
199
200
201    auto mainFunc = reinterpret_cast<MainFunctionType>(pxDriver.getMain());
202
203    mainFunc(fileBuffer, mFilesize);
204
205    mappedFile.close();
206    return 0;
207}
Note: See TracBrowser for help on using the repository browser.