source: icGREP/icgrep-devel/icgrep/character_deposit.cpp @ 5937

Last change on this file since 5937 was 5873, checked in by xwa163, 20 months ago
  1. Implement LZ4MultiplePdepKernel, which can do PDEP for more than one input streamset
  2. Add attributed Swizzled in StreamSet?, copy at least one whole block when doing temporary buffer copy for Swizzled InputStreamSet?
  3. Bug fixing for character_deposit pipeline
  4. Add more test files for character_deposit pipeline
File size: 8.3 KB
RevLine 
[5857]1
2/*
3 *  Copyright (c) 2017 International Characters.
4 *  This software is licensed to the public under the Open Software License 3.0.
5 *  icgrep is a trademark of International Characters.
6 */
7
8#include <llvm/IR/Module.h>
9#include <llvm/IR/Function.h>
10#include <llvm/Linker/Linker.h>
11#include <llvm/Support/CommandLine.h>
12#include <llvm/Support/PrettyStackTrace.h>
13#include <llvm/Support/Signals.h>
14#include <llvm/Support/ManagedStatic.h>
15#include <toolchain/toolchain.h>
16
17#include <IR_Gen/idisa_target.h>
18#include <boost/filesystem.hpp>
19#include <boost/iostreams/device/mapped_file.hpp>
20
21#include <cc/cc_compiler.h>
22#include <kernels/cc_kernel.h>
23#include <kernels/streamset.h>
24#include <kernels/s2p_kernel.h>
25#include <kernels/source_kernel.h>
26#include <kernels/stdout_kernel.h>
27#include <kernels/swizzle.h>
28
29#include <kernels/kernel_builder.h>
30#include <kernels/p2s_kernel.h>
31#include <toolchain/cpudriver.h>
32#include <iostream>
33#include <fstream>
34#include <kernels/deletion.h>
35#include <kernels/pdep_kernel.h>
[5873]36#include <kernels/lz4/lz4_multiple_pdep_kernel.h>
[5857]37
38namespace re { class CC; }
39
40using namespace llvm;
41using namespace parabix;
42using namespace kernel;
43
44static cl::OptionCategory lz4dFlags("Command Flags", "deletion options");
45//static cl::opt<char> characterToBeExtract(cl::Positional, cl::desc("<character to be extracted>"), cl::Required, cl::cat(lz4dFlags));
46static cl::opt<char> characterToBeDeposit(cl::Positional, cl::desc("<character to be depositted>"), cl::Required, cl::cat(lz4dFlags));
47static cl::opt<std::string> inputFile(cl::Positional, cl::desc("<input file>"), cl::Required, cl::cat(lz4dFlags));
48static cl::opt<std::string> outputFile(cl::Positional, cl::desc("<output file>"), cl::Required, cl::cat(lz4dFlags));
49
50
51typedef void (*MainFunctionType)(char * byte_data, size_t filesize);
52
53/*
54 * Usage:
55 *   character_deletion <character to be deleted> <input file name>
56 * It will delete the character from the input file and then print the output to stdout
57 * */
58
59int main(int argc, char *argv[]) {
60    // This boilerplate provides convenient stack traces and clean LLVM exit
61    // handling. It also initializes the built in support for convenient
62    // command line option handling.
63    sys::PrintStackTraceOnErrorSignal(argv[0]);
64    llvm::PrettyStackTraceProgram X(argc, argv);
65    llvm_shutdown_obj shutdown;
66    codegen::ParseCommandLineOptions(argc, argv, {&lz4dFlags, codegen::codegen_flags()});
67
68    std::string fileName = inputFile;
69
70    std::ifstream f(fileName, std::ios::binary | std::ios::ate);
71    if (f.fail()) {
72        return -1;
73    }
74    size_t mFilesize = f.tellg();
75
76    boost::iostreams::mapped_file_source mappedFile;
77    // Since mmap offset has to be multiples of pages, we can't use it to skip headers.
78    mappedFile.open(fileName , mFilesize);
79
80    char *fileBuffer = const_cast<char *>(mappedFile.data());
81
82    if (codegen::SegmentSize < 2) {
83        codegen::SegmentSize = 2;
84    }
85
[5865]86    const int inputBufferBlocks = codegen::BufferSegments * codegen::ThreadNum * codegen::SegmentSize;
87    const int outputBufferBlocks = inputBufferBlocks; // * 2;
[5857]88
89    ParabixDriver pxDriver("character_deletion");
90    auto & iBuilder = pxDriver.getBuilder();
91    Module * M = iBuilder->getModule();
92    Type * const sizeTy = iBuilder->getSizeTy();
93    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
94    Type * const voidTy = iBuilder->getVoidTy();
95    Type * const inputType = iBuilder->getInt8PtrTy();
96
97    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, sizeTy, boolTy, nullptr));
98    main->setCallingConv(CallingConv::C);
99    Function::arg_iterator args = main->arg_begin();
100    Value* inputStream = &*(args++);
101    inputStream->setName("input");
102
103    Value* fileSize = &*(args++);
104    fileSize->setName("fileSize");
105
106    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
107
108
109    // GeneratePipeline
110    StreamSetBuffer * ByteStream = pxDriver.addBuffer<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
111    StreamSetBuffer * BasisBits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1), inputBufferBlocks);
112
[5865]113    kernel::Kernel * sourceK = pxDriver.addKernelInstance<MemorySourceKernel>(iBuilder, iBuilder->getInt8PtrTy());
[5857]114    sourceK->setInitialArguments({inputStream, fileSize});
115    pxDriver.makeKernelCall(sourceK, {}, {ByteStream});
116    Kernel * s2pk = pxDriver.addKernelInstance<S2PKernel>(iBuilder, /*aligned = */ true);
117    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
118
119
[5865]120    StreamSetBuffer * const CharacterMarkerBuffer = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 1), inputBufferBlocks);
[5857]121    Kernel * ccK = pxDriver.addKernelInstance<ParabixCharacterClassKernelBuilder>(iBuilder, "extenders", std::vector<re::CC *>{re::makeCC(characterToBeDeposit)}, 8);
122    pxDriver.makeKernelCall(ccK, {BasisBits}, {CharacterMarkerBuffer});
123
124
[5865]125//    StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<SwizzledCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1);
126//    StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<SwizzledCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1);
127
128    StreamSetBuffer * u16Swizzle0 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1, 2);
129    StreamSetBuffer * u16Swizzle1 = pxDriver.addBuffer<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(4), inputBufferBlocks, 1, 2);
[5857]130    Kernel * delK = pxDriver.addKernelInstance<SwizzledDeleteByPEXTkernel>(iBuilder, 64, 8);
131    pxDriver.makeKernelCall(delK, {CharacterMarkerBuffer, BasisBits}, {u16Swizzle0, u16Swizzle1});
132
133    StreamSetBuffer * depositedSwizzle0 = pxDriver.addBuffer<SwizzledCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), outputBufferBlocks, 1);
[5873]134    StreamSetBuffer * depositedSwizzle1 = pxDriver.addBuffer<SwizzledCopybackBuffer>(iBuilder, iBuilder->getStreamSetTy(4), outputBufferBlocks, 1);
135
136    /*
[5857]137    Kernel * pdep0K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, 4, 64, "pdep0");
138    pxDriver.makeKernelCall(pdep0K, {CharacterMarkerBuffer, u16Swizzle0}, {depositedSwizzle0});
139
140    Kernel * pdep1K = pxDriver.addKernelInstance<PDEPkernel>(iBuilder, 4, 4, 64, "pdep1");
[5873]141    pxDriver.makeKernelCall(pdep1K, {CharacterMarkerBuffer, u16Swizzle1}, {u16Swizzle1});
142    */
[5857]143
[5873]144    Kernel * multiplePdepK = pxDriver.addKernelInstance<LZ4MultiplePDEPkernel>(iBuilder, 4, 2, 4);
145    pxDriver.makeKernelCall(multiplePdepK, {CharacterMarkerBuffer, u16Swizzle0, u16Swizzle1}, {depositedSwizzle0, depositedSwizzle1});
146
147    /*
148    Kernel * multiplePdepK1 = pxDriver.addKernelInstance<LZ4MultiplePDEPkernel>(iBuilder, 4, 1, 4);
149    pxDriver.makeKernelCall(multiplePdepK1, {CharacterMarkerBuffer, u16Swizzle0}, {depositedSwizzle0});
150    Kernel * multiplePdepK2 = pxDriver.addKernelInstance<LZ4MultiplePDEPkernel>(iBuilder, 4, 1, 4);
151    pxDriver.makeKernelCall(multiplePdepK2, {CharacterMarkerBuffer, u16Swizzle1}, {depositedSwizzle1});
152    */
153
[5857]154    // Produce unswizzled bit streams
155    StreamSetBuffer * resultbits = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(8), outputBufferBlocks);
156    Kernel * unSwizzleK = pxDriver.addKernelInstance<SwizzleGenerator>(iBuilder, 8, 1, 2);
157
158    pxDriver.makeKernelCall(unSwizzleK, {depositedSwizzle0, depositedSwizzle1}, {resultbits});
159//    pxDriver.makeKernelCall(unSwizzleK, {u16Swizzle0, u16Swizzle1}, {resultbits});
160
161    StreamSetBuffer * const ResultBytes = pxDriver.addBuffer<CircularBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), outputBufferBlocks);
162    Kernel * p2sK = pxDriver.addKernelInstance<P2SKernel>(iBuilder);
163    pxDriver.makeKernelCall(p2sK, {resultbits}, {ResultBytes});
164
165    // --------------------------------------------------------
166    // End
167/*
168    Kernel * outK = pxDriver.addKernelInstance<StdOutKernel>(iBuilder, 8);
169    pxDriver.makeKernelCall(outK, {DecompressedByteStream}, {});
170*/
171    Kernel * outK = pxDriver.addKernelInstance<FileSink>(iBuilder, 8);
172    outK->setInitialArguments({iBuilder->GetString(outputFile)});
173    pxDriver.makeKernelCall(outK, {ResultBytes}, {});
174
175    pxDriver.generatePipelineIR();
176
[5865]177    pxDriver.deallocateBuffers();
178
[5857]179    iBuilder->CreateRetVoid();
180
181    pxDriver.finalizeObject();
182
183
184    auto mainFunc = reinterpret_cast<MainFunctionType>(pxDriver.getMain());
185
186
187    mainFunc(fileBuffer, mFilesize);
188
189    mappedFile.close();
190    return 0;
191}
Note: See TracBrowser for help on using the repository browser.