source: icGREP/icgrep-devel/icgrep/base64.cpp @ 5240

Last change on this file since 5240 was 5240, checked in by nmedfort, 2 years ago

Cleaned up memory leaks + some warning messages.

File size: 8.8 KB
RevLine 
[5232]1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <string>
8#include <iostream>
9#include <fstream>
10#include <sstream>
11#include <stdlib.h>
12
13#include <llvm/IR/Function.h>
14#include <llvm/IR/Module.h>
15#include <llvm/ExecutionEngine/ExecutionEngine.h>
16#include <llvm/ExecutionEngine/MCJIT.h>
17#include <llvm/IRReader/IRReader.h>
18#include <llvm/IR/Verifier.h>
19#include <llvm/Support/Debug.h>
20
21#include <llvm/Support/CommandLine.h>
22
23#include <toolchain.h>
[5238]24#include <IR_Gen/idisa_builder.h>
25#include <IR_Gen/idisa_target.h>
[5232]26#include <kernels/pipeline.h>
27#include <kernels/interface.h>
28#include <kernels/kernel.h>
29#include <kernels/radix64.h>
30#include <kernels/stdout_kernel.h>
31
32
33// mmap system
34#include <boost/filesystem.hpp>
35#include <boost/iostreams/device/mapped_file.hpp>
36#include <boost/interprocess/anonymous_shared_memory.hpp>
37#include <boost/interprocess/mapped_region.hpp>
38#include <fcntl.h>
39static cl::OptionCategory base64Options("base64 Options",
40                                            "Transcoding control options.");
41
42static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(base64Options));
43
44static cl::opt<bool> segmentPipelineParallel("enable-segment-pipeline-parallel", cl::desc("Enable multithreading with segment pipeline parallelism."), cl::cat(base64Options));
45static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(base64Options));
46static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(base64Options));
47
48
49using namespace kernel;
50using namespace parabix;
51
52Function * base64Pipeline(Module * mMod, IDISA::IDISA_Builder * iBuilder) {
53    Type * mBitBlockType = iBuilder->getBitBlockType();
54
55    //Round up to a multiple of 4.
56    const unsigned segmentSize = ((codegen::SegmentSize + 3)/4) * 4;
57   
58    const unsigned bufferSegments = codegen::BufferSegments;
59   
60    ExternalFileBuffer ByteStream(iBuilder, iBuilder->getStreamSetTy(1, 8));
61
62    CircularBuffer Expanded3_4Out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * bufferSegments * 16);
63    CircularBuffer Radix64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * bufferSegments * 16);
64    LinearCopybackBuffer Base64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * bufferSegments * 16 + 2);
65
66    expand3_4Kernel expandK(iBuilder);
67    expandK.generateKernel({&ByteStream}, {&Expanded3_4Out});
68
69    radix64Kernel radix64K(iBuilder);
70    radix64K.generateKernel({&Expanded3_4Out}, {&Radix64out});
71
72    base64Kernel base64K(iBuilder);
73    base64K.generateKernel({&Radix64out}, {&Base64out});
74   
75    StdOutKernel stdoutK(iBuilder, 8);
76    stdoutK.generateKernel({&Base64out}, {});
77
78    Type * const size_ty = iBuilder->getSizeTy();
79    Type * const voidTy = Type::getVoidTy(mMod->getContext());
80    Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
81    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
82    Type * const int32ty = iBuilder->getInt32Ty();
83    Type * const int8PtrTy = iBuilder->getInt8PtrTy();
[5240]84    Type * const voidPtrTy = iBuilder->getVoidPtrTy();
[5232]85
86   
87    Function * const main = cast<Function>(mMod->getOrInsertFunction("Main", voidTy, inputType, outputType, size_ty, nullptr));
88    main->setCallingConv(CallingConv::C);
89    Function::arg_iterator args = main->arg_begin();
90   
91    Value * const inputStream = &*(args++);
92    inputStream->setName("inputStream");
93    Value * const outputStream = &*(args++);
94    outputStream->setName("outputStream");
95    Value * const fileSize = &*(args++);
96    fileSize->setName("fileSize");
97   
98    iBuilder->SetInsertPoint(BasicBlock::Create(mMod->getContext(), "entry", main,0));
99
100    ByteStream.setStreamSetBuffer(inputStream, fileSize);
101//    Radix64out.setEmptyBuffer(iBuilder->CreatePointerCast(outputStream, outputType));
102    mMod->dump();
103    Expanded3_4Out.allocateBuffer();
104    Radix64out.allocateBuffer();
105    Base64out.allocateBuffer();
106
107
108    Type * pthreadTy = size_ty;
109    FunctionType * funVoidPtrVoidTy = FunctionType::get(voidTy, int8PtrTy, false);
110   
111    Function * pthreadCreateFunc = cast<Function>(mMod->getOrInsertFunction("pthread_create",
112                                                                         int32ty,
113                                                                         pthreadTy->getPointerTo(),
114                                                                         voidPtrTy,
115                                                                         static_cast<Type *>(funVoidPtrVoidTy)->getPointerTo(),
116                                                                         voidPtrTy, nullptr));
117    pthreadCreateFunc->setCallingConv(llvm::CallingConv::C);
118    Function * pthreadJoinFunc = cast<Function>(mMod->getOrInsertFunction("pthread_join",
119                                                                       int32ty,
120                                                                       pthreadTy,
121                                                                       PointerType::get(int8PtrTy, 0), nullptr));
122    pthreadJoinFunc->setCallingConv(llvm::CallingConv::C);
123   
124    Function * pthreadExitFunc = cast<Function>(mMod->getOrInsertFunction("pthread_exit",
125                                                                       voidTy, 
126                                                                       voidPtrTy, nullptr));
127    pthreadExitFunc->addFnAttr(llvm::Attribute::NoReturn);
128    pthreadExitFunc->setCallingConv(llvm::CallingConv::C);
129
130    if (segmentPipelineParallel){
131        generateSegmentParallelPipeline(iBuilder, {&expandK, &radix64K, &base64K, &stdoutK});
132    }
133    else{
134        generatePipelineLoop(iBuilder, {&expandK, &radix64K, &base64K, &stdoutK});
135    }
136
137    iBuilder->CreateRetVoid();
138    return main;
139}
140
141
142typedef void (*base64FunctionType)(char * byte_data, char * output_data, size_t filesize);
143
144static ExecutionEngine * base64Engine = nullptr;
145
146base64FunctionType base64CodeGen(void) {
147    LLVMContext TheContext;                           
148    Module * M = new Module("base64", TheContext);
149    IDISA::IDISA_Builder * idb = IDISA::GetIDISA_Builder(M);
150
151   
152    llvm::Function * main_IR = base64Pipeline(M, idb);
153   
154    verifyModule(*M, &dbgs());
155    //std::cerr << "ExecuteKernels(); done\n";
156    base64Engine = JIT_to_ExecutionEngine(M);
157   
158    base64Engine->finalizeObject();
159    //std::cerr << "finalizeObject(); done\n";
160
161    delete idb;
162    return reinterpret_cast<base64FunctionType>(base64Engine->getPointerToFunction(main_IR));
163}
164
165void base64(base64FunctionType fn_ptr, const std::string & fileName) {
166    std::string mFileName = fileName;
167    size_t mFileSize;
168    char * mFileBuffer;
[5240]169
[5232]170    const boost::filesystem::path file(mFileName);
171    if (exists(file)) {
172        if (is_directory(file)) {
173            return;
174        }
175    } else {
176        std::cerr << "Error: cannot open " << mFileName << " for processing. Skipped.\n";
177        return;
178    }
179   
180    mFileSize = file_size(file);
181    boost::iostreams::mapped_file_source mFile;
182    if (mFileSize == 0) {
183        mFileBuffer = nullptr;
184    }
185    else {
186        try {
187            mFile.open(mFileName);
188        } catch (std::exception &e) {
189            std::cerr << "Error: Boost mmap of " << mFileName << ": " << e.what() << std::endl;
190            return;
191        }
192        mFileBuffer = const_cast<char *>(mFile.data());
193    }
194
195    if (mMapBuffering) {
196        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2*mFileSize));
197        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
198        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
199        std::cerr << "outputBuffer " << std::hex << (intptr_t)(outputBuffer.get_address()) << std::dec << "\n";
200        fn_ptr(mFileBuffer, static_cast<char*>(outputBuffer.get_address()), mFileSize);
201    }
202    else if (memAlignBuffering) {
203        char * outputBuffer;
[5240]204        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2*mFileSize)) {
205            throw std::bad_alloc();
206        }
[5232]207        fn_ptr(mFileBuffer, outputBuffer, mFileSize);
208        free(reinterpret_cast<void *>(outputBuffer));
209    }
210    else {
211        /* No external output buffer */
212        fn_ptr(mFileBuffer, nullptr, mFileSize);
213    }
214    mFile.close();
215   
216}
217
218
219int main(int argc, char *argv[]) {
220    cl::HideUnrelatedOptions(ArrayRef<const cl::OptionCategory *>{&base64Options, codegen::codegen_flags()});
221    cl::ParseCommandLineOptions(argc, argv);
222
223    base64FunctionType fn_ptr = base64CodeGen();
224
225    for (unsigned i = 0; i != inputFiles.size(); ++i) {
226        base64(fn_ptr, inputFiles[i]);
227    }
228
229    return 0;
230}
231
Note: See TracBrowser for help on using the repository browser.