source: icGREP/icgrep-devel/icgrep/base64.cpp @ 5260

Last change on this file since 5260 was 5260, checked in by nmedfort, 2 years ago

Changes working towards simplifying accessing stream elements + some modifications to simplify include / forward declarations within the CodeGen? library.

File size: 7.3 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <string>
8#include <iostream>
9#include <fstream>
10#include <sstream>
11#include <stdlib.h>
12
13#include <llvm/IR/Function.h>
14#include <llvm/IR/Module.h>
15#include <llvm/ExecutionEngine/ExecutionEngine.h>
16#include <llvm/ExecutionEngine/MCJIT.h>
17#include <llvm/IRReader/IRReader.h>
18#include <llvm/IR/Verifier.h>
19#include <llvm/Support/Debug.h>
20
21#include <llvm/Support/CommandLine.h>
22
23#include <toolchain.h>
24#include <IR_Gen/idisa_builder.h>
25#include <IR_Gen/idisa_target.h>
26#include <kernels/pipeline.h>
27#include <kernels/mmap_kernel.h>
28#include <kernels/interface.h>
29#include <kernels/kernel.h>
30#include <kernels/radix64.h>
31#include <kernels/stdout_kernel.h>
32
33
34// mmap system
35#include <boost/filesystem.hpp>
36#include <boost/iostreams/device/mapped_file.hpp>
37#include <boost/interprocess/anonymous_shared_memory.hpp>
38#include <boost/interprocess/mapped_region.hpp>
39#include <fcntl.h>
40
41
42using namespace llvm;
43
44static cl::OptionCategory base64Options("base64 Options",
45                                            "Transcoding control options.");
46
47static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(base64Options));
48
49static cl::opt<bool> segmentPipelineParallel("enable-segment-pipeline-parallel", cl::desc("Enable multithreading with segment pipeline parallelism."), cl::cat(base64Options));
50static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(base64Options));
51static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(base64Options));
52
53
54using namespace kernel;
55using namespace parabix;
56
57Function * base64Pipeline(Module * mMod, IDISA::IDISA_Builder * iBuilder) {
58    Type * mBitBlockType = iBuilder->getBitBlockType();
59
60    Type * const size_ty = iBuilder->getSizeTy();
61    Type * const voidTy = Type::getVoidTy(mMod->getContext());
62    Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
63    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
64   
65   
66    Function * const main = cast<Function>(mMod->getOrInsertFunction("Main", voidTy, inputType, outputType, size_ty, nullptr));
67    main->setCallingConv(CallingConv::C);
68    Function::arg_iterator args = main->arg_begin();
69   
70    Value * const inputStream = &*(args++);
71    inputStream->setName("inputStream");
72    Value * const outputStream = &*(args++);
73    outputStream->setName("outputStream");
74    Value * const fileSize = &*(args++);
75    fileSize->setName("fileSize");
76
77    //Round up to a multiple of 4.
78    const unsigned segmentSize = ((codegen::SegmentSize + 3)/4) * 4;
79   
80    const unsigned bufferSegments = codegen::BufferSegments;
81   
82    ExternalFileBuffer ByteStream(iBuilder, iBuilder->getStreamSetTy(1, 8));
83
84    CircularBuffer Expanded3_4Out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * bufferSegments * 16);
85    CircularBuffer Radix64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * bufferSegments * 16);
86    LinearCopybackBuffer Base64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * bufferSegments * 16 + 2);
87   
88    MMapSourceKernel mmapK(iBuilder, segmentSize * bufferSegments * 16); 
89    mmapK.generateKernel({}, {&ByteStream});
90    mmapK.setInitialArguments({fileSize});
91       
92    expand3_4Kernel expandK(iBuilder);
93    expandK.generateKernel({&ByteStream}, {&Expanded3_4Out});
94
95    radix64Kernel radix64K(iBuilder);
96    radix64K.generateKernel({&Expanded3_4Out}, {&Radix64out});
97
98    base64Kernel base64K(iBuilder);
99    base64K.generateKernel({&Radix64out}, {&Base64out});
100   
101    StdOutKernel stdoutK(iBuilder, 8);
102    stdoutK.generateKernel({&Base64out}, {});
103
104   
105    iBuilder->SetInsertPoint(BasicBlock::Create(mMod->getContext(), "entry", main,0));
106
107    ByteStream.setStreamSetBuffer(inputStream, fileSize);
108//    Radix64out.setEmptyBuffer(iBuilder->CreatePointerCast(outputStream, outputType));
109    Expanded3_4Out.allocateBuffer();
110    Radix64out.allocateBuffer();
111    Base64out.allocateBuffer();
112
113
114    if (segmentPipelineParallel){
115        generateSegmentParallelPipeline(iBuilder, {&mmapK, &expandK, &radix64K, &base64K, &stdoutK});
116    }
117    else{
118        generatePipelineLoop(iBuilder, {&mmapK, &expandK, &radix64K, &base64K, &stdoutK});
119    }
120
121    iBuilder->CreateRetVoid();
122    return main;
123}
124
125
126typedef void (*base64FunctionType)(char * byte_data, char * output_data, size_t filesize);
127
128static ExecutionEngine * base64Engine = nullptr;
129
130base64FunctionType base64CodeGen(void) {
131    LLVMContext TheContext;                           
132    Module * M = new Module("base64", TheContext);
133    IDISA::IDISA_Builder * idb = IDISA::GetIDISA_Builder(M);
134
135   
136    llvm::Function * main_IR = base64Pipeline(M, idb);
137   
138    verifyModule(*M, &dbgs());
139    //std::cerr << "ExecuteKernels(); done\n";
140    base64Engine = JIT_to_ExecutionEngine(M);
141   
142    base64Engine->finalizeObject();
143    //std::cerr << "finalizeObject(); done\n";
144
145    delete idb;
146    return reinterpret_cast<base64FunctionType>(base64Engine->getPointerToFunction(main_IR));
147}
148
149void base64(base64FunctionType fn_ptr, const std::string & fileName) {
150    std::string mFileName = fileName;
151    size_t mFileSize;
152    char * mFileBuffer;
153
154    const boost::filesystem::path file(mFileName);
155    if (exists(file)) {
156        if (is_directory(file)) {
157            return;
158        }
159    } else {
160        std::cerr << "Error: cannot open " << mFileName << " for processing. Skipped.\n";
161        return;
162    }
163   
164    mFileSize = file_size(file);
165    boost::iostreams::mapped_file_source mFile;
166    if (mFileSize == 0) {
167        mFileBuffer = nullptr;
168    }
169    else {
170        try {
171            mFile.open(mFileName);
172        } catch (std::exception &e) {
173            std::cerr << "Error: Boost mmap of " << mFileName << ": " << e.what() << std::endl;
174            return;
175        }
176        mFileBuffer = const_cast<char *>(mFile.data());
177    }
178
179    if (mMapBuffering) {
180        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2*mFileSize));
181        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
182        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
183        std::cerr << "outputBuffer " << std::hex << (intptr_t)(outputBuffer.get_address()) << std::dec << "\n";
184        fn_ptr(mFileBuffer, static_cast<char*>(outputBuffer.get_address()), mFileSize);
185    }
186    else if (memAlignBuffering) {
187        char * outputBuffer;
188        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2*mFileSize)) {
189            throw std::bad_alloc();
190        }
191        fn_ptr(mFileBuffer, outputBuffer, mFileSize);
192        free(reinterpret_cast<void *>(outputBuffer));
193    }
194    else {
195        /* No external output buffer */
196        fn_ptr(mFileBuffer, nullptr, mFileSize);
197    }
198    mFile.close();
199   
200}
201
202
203int main(int argc, char *argv[]) {
204    cl::HideUnrelatedOptions(ArrayRef<const cl::OptionCategory *>{&base64Options, codegen::codegen_flags()});
205    cl::ParseCommandLineOptions(argc, argv);
206
207    base64FunctionType fn_ptr = base64CodeGen();
208
209    for (unsigned i = 0; i != inputFiles.size(); ++i) {
210        base64(fn_ptr, inputFiles[i]);
211    }
212
213    return 0;
214}
215
Note: See TracBrowser for help on using the repository browser.