source: icGREP/icgrep-devel/icgrep/base64.cpp @ 5277

Last change on this file since 5277 was 5277, checked in by cameron, 2 years ago

radix64/base64 fixes

File size: 7.0 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <iostream>
8#include <llvm/IR/Function.h>
9#include <llvm/IR/Module.h>
10#include <llvm/ExecutionEngine/ExecutionEngine.h>
11#include <llvm/IR/Verifier.h>
12#include <llvm/Support/CommandLine.h>
13#include <toolchain.h>
14#include <IR_Gen/idisa_builder.h>
15#include <IR_Gen/idisa_target.h>
16#include <kernels/pipeline.h>
17#include <kernels/mmap_kernel.h>
18#include <kernels/streamset.h>
19#include <kernels/radix64.h>
20#include <kernels/stdout_kernel.h>
21#include <boost/filesystem.hpp>
22#include <boost/iostreams/device/mapped_file.hpp>
23#include <boost/interprocess/anonymous_shared_memory.hpp>
24
25using namespace llvm;
26
27static cl::OptionCategory base64Options("base64 Options",
28                                            "Transcoding control options.");
29
30static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(base64Options));
31
32static cl::opt<bool> segmentPipelineParallel("enable-segment-pipeline-parallel", cl::desc("Enable multithreading with segment pipeline parallelism."), cl::cat(base64Options));
33static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(base64Options));
34static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(base64Options));
35
36
37using namespace kernel;
38using namespace parabix;
39
40Function * base64Pipeline(Module * mMod, IDISA::IDISA_Builder * iBuilder) {
41    Type * mBitBlockType = iBuilder->getBitBlockType();
42
43    Type * const size_ty = iBuilder->getSizeTy();
44    Type * const voidTy = Type::getVoidTy(mMod->getContext());
45    Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
46    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
47   
48   
49    Function * const main = cast<Function>(mMod->getOrInsertFunction("Main", voidTy, inputType, outputType, size_ty, nullptr));
50    main->setCallingConv(CallingConv::C);
51    Function::arg_iterator args = main->arg_begin();
52   
53    Value * const inputStream = &*(args++);
54    inputStream->setName("inputStream");
55    Value * const outputStream = &*(args++);
56    outputStream->setName("outputStream");
57    Value * const fileSize = &*(args++);
58    fileSize->setName("fileSize");
59
60    //Round up to a multiple of 3.
61    const unsigned segmentSize = ((codegen::SegmentSize + 2)/3) * 3;
62   
63    const unsigned bufferSegments = codegen::BufferSegments;
64   
65    ExternalFileBuffer ByteStream(iBuilder, iBuilder->getStreamSetTy(1, 8));
66
67    CircularBuffer Expanded3_4Out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
68    CircularBuffer Radix64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
69    LinearCopybackBuffer Base64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
70   
71    MMapSourceKernel mmapK(iBuilder, segmentSize);
72    mmapK.generateKernel({}, {&ByteStream});
73    mmapK.setInitialArguments({fileSize});
74       
75    expand3_4Kernel expandK(iBuilder);
76    expandK.generateKernel({&ByteStream}, {&Expanded3_4Out});
77
78    radix64Kernel radix64K(iBuilder);
79    radix64K.generateKernel({&Expanded3_4Out}, {&Radix64out});
80
81    base64Kernel base64K(iBuilder);
82    base64K.generateKernel({&Radix64out}, {&Base64out});
83   
84    StdOutKernel stdoutK(iBuilder, 8);
85    stdoutK.generateKernel({&Base64out}, {});
86
87   
88    iBuilder->SetInsertPoint(BasicBlock::Create(mMod->getContext(), "entry", main,0));
89
90    ByteStream.setStreamSetBuffer(inputStream, fileSize);
91//    Radix64out.setEmptyBuffer(iBuilder->CreatePointerCast(outputStream, outputType));
92    Expanded3_4Out.allocateBuffer();
93    Radix64out.allocateBuffer();
94    Base64out.allocateBuffer();
95
96
97    if (segmentPipelineParallel){
98        generateSegmentParallelPipeline(iBuilder, {&mmapK, &expandK, &radix64K, &base64K, &stdoutK});
99    }
100    else{
101        generatePipelineLoop(iBuilder, {&mmapK, &expandK, &radix64K, &base64K, &stdoutK});
102    }
103
104    iBuilder->CreateRetVoid();
105    return main;
106}
107
108
109typedef void (*base64FunctionType)(char * byte_data, char * output_data, size_t filesize);
110
111static ExecutionEngine * base64Engine = nullptr;
112
113base64FunctionType base64CodeGen(void) {
114    LLVMContext TheContext;                           
115    Module * M = new Module("base64", TheContext);
116    IDISA::IDISA_Builder * idb = IDISA::GetIDISA_Builder(M);
117
118   
119    llvm::Function * main_IR = base64Pipeline(M, idb);
120   
121    verifyModule(*M, &dbgs());
122    //std::cerr << "ExecuteKernels(); done\n";
123    base64Engine = JIT_to_ExecutionEngine(M);
124   
125    base64Engine->finalizeObject();
126    //std::cerr << "finalizeObject(); done\n";
127
128    delete idb;
129    return reinterpret_cast<base64FunctionType>(base64Engine->getPointerToFunction(main_IR));
130}
131
132void base64(base64FunctionType fn_ptr, const std::string & fileName) {
133    std::string mFileName = fileName;
134    size_t mFileSize;
135    char * mFileBuffer;
136
137    const boost::filesystem::path file(mFileName);
138    if (exists(file)) {
139        if (is_directory(file)) {
140            return;
141        }
142    } else {
143        std::cerr << "Error: cannot open " << mFileName << " for processing. Skipped.\n";
144        return;
145    }
146   
147    mFileSize = file_size(file);
148    boost::iostreams::mapped_file_source mFile;
149    if (mFileSize == 0) {
150        mFileBuffer = nullptr;
151    }
152    else {
153        try {
154            mFile.open(mFileName);
155        } catch (std::exception &e) {
156            std::cerr << "Error: Boost mmap of " << mFileName << ": " << e.what() << std::endl;
157            return;
158        }
159        mFileBuffer = const_cast<char *>(mFile.data());
160    }
161
162    if (mMapBuffering) {
163        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2*mFileSize));
164        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
165        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
166        std::cerr << "outputBuffer " << std::hex << (intptr_t)(outputBuffer.get_address()) << std::dec << "\n";
167        fn_ptr(mFileBuffer, static_cast<char*>(outputBuffer.get_address()), mFileSize);
168    }
169    else if (memAlignBuffering) {
170        char * outputBuffer;
171        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2*mFileSize)) {
172            throw std::bad_alloc();
173        }
174        fn_ptr(mFileBuffer, outputBuffer, mFileSize);
175        free(reinterpret_cast<void *>(outputBuffer));
176    }
177    else {
178        /* No external output buffer */
179        fn_ptr(mFileBuffer, nullptr, mFileSize);
180    }
181    mFile.close();
182   
183}
184
185
186int main(int argc, char *argv[]) {
187    cl::HideUnrelatedOptions(ArrayRef<const cl::OptionCategory *>{&base64Options, codegen::codegen_flags()});
188    cl::ParseCommandLineOptions(argc, argv);
189
190    base64FunctionType fn_ptr = base64CodeGen();
191
192    for (unsigned i = 0; i != inputFiles.size(); ++i) {
193        base64(fn_ptr, inputFiles[i]);
194    }
195
196    return 0;
197}
198
Note: See TracBrowser for help on using the repository browser.