source: icGREP/icgrep-devel/icgrep/base64.cpp @ 5396

Last change on this file since 5396 was 5396, checked in by cameron, 2 years ago

Update base64 and radix64 kernels for Parabix Driver and object cache system

File size: 6.4 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <iostream>
8#include <llvm/IR/Function.h>
9#include <llvm/IR/Module.h>
10#include <llvm/ExecutionEngine/ExecutionEngine.h>
11#include <llvm/IR/Verifier.h>
12#include <llvm/Support/CommandLine.h>
13#include <toolchain.h>
14#include <IR_Gen/idisa_builder.h>
15#include <IR_Gen/idisa_target.h>
16#include <kernels/pipeline.h>
17#include <kernels/mmap_kernel.h>
18#include <kernels/streamset.h>
19#include <kernels/radix64.h>
20#include <kernels/stdout_kernel.h>
21#include <boost/filesystem.hpp>
22#include <boost/iostreams/device/mapped_file.hpp>
23#include <boost/interprocess/anonymous_shared_memory.hpp>
24
25using namespace llvm;
26
27static cl::OptionCategory base64Options("base64 Options",
28                                            "Transcoding control options.");
29
30static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(base64Options));
31
32static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(base64Options));
33static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(base64Options));
34
35
36using namespace kernel;
37using namespace parabix;
38
39void base64PipelineGen(ParabixDriver & pxDriver) {
40       
41    IDISA::IDISA_Builder * iBuilder = pxDriver.getIDISA_Builder();
42    Module * mod = iBuilder->getModule();
43    Type * mBitBlockType = iBuilder->getBitBlockType();
44
45    Type * const size_ty = iBuilder->getSizeTy();
46    Type * const voidTy = Type::getVoidTy(mod->getContext());
47    Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
48    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
49   
50   
51    Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, inputType, outputType, size_ty, nullptr));
52    main->setCallingConv(CallingConv::C);
53    Function::arg_iterator args = main->arg_begin();
54   
55    Value * const inputStream = &*(args++);
56    inputStream->setName("inputStream");
57    Value * const outputStream = &*(args++);
58    outputStream->setName("outputStream");
59    Value * const fileSize = &*(args++);
60    fileSize->setName("fileSize");
61
62    //Round up to a multiple of 3.
63    const unsigned segmentSize = ((codegen::SegmentSize + 2)/3) * 3;
64   
65    const unsigned bufferSegments = codegen::BufferSegments;
66   
67    ExternalFileBuffer ByteStream(iBuilder, iBuilder->getStreamSetTy(1, 8));
68
69    CircularBuffer Expanded3_4Out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
70    CircularBuffer Radix64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
71    CircularCopybackBuffer Base64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments, 1);
72   
73    MMapSourceKernel mmapK(iBuilder, segmentSize);
74    mmapK.setInitialArguments({fileSize});
75    pxDriver.addKernelCall(mmapK, {}, {&ByteStream});
76   
77    expand3_4Kernel expandK(iBuilder);
78    pxDriver.addKernelCall(expandK, {&ByteStream}, {&Expanded3_4Out});
79
80    radix64Kernel radix64K(iBuilder);
81    pxDriver.addKernelCall(radix64K, {&Expanded3_4Out}, {&Radix64out});
82
83    base64Kernel base64K(iBuilder);
84    pxDriver.addKernelCall(base64K, {&Radix64out}, {&Base64out});
85   
86    StdOutKernel stdoutK(iBuilder, 8);
87    pxDriver.addKernelCall(stdoutK, {&Base64out}, {});
88   
89    iBuilder->SetInsertPoint(BasicBlock::Create(mod->getContext(), "entry", main,0));
90
91    ByteStream.setStreamSetBuffer(inputStream);
92    Expanded3_4Out.allocateBuffer();
93    Radix64out.allocateBuffer();
94    Base64out.allocateBuffer();
95
96    pxDriver.generatePipelineIR();
97
98    iBuilder->CreateRetVoid();
99    pxDriver.JITcompileMain();
100    pxDriver.linkAndFinalize();
101}
102
103
104typedef void (*base64FunctionType)(char * byte_data, char * output_data, size_t filesize);
105
106base64FunctionType base64CodeGen(void) {
107    LLVMContext TheContext;                           
108    Module * M = new Module("base64", TheContext);
109    IDISA::IDISA_Builder * idb = IDISA::GetIDISA_Builder(M);
110    ParabixDriver pxDriver(idb);
111   
112    base64PipelineGen(pxDriver);
113    base64FunctionType main = reinterpret_cast<base64FunctionType>(pxDriver.getPointerToMain());
114   
115    delete idb;
116    return main;
117}
118
119void base64(base64FunctionType fn_ptr, const std::string & fileName) {
120    std::string mFileName = fileName;
121    size_t mFileSize;
122    char * mFileBuffer;
123
124    const boost::filesystem::path file(mFileName);
125    if (exists(file)) {
126        if (is_directory(file)) {
127            return;
128        }
129    } else {
130        std::cerr << "Error: cannot open " << mFileName << " for processing. Skipped.\n";
131        return;
132    }
133   
134    mFileSize = file_size(file);
135    boost::iostreams::mapped_file_source mFile;
136    if (mFileSize == 0) {
137        mFileBuffer = nullptr;
138    }
139    else {
140        try {
141            mFile.open(mFileName);
142        } catch (std::exception &e) {
143            std::cerr << "Error: Boost mmap of " << mFileName << ": " << e.what() << std::endl;
144            return;
145        }
146        mFileBuffer = const_cast<char *>(mFile.data());
147    }
148
149    if (mMapBuffering) {
150        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2*mFileSize));
151        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
152        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
153        fn_ptr(mFileBuffer, static_cast<char*>(outputBuffer.get_address()), mFileSize);
154    }
155    else if (memAlignBuffering) {
156        char * outputBuffer;
157        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2*mFileSize)) {
158            throw std::bad_alloc();
159        }
160        fn_ptr(mFileBuffer, outputBuffer, mFileSize);
161        free(reinterpret_cast<void *>(outputBuffer));
162    }
163    else {
164        /* No external output buffer */
165        fn_ptr(mFileBuffer, nullptr, mFileSize);
166    }
167    mFile.close();
168   
169}
170
171
172int main(int argc, char *argv[]) {
173    AddParabixVersionPrinter();
174    cl::HideUnrelatedOptions(ArrayRef<const cl::OptionCategory *>{&base64Options, codegen::codegen_flags()});
175    cl::ParseCommandLineOptions(argc, argv);
176
177    base64FunctionType fn_ptr = base64CodeGen();
178
179    for (unsigned i = 0; i != inputFiles.size(); ++i) {
180        base64(fn_ptr, inputFiles[i]);
181    }
182
183    return 0;
184}
185
Note: See TracBrowser for help on using the repository browser.