source: icGREP/icgrep-devel/icgrep/base64.cpp @ 5401

Last change on this file since 5401 was 5401, checked in by nmedfort, 2 years ago

Updated all projects to use ParabixDriver?. Deprecated original pipeline generation methods. Enabled LLVM optimizations, IR and ASM printing for Kernel modules. Enabled object cache by default. Begun work on moving consumed position information back to producing kernels.

File size: 6.4 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <iostream>
8#include <llvm/IR/Function.h>
9#include <llvm/IR/Module.h>
10#include <llvm/ExecutionEngine/ExecutionEngine.h>
11#include <llvm/IR/Verifier.h>
12#include <llvm/Support/CommandLine.h>
13#include <toolchain.h>
14#include <IR_Gen/idisa_builder.h>
15#include <IR_Gen/idisa_target.h>
16#include <kernels/pipeline.h>
17#include <kernels/mmap_kernel.h>
18#include <kernels/streamset.h>
19#include <kernels/radix64.h>
20#include <kernels/stdout_kernel.h>
21#include <boost/filesystem.hpp>
22#include <boost/iostreams/device/mapped_file.hpp>
23#include <boost/interprocess/anonymous_shared_memory.hpp>
24
25using namespace llvm;
26
27static cl::OptionCategory base64Options("base64 Options",
28                                            "Transcoding control options.");
29
30static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(base64Options));
31
32static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(base64Options));
33static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(base64Options));
34
35
36using namespace kernel;
37using namespace parabix;
38
39void base64PipelineGen(ParabixDriver & pxDriver) {
40       
41    IDISA::IDISA_Builder * iBuilder = pxDriver.getIDISA_Builder();
42    Module * mod = iBuilder->getModule();
43    Type * mBitBlockType = iBuilder->getBitBlockType();
44
45    Type * const size_ty = iBuilder->getSizeTy();
46    Type * const voidTy = Type::getVoidTy(mod->getContext());
47    Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
48    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
49   
50   
51    Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, inputType, outputType, size_ty, nullptr));
52    main->setCallingConv(CallingConv::C);
53    Function::arg_iterator args = main->arg_begin();
54   
55    Value * const inputStream = &*(args++);
56    inputStream->setName("inputStream");
57    Value * const outputStream = &*(args++);
58    outputStream->setName("outputStream");
59    Value * const fileSize = &*(args++);
60    fileSize->setName("fileSize");
61
62    //Round up to a multiple of 3.
63    const unsigned segmentSize = ((codegen::SegmentSize + 2)/3) * 3;
64   
65    const unsigned bufferSegments = codegen::BufferSegments;
66   
67    ExternalFileBuffer ByteStream(iBuilder, iBuilder->getStreamSetTy(1, 8));
68
69    CircularBuffer Expanded3_4Out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
70    CircularBuffer Radix64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
71    CircularCopybackBuffer Base64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments, 1);
72   
73    MMapSourceKernel mmapK(iBuilder, segmentSize);
74    mmapK.setInitialArguments({fileSize});
75    pxDriver.addKernelCall(mmapK, {}, {&ByteStream});
76   
77    expand3_4Kernel expandK(iBuilder);
78    pxDriver.addKernelCall(expandK, {&ByteStream}, {&Expanded3_4Out});
79
80    radix64Kernel radix64K(iBuilder);
81    pxDriver.addKernelCall(radix64K, {&Expanded3_4Out}, {&Radix64out});
82
83    base64Kernel base64K(iBuilder);
84    pxDriver.addKernelCall(base64K, {&Radix64out}, {&Base64out});
85   
86    StdOutKernel stdoutK(iBuilder, 8);
87    pxDriver.addKernelCall(stdoutK, {&Base64out}, {});
88   
89    iBuilder->SetInsertPoint(BasicBlock::Create(mod->getContext(), "entry", main,0));
90
91    ByteStream.setStreamSetBuffer(inputStream);
92    Expanded3_4Out.allocateBuffer();
93    Radix64out.allocateBuffer();
94    Base64out.allocateBuffer();
95
96    pxDriver.generatePipelineIR();
97
98    iBuilder->CreateRetVoid();
99
100    pxDriver.linkAndFinalize();
101}
102
103
104typedef void (*base64FunctionType)(char * byte_data, char * output_data, size_t filesize);
105
106base64FunctionType base64CodeGen(void) {
107    LLVMContext TheContext;                           
108    Module * M = new Module("base64", TheContext);
109    IDISA::IDISA_Builder * idb = IDISA::GetIDISA_Builder(M);
110    ParabixDriver pxDriver(idb);
111   
112    base64PipelineGen(pxDriver);
113    base64FunctionType main = reinterpret_cast<base64FunctionType>(pxDriver.getPointerToMain());
114   
115    delete idb;
116    return main;
117}
118
119void base64(base64FunctionType fn_ptr, const std::string & fileName) {
120    std::string mFileName = fileName;
121    size_t mFileSize;
122    char * mFileBuffer;
123
124    const boost::filesystem::path file(mFileName);
125    if (exists(file)) {
126        if (is_directory(file)) {
127            return;
128        }
129    } else {
130        std::cerr << "Error: cannot open " << mFileName << " for processing. Skipped.\n";
131        return;
132    }
133   
134    mFileSize = file_size(file);
135    boost::iostreams::mapped_file_source mFile;
136    if (mFileSize == 0) {
137        mFileBuffer = nullptr;
138    }
139    else {
140        try {
141            mFile.open(mFileName);
142        } catch (std::exception &e) {
143            std::cerr << "Error: Boost mmap of " << mFileName << ": " << e.what() << std::endl;
144            return;
145        }
146        mFileBuffer = const_cast<char *>(mFile.data());
147    }
148
149    if (mMapBuffering) {
150        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2*mFileSize));
151        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
152        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
153        fn_ptr(mFileBuffer, static_cast<char*>(outputBuffer.get_address()), mFileSize);
154    }
155    else if (memAlignBuffering) {
156        char * outputBuffer;
157        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2*mFileSize)) {
158            throw std::bad_alloc();
159        }
160        fn_ptr(mFileBuffer, outputBuffer, mFileSize);
161        free(reinterpret_cast<void *>(outputBuffer));
162    }
163    else {
164        /* No external output buffer */
165        fn_ptr(mFileBuffer, nullptr, mFileSize);
166    }
167    mFile.close();
168   
169}
170
171
172int main(int argc, char *argv[]) {
173    AddParabixVersionPrinter();
174    cl::HideUnrelatedOptions(ArrayRef<const cl::OptionCategory *>{&base64Options, codegen::codegen_flags()});
175    cl::ParseCommandLineOptions(argc, argv);
176
177    base64FunctionType fn_ptr = base64CodeGen();
178
179    for (unsigned i = 0; i != inputFiles.size(); ++i) {
180        base64(fn_ptr, inputFiles[i]);
181    }
182
183    return 0;
184}
185
Note: See TracBrowser for help on using the repository browser.