source: icGREP/icgrep-devel/icgrep/base64.cpp @ 5418

Last change on this file since 5418 was 5418, checked in by nmedfort, 2 years ago

Removed non-functional CUDA code from icgrep and consolidated grep and multigrep mode into a single function; allowed segment parallel pipeline to utilize process as its initial thread; modified MMapSourceKernel to map and perform mmap directly and advise the OS to drop consumed data streams.

File size: 5.7 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <iostream>
8#include <llvm/IR/Function.h>
9#include <llvm/IR/Module.h>
10#include <llvm/ExecutionEngine/ExecutionEngine.h>
11#include <llvm/IR/Verifier.h>
12#include <llvm/Support/CommandLine.h>
13#include <kernels/toolchain.h>
14#include <IR_Gen/idisa_builder.h>
15#include <IR_Gen/idisa_target.h>
16#include <kernels/pipeline.h>
17#include <kernels/mmap_kernel.h>
18#include <kernels/streamset.h>
19#include <kernels/radix64.h>
20#include <kernels/stdout_kernel.h>
21#include <boost/interprocess/mapped_region.hpp>
22#include <boost/interprocess/anonymous_shared_memory.hpp>
23#include <sys/stat.h>
24#include <fcntl.h>
25
26using namespace llvm;
27
28static cl::OptionCategory base64Options("base64 Options",
29                                            "Transcoding control options.");
30
31static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(base64Options));
32
33static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(base64Options));
34static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(base64Options));
35
36
37using namespace kernel;
38using namespace parabix;
39
40void base64PipelineGen(ParabixDriver & pxDriver) {
41       
42    IDISA::IDISA_Builder * iBuilder = pxDriver.getIDISA_Builder();
43    Module * mod = iBuilder->getModule();
44    Type * mBitBlockType = iBuilder->getBitBlockType();
45
46    Type * const voidTy = Type::getVoidTy(mod->getContext());
47    Type * const int32Ty = iBuilder->getInt32Ty();
48    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
49   
50   
51    Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, int32Ty, outputType, nullptr));
52    main->setCallingConv(CallingConv::C);
53    Function::arg_iterator args = main->arg_begin();
54   
55    Value * const fileDescriptor = &*(args++);
56    fileDescriptor->setName("fileDescriptor");
57    Value * const outputStream = &*(args++);
58    outputStream->setName("outputStream");
59
60    //Round up to a multiple of 3.
61    const unsigned segmentSize = ((codegen::SegmentSize + 2)/3) * 3;
62   
63    const unsigned bufferSegments = codegen::BufferSegments;
64   
65    SourceFileBuffer ByteStream(iBuilder, iBuilder->getStreamSetTy(1, 8));
66
67    CircularBuffer Expanded3_4Out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
68    CircularBuffer Radix64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments);
69    CircularCopybackBuffer Base64out(iBuilder, iBuilder->getStreamSetTy(1, 8), segmentSize * 4/3 * bufferSegments, 1);
70   
71    MMapSourceKernel mmapK(iBuilder, segmentSize);
72    mmapK.setInitialArguments({fileDescriptor});
73    pxDriver.addKernelCall(mmapK, {}, {&ByteStream});
74   
75    expand3_4Kernel expandK(iBuilder);
76    pxDriver.addKernelCall(expandK, {&ByteStream}, {&Expanded3_4Out});
77
78    radix64Kernel radix64K(iBuilder);
79    pxDriver.addKernelCall(radix64K, {&Expanded3_4Out}, {&Radix64out});
80
81    base64Kernel base64K(iBuilder);
82    pxDriver.addKernelCall(base64K, {&Radix64out}, {&Base64out});
83   
84    StdOutKernel stdoutK(iBuilder, 8);
85    pxDriver.addKernelCall(stdoutK, {&Base64out}, {});
86   
87    iBuilder->SetInsertPoint(BasicBlock::Create(mod->getContext(), "entry", main,0));
88
89    ByteStream.allocateBuffer();
90    Expanded3_4Out.allocateBuffer();
91    Radix64out.allocateBuffer();
92    Base64out.allocateBuffer();
93
94    pxDriver.generatePipelineIR();
95
96    iBuilder->CreateRetVoid();
97
98    pxDriver.linkAndFinalize();
99}
100
101
102typedef void (*base64FunctionType)(const uint32_t fd, char * outputBuffer);
103
104base64FunctionType base64CodeGen(void) {
105    LLVMContext TheContext;                           
106    Module * M = new Module("base64", TheContext);
107    IDISA::IDISA_Builder * idb = IDISA::GetIDISA_Builder(M);
108    ParabixDriver pxDriver(idb);
109   
110    base64PipelineGen(pxDriver);
111    base64FunctionType main = reinterpret_cast<base64FunctionType>(pxDriver.getPointerToMain());
112   
113    delete idb;
114    return main;
115}
116
117size_t file_size(const int fd) {
118    struct stat st;
119    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
120        st.st_size = 0;
121    }
122    return st.st_size;
123}
124
125void base64(base64FunctionType fn_ptr, const std::string & fileName) {
126
127    const int fd = open(fileName.c_str(), O_RDONLY);
128    if (LLVM_UNLIKELY(fd == -1)) {
129        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
130        return;
131    }
132    if (mMapBuffering) {
133        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2 * file_size(fd)));
134        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
135        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
136        fn_ptr(fd, static_cast<char*>(outputBuffer.get_address()));
137    } else if (memAlignBuffering) {
138        char * outputBuffer;
139        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2 * file_size(fd))) {
140            throw std::bad_alloc();
141        }
142        fn_ptr(fd, outputBuffer);
143        free(reinterpret_cast<void *>(outputBuffer));
144    } else { /* No external output buffer */
145        fn_ptr(fd, nullptr);
146    }
147    close(fd);
148   
149}
150
151
152int main(int argc, char *argv[]) {
153    AddParabixVersionPrinter();
154    cl::HideUnrelatedOptions(ArrayRef<const cl::OptionCategory *>{&base64Options, codegen::codegen_flags()});
155    cl::ParseCommandLineOptions(argc, argv);
156
157    base64FunctionType fn_ptr = base64CodeGen();
158
159    for (unsigned i = 0; i != inputFiles.size(); ++i) {
160        base64(fn_ptr, inputFiles[i]);
161    }
162
163    return 0;
164}
165
Note: See TracBrowser for help on using the repository browser.