source: icGREP/icgrep-devel/icgrep/base64.cpp @ 5733

Last change on this file since 5733 was 5620, checked in by nmedfort, 21 months ago

Bug fixes for multigrep mode. Optional PabloKernel? branch hit counter added. Minor optimizations.

File size: 8.5 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <iostream>
8#include <llvm/IR/Function.h>
9#include <llvm/IR/Module.h>
10#include <llvm/ExecutionEngine/ExecutionEngine.h>
11#include <llvm/IR/Verifier.h>
12#include <llvm/Support/CommandLine.h>
13#include <toolchain/toolchain.h>
14#include <toolchain/cpudriver.h>
15#include <IR_Gen/idisa_target.h>
16#include <kernels/source_kernel.h>
17#include <kernels/streamset.h>
18#include <kernels/radix64.h>
19#include <kernels/stdout_kernel.h>
20#include <kernels/kernel_builder.h>
21#include <boost/interprocess/mapped_region.hpp>
22#include <boost/interprocess/anonymous_shared_memory.hpp>
23#include <sys/stat.h>
24#include <fcntl.h>
25#include <mutex>
26
27using namespace llvm;
28
29static cl::OptionCategory base64Options("base64 Options",
30                                            "Transcoding control options.");
31
32static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(base64Options));
33
34static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(base64Options));
35static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(base64Options));
36static cl::opt<int> Threads("threads", cl::desc("Total number of threads."), cl::init(1));
37
38
39using namespace kernel;
40using namespace parabix;
41
42void base64PipelineGen(ParabixDriver & pxDriver) {
43       
44    auto & iBuilder = pxDriver.getBuilder();
45    Module * mod = iBuilder->getModule();
46    Type * bitBlockType = iBuilder->getBitBlockType();
47
48    Type * const voidTy = iBuilder->getVoidTy();
49    Type * const int32Ty = iBuilder->getInt32Ty();
50    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(bitBlockType, 8), 1), 0);
51   
52   
53    Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, int32Ty, outputType, nullptr));
54    main->setCallingConv(CallingConv::C);
55    Function::arg_iterator args = main->arg_begin();
56   
57    Value * const fileDescriptor = &*(args++);
58    fileDescriptor->setName("fileDescriptor");
59    Value * const outputStream = &*(args++);
60    outputStream->setName("outputStream");
61    iBuilder->SetInsertPoint(BasicBlock::Create(mod->getContext(), "entry", main,0));
62
63    //Round up to a multiple of 3.
64    const unsigned initSegSize = ((codegen::SegmentSize + 2)/3) * 3;
65    const unsigned bufferSize = (4 * initSegSize * codegen::BufferSegments) / 3;
66
67    StreamSetBuffer * ByteStream = pxDriver.addBuffer(make_unique<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)));
68
69    Kernel * mmapK = pxDriver.addKernelInstance(make_unique<MMapSourceKernel>(iBuilder, initSegSize));
70    mmapK->setInitialArguments({fileDescriptor});
71    pxDriver.makeKernelCall(mmapK, {}, {ByteStream});
72   
73    StreamSetBuffer * Expanded3_4Out = pxDriver.addBuffer(make_unique<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferSize));
74    Kernel * expandK = pxDriver.addKernelInstance(make_unique<expand3_4Kernel>(iBuilder));
75    pxDriver.makeKernelCall(expandK, {ByteStream}, {Expanded3_4Out});
76   
77    StreamSetBuffer * Radix64out = pxDriver.addBuffer(make_unique<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferSize));
78    Kernel * radix64K = pxDriver.addKernelInstance(make_unique<radix64Kernel>(iBuilder));
79    pxDriver.makeKernelCall(radix64K, {Expanded3_4Out}, {Radix64out});
80   
81    if (memAlignBuffering){
82        auto Base64out = pxDriver.addExternalBuffer(make_unique<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), outputStream));
83        Kernel * base64K = pxDriver.addKernelInstance(make_unique<base64Kernel>(iBuilder));
84        pxDriver.makeKernelCall(base64K, {Radix64out}, {Base64out});
85    }
86    else {
87        StreamSetBuffer * Base64out = pxDriver.addBuffer(make_unique<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferSize));
88        Kernel * base64K = pxDriver.addKernelInstance(make_unique<base64Kernel>(iBuilder));
89        pxDriver.makeKernelCall(base64K, {Radix64out}, {Base64out});
90       
91        Kernel * outK = pxDriver.addKernelInstance(make_unique<StdOutKernel>(iBuilder, 8));
92        pxDriver.makeKernelCall(outK, {Base64out}, {});
93    }
94   
95    pxDriver.generatePipelineIR();
96    pxDriver.deallocateBuffers();
97    iBuilder->CreateRetVoid();
98
99    pxDriver.finalizeObject();
100}
101
102typedef void (*base64FunctionType)(const uint32_t fd, char * outputBuffer);
103
104size_t file_size(const int fd) {
105    struct stat st;
106    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
107        st.st_size = 0;
108    }
109    return st.st_size;
110}
111
112void base64(base64FunctionType fn_ptr, const std::string & fileName) {
113
114    const int fd = open(fileName.c_str(), O_RDONLY);
115    if (LLVM_UNLIKELY(fd == -1)) {
116        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
117        return;
118    }
119    if (mMapBuffering) {
120        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2 * file_size(fd)));
121        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
122        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
123        fn_ptr(fd, static_cast<char*>(outputBuffer.get_address()));
124    } else if (memAlignBuffering) {
125        unsigned inputSize = file_size(fd);
126        unsigned paddingSize = (inputSize % 3) ? (4 - (inputSize % 3)) : 0;
127        unsigned outputSize = inputSize * 4/3 + paddingSize;
128
129        char * outputBuffer;
130        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, inputSize * 2)) {
131            throw std::bad_alloc();
132        }
133        fn_ptr(fd, outputBuffer);
134        fwrite(outputBuffer, outputSize, 1, stdout);
135        free(reinterpret_cast<void *>(outputBuffer));
136    } else { /* No external output buffer */
137        fn_ptr(fd, nullptr);
138    }
139    close(fd);
140   
141}
142
143std::mutex count_mutex;
144size_t fileCount;
145base64FunctionType fn_ptr;
146
147std::vector<char *> resultStrs;
148std::vector<int> filesizes;
149
150void *Base64ThreadFunction(void *args)
151{
152    size_t fileIdx;
153
154    count_mutex.lock();
155    fileIdx = fileCount;
156    fileCount++;
157    count_mutex.unlock();
158
159    while (fileIdx < inputFiles.size()) {
160        const int fd = open(inputFiles[fileIdx].c_str(), O_RDONLY);
161        if (LLVM_UNLIKELY(fd == -1)) {
162            std::cerr << "Error: cannot open " << inputFiles[fileIdx] << " for processing. Skipped.\n";
163            exit(-1);
164        }
165
166        char * outputBuffer;
167        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2 * file_size(fd))) {
168            throw std::bad_alloc();
169        }
170   
171        fn_ptr(fd, outputBuffer);
172        resultStrs[fileIdx] = outputBuffer;
173        filesizes[fileIdx] = file_size(fd);
174
175        count_mutex.lock();
176        fileIdx = fileCount;
177        fileCount++;
178        count_mutex.unlock();
179    }
180
181    pthread_exit(nullptr);
182}
183
184int main(int argc, char *argv[]) {
185    codegen::ParseCommandLineOptions(argc, argv, {&base64Options, codegen::codegen_flags()});
186
187    if (Threads == 1) {
188        ParabixDriver pxDriver("base64");
189        base64PipelineGen(pxDriver);
190        fn_ptr = reinterpret_cast<base64FunctionType>(pxDriver.getMain());     
191        for (unsigned i = 0; i != inputFiles.size(); ++i) {
192            base64(fn_ptr, inputFiles[i]);
193        }
194    }
195    else{
196        memAlignBuffering = true;
197        ParabixDriver pxDriver("base64");
198        base64PipelineGen(pxDriver);
199        fn_ptr = reinterpret_cast<base64FunctionType>(pxDriver.getMain());
200
201        fileCount = 0;
202        const unsigned n = inputFiles.size();
203        resultStrs.resize(n);
204        filesizes.resize(n);
205
206        const unsigned numOfThreads = Threads;
207        pthread_t threads[numOfThreads];
208
209        for(unsigned long i = 0; i < numOfThreads; ++i){
210            const int rc = pthread_create(&threads[i], NULL, Base64ThreadFunction, (void *)i);
211            if (rc) {
212                llvm::report_fatal_error("Failed to create thread: code " + std::to_string(rc));
213            }
214        }
215
216        for(unsigned i = 0; i < numOfThreads; ++i) {
217            void * status = nullptr;
218            const int rc = pthread_join(threads[i], &status);
219            if (rc) {
220                llvm::report_fatal_error("Failed to join thread: code " + std::to_string(rc));
221            }
222        }
223
224        for (unsigned i=0; i<resultStrs.size(); i++){
225            unsigned paddingSize = (filesizes[i] % 3) ? (4 - (filesizes[i] % 3)) : 0;
226            fwrite(resultStrs[i], filesizes[i] * 4/3 + paddingSize, 1, stdout);
227        }
228    }   
229
230    return 0;
231}
232
Note: See TracBrowser for help on using the repository browser.