source: icGREP/icgrep-devel/icgrep/base64.cpp @ 5636

Last change on this file since 5636 was 5620, checked in by nmedfort, 2 years ago

Bug fixes for multigrep mode. Optional PabloKernel? branch hit counter added. Minor optimizations.

File size: 8.5 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <iostream>
8#include <llvm/IR/Function.h>
9#include <llvm/IR/Module.h>
10#include <llvm/ExecutionEngine/ExecutionEngine.h>
11#include <llvm/IR/Verifier.h>
12#include <llvm/Support/CommandLine.h>
13#include <toolchain/toolchain.h>
14#include <toolchain/cpudriver.h>
15#include <IR_Gen/idisa_target.h>
16#include <kernels/source_kernel.h>
17#include <kernels/streamset.h>
18#include <kernels/radix64.h>
19#include <kernels/stdout_kernel.h>
20#include <kernels/kernel_builder.h>
21#include <boost/interprocess/mapped_region.hpp>
22#include <boost/interprocess/anonymous_shared_memory.hpp>
23#include <sys/stat.h>
24#include <fcntl.h>
25#include <mutex>
26
27using namespace llvm;
28
29static cl::OptionCategory base64Options("base64 Options",
30                                            "Transcoding control options.");
31
32static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(base64Options));
33
34static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(base64Options));
35static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(base64Options));
36static cl::opt<int> Threads("threads", cl::desc("Total number of threads."), cl::init(1));
37
38
39using namespace kernel;
40using namespace parabix;
41
42void base64PipelineGen(ParabixDriver & pxDriver) {
43       
44    auto & iBuilder = pxDriver.getBuilder();
45    Module * mod = iBuilder->getModule();
46    Type * bitBlockType = iBuilder->getBitBlockType();
47
48    Type * const voidTy = iBuilder->getVoidTy();
49    Type * const int32Ty = iBuilder->getInt32Ty();
50    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(bitBlockType, 8), 1), 0);
51   
52   
53    Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, int32Ty, outputType, nullptr));
54    main->setCallingConv(CallingConv::C);
55    Function::arg_iterator args = main->arg_begin();
56   
57    Value * const fileDescriptor = &*(args++);
58    fileDescriptor->setName("fileDescriptor");
59    Value * const outputStream = &*(args++);
60    outputStream->setName("outputStream");
61    iBuilder->SetInsertPoint(BasicBlock::Create(mod->getContext(), "entry", main,0));
62
63    //Round up to a multiple of 3.
64    const unsigned initSegSize = ((codegen::SegmentSize + 2)/3) * 3;
65    const unsigned bufferSize = (4 * initSegSize * codegen::BufferSegments) / 3;
66
67    StreamSetBuffer * ByteStream = pxDriver.addBuffer(make_unique<SourceBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)));
68
69    Kernel * mmapK = pxDriver.addKernelInstance(make_unique<MMapSourceKernel>(iBuilder, initSegSize));
70    mmapK->setInitialArguments({fileDescriptor});
71    pxDriver.makeKernelCall(mmapK, {}, {ByteStream});
72   
73    StreamSetBuffer * Expanded3_4Out = pxDriver.addBuffer(make_unique<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferSize));
74    Kernel * expandK = pxDriver.addKernelInstance(make_unique<expand3_4Kernel>(iBuilder));
75    pxDriver.makeKernelCall(expandK, {ByteStream}, {Expanded3_4Out});
76   
77    StreamSetBuffer * Radix64out = pxDriver.addBuffer(make_unique<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferSize));
78    Kernel * radix64K = pxDriver.addKernelInstance(make_unique<radix64Kernel>(iBuilder));
79    pxDriver.makeKernelCall(radix64K, {Expanded3_4Out}, {Radix64out});
80   
81    if (memAlignBuffering){
82        auto Base64out = pxDriver.addExternalBuffer(make_unique<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), outputStream));
83        Kernel * base64K = pxDriver.addKernelInstance(make_unique<base64Kernel>(iBuilder));
84        pxDriver.makeKernelCall(base64K, {Radix64out}, {Base64out});
85    }
86    else {
87        StreamSetBuffer * Base64out = pxDriver.addBuffer(make_unique<DynamicBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferSize));
88        Kernel * base64K = pxDriver.addKernelInstance(make_unique<base64Kernel>(iBuilder));
89        pxDriver.makeKernelCall(base64K, {Radix64out}, {Base64out});
90       
91        Kernel * outK = pxDriver.addKernelInstance(make_unique<StdOutKernel>(iBuilder, 8));
92        pxDriver.makeKernelCall(outK, {Base64out}, {});
93    }
94   
95    pxDriver.generatePipelineIR();
96    pxDriver.deallocateBuffers();
97    iBuilder->CreateRetVoid();
98
99    pxDriver.finalizeObject();
100}
101
102typedef void (*base64FunctionType)(const uint32_t fd, char * outputBuffer);
103
104size_t file_size(const int fd) {
105    struct stat st;
106    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
107        st.st_size = 0;
108    }
109    return st.st_size;
110}
111
112void base64(base64FunctionType fn_ptr, const std::string & fileName) {
113
114    const int fd = open(fileName.c_str(), O_RDONLY);
115    if (LLVM_UNLIKELY(fd == -1)) {
116        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
117        return;
118    }
119    if (mMapBuffering) {
120        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2 * file_size(fd)));
121        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
122        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
123        fn_ptr(fd, static_cast<char*>(outputBuffer.get_address()));
124    } else if (memAlignBuffering) {
125        unsigned inputSize = file_size(fd);
126        unsigned paddingSize = (inputSize % 3) ? (4 - (inputSize % 3)) : 0;
127        unsigned outputSize = inputSize * 4/3 + paddingSize;
128
129        char * outputBuffer;
130        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, inputSize * 2)) {
131            throw std::bad_alloc();
132        }
133        fn_ptr(fd, outputBuffer);
134        fwrite(outputBuffer, outputSize, 1, stdout);
135        free(reinterpret_cast<void *>(outputBuffer));
136    } else { /* No external output buffer */
137        fn_ptr(fd, nullptr);
138    }
139    close(fd);
140   
141}
142
143std::mutex count_mutex;
144size_t fileCount;
145base64FunctionType fn_ptr;
146
147std::vector<char *> resultStrs;
148std::vector<int> filesizes;
149
150void *Base64ThreadFunction(void *args)
151{
152    size_t fileIdx;
153
154    count_mutex.lock();
155    fileIdx = fileCount;
156    fileCount++;
157    count_mutex.unlock();
158
159    while (fileIdx < inputFiles.size()) {
160        const int fd = open(inputFiles[fileIdx].c_str(), O_RDONLY);
161        if (LLVM_UNLIKELY(fd == -1)) {
162            std::cerr << "Error: cannot open " << inputFiles[fileIdx] << " for processing. Skipped.\n";
163            exit(-1);
164        }
165
166        char * outputBuffer;
167        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2 * file_size(fd))) {
168            throw std::bad_alloc();
169        }
170   
171        fn_ptr(fd, outputBuffer);
172        resultStrs[fileIdx] = outputBuffer;
173        filesizes[fileIdx] = file_size(fd);
174
175        count_mutex.lock();
176        fileIdx = fileCount;
177        fileCount++;
178        count_mutex.unlock();
179    }
180
181    pthread_exit(nullptr);
182}
183
184int main(int argc, char *argv[]) {
185    codegen::ParseCommandLineOptions(argc, argv, {&base64Options, codegen::codegen_flags()});
186
187    if (Threads == 1) {
188        ParabixDriver pxDriver("base64");
189        base64PipelineGen(pxDriver);
190        fn_ptr = reinterpret_cast<base64FunctionType>(pxDriver.getMain());     
191        for (unsigned i = 0; i != inputFiles.size(); ++i) {
192            base64(fn_ptr, inputFiles[i]);
193        }
194    }
195    else{
196        memAlignBuffering = true;
197        ParabixDriver pxDriver("base64");
198        base64PipelineGen(pxDriver);
199        fn_ptr = reinterpret_cast<base64FunctionType>(pxDriver.getMain());
200
201        fileCount = 0;
202        const unsigned n = inputFiles.size();
203        resultStrs.resize(n);
204        filesizes.resize(n);
205
206        const unsigned numOfThreads = Threads;
207        pthread_t threads[numOfThreads];
208
209        for(unsigned long i = 0; i < numOfThreads; ++i){
210            const int rc = pthread_create(&threads[i], NULL, Base64ThreadFunction, (void *)i);
211            if (rc) {
212                llvm::report_fatal_error("Failed to create thread: code " + std::to_string(rc));
213            }
214        }
215
216        for(unsigned i = 0; i < numOfThreads; ++i) {
217            void * status = nullptr;
218            const int rc = pthread_join(threads[i], &status);
219            if (rc) {
220                llvm::report_fatal_error("Failed to join thread: code " + std::to_string(rc));
221            }
222        }
223
224        for (unsigned i=0; i<resultStrs.size(); i++){
225            unsigned paddingSize = (filesizes[i] % 3) ? (4 - (filesizes[i] % 3)) : 0;
226            fwrite(resultStrs[i], filesizes[i] * 4/3 + paddingSize, 1, stdout);
227        }
228    }   
229
230    return 0;
231}
232
Note: See TracBrowser for help on using the repository browser.