source: icGREP/icgrep-devel/icgrep/base64.cpp @ 6161

Last change on this file since 6161 was 6047, checked in by nmedfort, 12 months ago

Major refactoring of buffer types. Static buffers replace Circular and CircularCopyback?. External buffers unify Source/External?.

File size: 8.4 KB
Line 
1/*
2 *  Copyright (c) 2016 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <iostream>
8#include <llvm/IR/Function.h>
9#include <llvm/IR/Module.h>
10#include <llvm/ExecutionEngine/ExecutionEngine.h>
11#include <llvm/IR/Verifier.h>
12#include <llvm/Support/CommandLine.h>
13#include <toolchain/toolchain.h>
14#include <toolchain/cpudriver.h>
15#include <IR_Gen/idisa_target.h>
16#include <kernels/source_kernel.h>
17#include <kernels/streamset.h>
18#include <kernels/radix64.h>
19#include <kernels/stdout_kernel.h>
20#include <kernels/kernel_builder.h>
21#include <boost/interprocess/mapped_region.hpp>
22#include <boost/interprocess/anonymous_shared_memory.hpp>
23#include <boost/math/common_factor_rt.hpp>
24#include <sys/stat.h>
25#include <fcntl.h>
26#include <mutex>
27
28using namespace llvm;
29
30static cl::OptionCategory base64Options("base64 Options",
31                                            "Transcoding control options.");
32
33static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(base64Options));
34
35static cl::opt<bool> mMapBuffering("mmap-buffering", cl::desc("Enable mmap buffering."), cl::cat(base64Options));
36static cl::opt<bool> memAlignBuffering("memalign-buffering", cl::desc("Enable posix_memalign buffering."), cl::cat(base64Options));
37static cl::opt<int> Threads("threads", cl::desc("Total number of threads."), cl::init(1));
38
39
40using namespace kernel;
41using namespace parabix;
42
43void base64PipelineGen(ParabixDriver & pxDriver) {
44       
45    auto & iBuilder = pxDriver.getBuilder();
46    Module * mod = iBuilder->getModule();
47    Type * bitBlockType = iBuilder->getBitBlockType();
48
49    Type * const voidTy = iBuilder->getVoidTy();
50    Type * const int32Ty = iBuilder->getInt32Ty();
51    Type * const outputType = PointerType::get(ArrayType::get(ArrayType::get(bitBlockType, 8), 1), 0);
52   
53   
54    Function * const main = cast<Function>(mod->getOrInsertFunction("Main", voidTy, int32Ty, outputType, nullptr));
55    main->setCallingConv(CallingConv::C);
56    auto args = main->arg_begin();
57   
58    Value * const fileDescriptor = &*(args++);
59    fileDescriptor->setName("fileDescriptor");
60    Value * const outputStream = &*(args++);
61    outputStream->setName("outputStream");
62    iBuilder->SetInsertPoint(BasicBlock::Create(mod->getContext(), "entry", main,0));
63
64    //Round up to a multiple of 3.
65    const auto bufferSize = (codegen::SegmentSize * codegen::BufferSegments);
66    const auto expandedSize = boost::lcm(boost::lcm(bufferSize, 3U), 4U);
67
68    StreamSetBuffer * ByteStream = pxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8));
69    Kernel * mmapK = pxDriver.addKernelInstance<MMapSourceKernel>(iBuilder);
70    mmapK->setInitialArguments({fileDescriptor});
71    pxDriver.makeKernelCall(mmapK, {}, {ByteStream});
72   
73    StreamSetBuffer * Expanded3_4Out = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), expandedSize);
74    Kernel * expandK = pxDriver.addKernelInstance<expand3_4Kernel>(iBuilder);
75    pxDriver.makeKernelCall(expandK, {ByteStream}, {Expanded3_4Out});
76   
77    StreamSetBuffer * Radix64out = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferSize);
78    Kernel * radix64K = pxDriver.addKernelInstance<radix64Kernel>(iBuilder);
79    pxDriver.makeKernelCall(radix64K, {Expanded3_4Out}, {Radix64out});
80   
81    if (memAlignBuffering){
82        auto Base64out = pxDriver.addBuffer<ExternalBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), outputStream);
83        Kernel * base64K = pxDriver.addKernelInstance<base64Kernel>(iBuilder);
84        pxDriver.makeKernelCall(base64K, {Radix64out}, {Base64out});
85    } else {
86        StreamSetBuffer * Base64out = pxDriver.addBuffer<StaticBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8), bufferSize);
87        Kernel * base64K = pxDriver.addKernelInstance<base64Kernel>(iBuilder);
88        pxDriver.makeKernelCall(base64K, {Radix64out}, {Base64out});       
89        Kernel * outK = pxDriver.addKernelInstance<StdOutKernel>(iBuilder, 8);
90        pxDriver.makeKernelCall(outK, {Base64out}, {});
91    }
92   
93    pxDriver.generatePipelineIR();
94    pxDriver.deallocateBuffers();
95    iBuilder->CreateRetVoid();
96
97    pxDriver.finalizeObject();
98}
99
100typedef void (*base64FunctionType)(const uint32_t fd, char * outputBuffer);
101
102size_t file_size(const int fd) {
103    struct stat st;
104    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
105        st.st_size = 0;
106    }
107    return st.st_size;
108}
109
110void base64(base64FunctionType fn_ptr, const std::string & fileName) {
111
112    const int fd = open(fileName.c_str(), O_RDONLY);
113    if (LLVM_UNLIKELY(fd == -1)) {
114        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
115        return;
116    }
117    if (mMapBuffering) {
118        boost::interprocess::mapped_region outputBuffer(boost::interprocess::anonymous_shared_memory(2 * file_size(fd)));
119        outputBuffer.advise(boost::interprocess::mapped_region::advice_willneed);
120        outputBuffer.advise(boost::interprocess::mapped_region::advice_sequential);
121        fn_ptr(fd, static_cast<char*>(outputBuffer.get_address()));
122    } else if (memAlignBuffering) {
123        unsigned inputSize = file_size(fd);
124        unsigned paddingSize = (inputSize % 3) ? (4 - (inputSize % 3)) : 0;
125        unsigned outputSize = inputSize * 4/3 + paddingSize;
126
127        char * outputBuffer;
128        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, inputSize * 2)) {
129            throw std::bad_alloc();
130        }
131        fn_ptr(fd, outputBuffer);
132        fwrite(outputBuffer, outputSize, 1, stdout);
133        free(reinterpret_cast<void *>(outputBuffer));
134    } else { /* No external output buffer */
135        fn_ptr(fd, nullptr);
136    }
137    close(fd);
138   
139}
140
141std::mutex count_mutex;
142size_t fileCount;
143base64FunctionType fn_ptr;
144
145std::vector<char *> resultStrs;
146std::vector<int> filesizes;
147
148void *Base64ThreadFunction(void *args)
149{
150    size_t fileIdx;
151
152    count_mutex.lock();
153    fileIdx = fileCount;
154    fileCount++;
155    count_mutex.unlock();
156
157    while (fileIdx < inputFiles.size()) {
158        const int fd = open(inputFiles[fileIdx].c_str(), O_RDONLY);
159        if (LLVM_UNLIKELY(fd == -1)) {
160            std::cerr << "Error: cannot open " << inputFiles[fileIdx] << " for processing. Skipped.\n";
161            exit(-1);
162        }
163
164        char * outputBuffer;
165        if (posix_memalign(reinterpret_cast<void **>(&outputBuffer), 32, 2 * file_size(fd))) {
166            throw std::bad_alloc();
167        }
168   
169        fn_ptr(fd, outputBuffer);
170        resultStrs[fileIdx] = outputBuffer;
171        filesizes[fileIdx] = file_size(fd);
172
173        count_mutex.lock();
174        fileIdx = fileCount;
175        fileCount++;
176        count_mutex.unlock();
177    }
178
179    pthread_exit(nullptr);
180}
181
182int main(int argc, char *argv[]) {
183    codegen::ParseCommandLineOptions(argc, argv, {&base64Options, codegen::codegen_flags()});
184
185    if (Threads == 1) {
186        ParabixDriver pxDriver("base64");
187        base64PipelineGen(pxDriver);
188        fn_ptr = reinterpret_cast<base64FunctionType>(pxDriver.getMain());     
189        for (unsigned i = 0; i != inputFiles.size(); ++i) {
190            base64(fn_ptr, inputFiles[i]);
191        }
192    }
193    else{
194        memAlignBuffering = true;
195        ParabixDriver pxDriver("base64");
196        base64PipelineGen(pxDriver);
197        fn_ptr = reinterpret_cast<base64FunctionType>(pxDriver.getMain());
198
199        fileCount = 0;
200        const unsigned n = inputFiles.size();
201        resultStrs.resize(n);
202        filesizes.resize(n);
203
204        const unsigned numOfThreads = Threads;
205        pthread_t threads[numOfThreads];
206
207        for(unsigned long i = 0; i < numOfThreads; ++i){
208            const int rc = pthread_create(&threads[i], NULL, Base64ThreadFunction, (void *)i);
209            if (rc) {
210                llvm::report_fatal_error("Failed to create thread: code " + std::to_string(rc));
211            }
212        }
213
214        for(unsigned i = 0; i < numOfThreads; ++i) {
215            void * status = nullptr;
216            const int rc = pthread_join(threads[i], &status);
217            if (rc) {
218                llvm::report_fatal_error("Failed to join thread: code " + std::to_string(rc));
219            }
220        }
221
222        for (unsigned i=0; i<resultStrs.size(); i++){
223            unsigned paddingSize = (filesizes[i] % 3) ? (4 - (filesizes[i] % 3)) : 0;
224            fwrite(resultStrs[i], filesizes[i] * 4/3 + paddingSize, 1, stdout);
225        }
226    }   
227
228    return 0;
229}
230
Note: See TracBrowser for help on using the repository browser.