source: icGREP/icgrep-devel/icgrep/wc.cpp @ 5026

Last change on this file since 5026 was 5026, checked in by cameron, 3 years ago

Updates for LLVM 3.8; hide unrelated command line options

File size: 17.4 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <string>
8#include <iostream>
9#include <iomanip>
10#include <fstream>
11#include <sstream>
12
13#include <llvm/IR/Function.h>
14#include <llvm/IR/Module.h>
15#include <llvm/ExecutionEngine/ExecutionEngine.h>
16#include <llvm/ExecutionEngine/MCJIT.h>
17#include <llvm/IRReader/IRReader.h>
18#include <llvm/IR/Verifier.h>
19#include <llvm/Support/Debug.h>
20
21#include <llvm/Support/CommandLine.h>
22#include <llvm/CodeGen/CommandFlags.h>
23#include <llvm/Support/SourceMgr.h>
24#include <llvm/Support/TargetSelect.h>
25#include <llvm/Support/Host.h>
26#include <llvm/Support/raw_ostream.h>
27
28#include <utf_encoding.h>
29#include <re/re_cc.h>
30#include <cc/cc_compiler.h>
31#include <pablo/function.h>
32#include <IDISA/idisa_builder.h>
33#include <IDISA/idisa_target.h>
34#include <kernels/instance.h>
35#include <kernels/kernel.h>
36#include <kernels/s2p_kernel.h>
37
38#include <pablo/pablo_compiler.h>
39#include <pablo/pablo_toolchain.h>
40
41// Dynamic processor detection
42#define ISPC_LLVM_VERSION ISPC_LLVM_3_6
43#include <util/ispc.cpp>
44
45#include <utf_encoding.h>
46
47// mmap system
48#include <boost/filesystem.hpp>
49#include <boost/iostreams/device/mapped_file.hpp>
50using namespace boost::iostreams;
51using namespace boost::filesystem;
52
53#include <fcntl.h>
54static cl::OptionCategory wcFlags("Command Flags", "wc options");
55
56static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore, cl::cat(wcFlags));
57
58static cl::opt<bool> CountLines("l", cl::desc("Report the number of lines in each input file."), cl::init(false), cl::cat(wcFlags));
59static cl::opt<bool> CountWords("w", cl::desc("Report the number of lines in each input file."), cl::init(false), cl::cat(wcFlags));
60static cl::opt<bool> CountBytes("c", cl::desc("Report the number of bytes in each input file."), cl::init(false), cl::cat(wcFlags));
61static cl::opt<bool> CountChars("m", cl::desc("Report the number of characters in each input file."), cl::init(false), cl::cat(wcFlags));
62
63
64static cl::OptionCategory eIRDumpOptions("LLVM IR Dump Options", "These options control dumping of LLVM IR.");
65static cl::opt<bool> DumpGeneratedIR("dump-generated-IR", cl::init(false), cl::desc("Print LLVM IR generated by Pablo Compiler."), cl::cat(eIRDumpOptions));
66
67static cl::OptionCategory cMachineCodeOptimization("Machine Code Optimizations", "These options control back-end compilier optimization levels.");
68
69static cl::opt<char> OptLevel("O", cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] (default = '-O0')"),
70                              cl::cat(cMachineCodeOptimization), cl::Prefix, cl::ZeroOrMore, cl::init('0'));
71
72static cl::opt<unsigned> SegmentSize("segment-size", cl::desc("Segment Size"), cl::value_desc("positive integer"), cl::init(1));
73
74
75static int defaultFieldWidth = 7;  // default field width
76
77std::vector<uint64_t> lineCount;
78std::vector<uint64_t> wordCount;
79std::vector<uint64_t> charCount;
80std::vector<uint64_t> byteCount;
81
82uint64_t TotalLines = 0;
83uint64_t TotalWords = 0;
84uint64_t TotalChars = 0;
85uint64_t TotalBytes = 0;
86
87//
88//
89extern "C" {
90    void report_counts(uint64_t lines, uint64_t words, uint64_t chars, uint64_t bytes, uint64_t fileIdx) {
91        lineCount[fileIdx] = lines;
92        wordCount[fileIdx] = words;
93        charCount[fileIdx] = chars;
94        byteCount[fileIdx] = bytes;
95        TotalLines += lines;
96        TotalWords += words;
97        TotalChars += chars;
98        TotalBytes += bytes;
99    }
100}
101
102
103
104//
105//
106
107ExecutionEngine * wcJIT_to_ExecutionEngine (Module * m) {
108
109    InitializeNativeTarget();
110    InitializeNativeTargetAsmPrinter();
111    InitializeNativeTargetAsmParser();
112
113    PassRegistry * Registry = PassRegistry::getPassRegistry();
114    initializeCore(*Registry);
115    initializeCodeGen(*Registry);
116    initializeLowerIntrinsicsPass(*Registry);
117
118    std::string errMessage;
119    EngineBuilder builder(std::move(std::unique_ptr<Module>(m)));
120    builder.setErrorStr(&errMessage);
121    builder.setMCPU(sys::getHostCPUName());
122    CodeGenOpt::Level optLevel = CodeGenOpt::Level::None;
123    switch (OptLevel) {
124        case '0': optLevel = CodeGenOpt::None; break;
125        case '1': optLevel = CodeGenOpt::Less; break;
126        case '2': optLevel = CodeGenOpt::Default; break;
127        case '3': optLevel = CodeGenOpt::Aggressive; break;
128        default: errs() << OptLevel << " is an invalid optimization level.\n";
129    }
130    builder.setOptLevel(optLevel);
131
132    if ((strncmp(lGetSystemISA(), "avx2", 4) == 0)) {
133            std::vector<std::string> attrs;
134            attrs.push_back("avx2");
135            builder.setMAttrs(attrs);
136    }
137
138    // builder.selectTarget();
139
140    //builder.setOptLevel(mMaxWhileDepth ? CodeGenOpt::Level::Less : CodeGenOpt::Level::None);
141    ExecutionEngine * engine = builder.create();
142    if (engine == nullptr) {
143        throw std::runtime_error("Could not create ExecutionEngine: " + errMessage);
144    }
145    return engine;
146}
147
148
149pablo::PabloFunction * wc_gen(Encoding encoding) {
150    //  input: 8 basis bit streams
151    //  output: 3 count streams
152   
153    pablo::PabloFunction * function = pablo::PabloFunction::Create("wc", 8, 3);
154    cc::CC_Compiler ccc(*function, encoding);
155   
156    pablo::PabloBuilder pBuilder(ccc.getBuilder().getPabloBlock(), ccc.getBuilder());
157    const std::vector<pablo::Var *> u8_bits = ccc.getBasisBits();
158
159    if (CountLines) {
160        pablo::PabloAST * LF = ccc.compileCC(re::makeCC(0x0A));
161        function->setResult(0, pBuilder.createAssign("lineCount", pBuilder.createCount(LF)));
162    }
163    else function->setResult(0, pBuilder.createAssign("lineCount", pBuilder.createZeroes()));
164    // FIXME - we need to limit this to pablo.inFile() because null bytes past EOF are matched by wordChar
165    if (CountWords) {
166        pablo::PabloAST * WS = ccc.compileCC(re::makeCC(re::makeCC(0x09, 0x0D), re::makeCC(0x20)));
167       
168        pablo::PabloAST * wordChar = ccc.compileCC(re::makeCC(re::makeCC(re::makeCC(0x00, 0x08), re::makeCC(0xE, 0x1F)), re::makeCC(0x21, 0xFF)));
169        // WS_follow_or_start = 1 past WS or at start of file
170        pablo::PabloAST * WS_follow_or_start = pBuilder.createNot(pBuilder.createAdvance(pBuilder.createNot(WS), 1));
171        //
172        pablo::PabloAST * wordStart = pBuilder.createAnd(wordChar, WS_follow_or_start);
173        function->setResult(1, pBuilder.createAssign("wordCount", pBuilder.createCount(wordStart)));
174    }
175    else function->setResult(1, pBuilder.createAssign("wordCount", pBuilder.createZeroes()));
176    if (CountChars) {
177        //
178        // FIXME: This correctly counts characters assuming valid UTF-8 input.  But what if input is
179        // not UTF-8, or is not valid?
180        //
181        pablo::PabloAST * u8Begin = ccc.compileCC(re::makeCC(re::makeCC(0, 0x7F), re::makeCC(0xC2, 0xF4)));
182        function->setResult(2, pBuilder.createAssign("charCount", pBuilder.createCount(u8Begin)));
183    }
184    else function->setResult(2, pBuilder.createAssign("charCount", pBuilder.createZeroes()));
185    return function;
186}
187
188using namespace kernel;
189
190
191class wcPipelineBuilder {
192public:
193    wcPipelineBuilder(llvm::Module * m, IDISA::IDISA_Builder * b);
194   
195    ~wcPipelineBuilder();
196   
197    void CreateKernels(pablo::PabloFunction * function);
198    llvm::Function * ExecuteKernels();
199   
200private:
201    llvm::Module *                      mMod;
202    IDISA::IDISA_Builder *              iBuilder;
203    KernelBuilder *                     mS2PKernel;
204    KernelBuilder *                     mWC_Kernel;
205    llvm::Type *                        mBitBlockType;
206    int                                 mBlockSize;
207};
208
209
210using namespace pablo;
211using namespace kernel;
212
213wcPipelineBuilder::wcPipelineBuilder(Module * m, IDISA::IDISA_Builder * b)
214: mMod(m)
215, iBuilder(b)
216, mBitBlockType(b->getBitBlockType())
217, mBlockSize(b->getBitBlockWidth()){
218   
219}
220
221wcPipelineBuilder::~wcPipelineBuilder(){
222    delete mS2PKernel;
223    delete mWC_Kernel;
224}
225
226void wcPipelineBuilder::CreateKernels(PabloFunction * function){
227    mS2PKernel = new KernelBuilder(iBuilder, "s2p", SegmentSize);
228    mWC_Kernel = new KernelBuilder(iBuilder, "wc", SegmentSize);
229   
230    generateS2PKernel(mMod, iBuilder, mS2PKernel);
231   
232    pablo_function_passes(function);
233   
234    PabloCompiler pablo_compiler(mMod, iBuilder);
235    try {
236        pablo_compiler.setKernel(mWC_Kernel);
237        pablo_compiler.compile(function);
238        delete function;
239        releaseSlabAllocatorMemory();
240    } catch (std::runtime_error e) {
241        delete function;
242        releaseSlabAllocatorMemory();
243        std::cerr << "Runtime error: " << e.what() << std::endl;
244        exit(1);
245    }
246   
247}
248
249
250
251
252Function * wcPipelineBuilder::ExecuteKernels() {
253    Constant * report_counts_routine;
254    Type * const int64ty = iBuilder->getInt64Ty();
255    Type * const voidTy = Type::getVoidTy(mMod->getContext());
256    report_counts_routine = mMod->getOrInsertFunction("report_counts", voidTy, int64ty, int64ty, int64ty, int64ty, int64ty, nullptr);
257    Type * const inputType = PointerType::get(ArrayType::get(StructType::get(mMod->getContext(), std::vector<Type *>({ArrayType::get(mBitBlockType, 8)})), 1), 0);
258   
259    Function * const main = cast<Function>(mMod->getOrInsertFunction("Main", voidTy, inputType, int64ty, int64ty, nullptr));
260    main->setCallingConv(CallingConv::C);
261    Function::arg_iterator args = main->arg_begin();
262   
263    Value * const inputStream = &*(args++);
264    inputStream->setName("input");
265    Value * const bufferSize = &*(args++);
266    bufferSize->setName("bufferSize");
267    Value * const fileIdx = &*(args++);
268    fileIdx->setName("fileIdx");
269   
270    iBuilder->SetInsertPoint(BasicBlock::Create(mMod->getContext(), "entry", main,0));
271   
272    BasicBlock * entryBlock = iBuilder->GetInsertBlock();
273
274    BasicBlock * segmentCondBlock = nullptr;
275    BasicBlock * segmentBodyBlock = nullptr;
276    const unsigned segmentSize = SegmentSize;
277    if (segmentSize > 1) {
278        segmentCondBlock = BasicBlock::Create(mMod->getContext(), "segmentCond", main, 0);
279        segmentBodyBlock = BasicBlock::Create(mMod->getContext(), "segmentBody", main, 0);
280    }
281    BasicBlock * fullCondBlock = BasicBlock::Create(mMod->getContext(), "fullCond", main, 0);
282    BasicBlock * fullBodyBlock = BasicBlock::Create(mMod->getContext(), "fullBody", main, 0);
283    BasicBlock * finalBlock = BasicBlock::Create(mMod->getContext(), "final", main, 0);
284    BasicBlock * finalPartialBlock = BasicBlock::Create(mMod->getContext(), "partial", main, 0);
285    BasicBlock * finalEmptyBlock = BasicBlock::Create(mMod->getContext(), "empty", main, 0);
286    BasicBlock * endBlock = BasicBlock::Create(mMod->getContext(), "end", main, 0);
287
288    Instance * s2pInstance = mS2PKernel->instantiate(inputStream);
289    Instance * wcInstance = mWC_Kernel->instantiate(s2pInstance->getOutputStreamBuffer());
290
291    Value * initialBufferSize = nullptr;
292    BasicBlock * initialBlock = nullptr;
293   
294    if (segmentSize > 1) {
295        iBuilder->CreateBr(segmentCondBlock);
296        iBuilder->SetInsertPoint(segmentCondBlock);
297        PHINode * remainingBytes = iBuilder->CreatePHI(int64ty, 2, "remainingBytes");
298        remainingBytes->addIncoming(bufferSize, entryBlock);
299        Constant * const step = ConstantInt::get(int64ty, mBlockSize * segmentSize);
300        Value * segmentCondTest = iBuilder->CreateICmpULT(remainingBytes, step);
301        iBuilder->CreateCondBr(segmentCondTest, fullCondBlock, segmentBodyBlock);
302        iBuilder->SetInsertPoint(segmentBodyBlock);
303        for (unsigned i = 0; i < segmentSize; ++i) {
304            s2pInstance->CreateDoBlockCall();
305        }
306        for (unsigned i = 0; i < segmentSize; ++i) {
307            wcInstance->CreateDoBlockCall();
308        }
309        remainingBytes->addIncoming(iBuilder->CreateSub(remainingBytes, step), segmentBodyBlock);
310        iBuilder->CreateBr(segmentCondBlock);
311        initialBufferSize = remainingBytes;
312        initialBlock = segmentCondBlock;
313    } else {
314        initialBufferSize = bufferSize;
315        initialBlock = entryBlock;
316        iBuilder->CreateBr(fullCondBlock);
317    }
318
319    iBuilder->SetInsertPoint(fullCondBlock);
320    PHINode * remainingBytes = iBuilder->CreatePHI(int64ty, 2, "remainingBytes");
321    remainingBytes->addIncoming(initialBufferSize, initialBlock);
322
323    Constant * const step = ConstantInt::get(int64ty, mBlockSize);
324    Value * fullCondTest = iBuilder->CreateICmpULT(remainingBytes, step);
325    iBuilder->CreateCondBr(fullCondTest, finalBlock, fullBodyBlock);
326   
327    iBuilder->SetInsertPoint(fullBodyBlock);
328
329    s2pInstance->CreateDoBlockCall();
330    wcInstance->CreateDoBlockCall();
331
332    Value * diff = iBuilder->CreateSub(remainingBytes, step);
333
334    remainingBytes->addIncoming(diff, fullBodyBlock);
335    iBuilder->CreateBr(fullCondBlock);
336   
337    iBuilder->SetInsertPoint(finalBlock);
338    Value * emptyBlockCond = iBuilder->CreateICmpEQ(remainingBytes, ConstantInt::get(int64ty, 0));
339    iBuilder->CreateCondBr(emptyBlockCond, finalEmptyBlock, finalPartialBlock);
340   
341   
342    iBuilder->SetInsertPoint(finalPartialBlock);
343    s2pInstance->CreateDoBlockCall();
344    iBuilder->CreateBr(endBlock);
345   
346    iBuilder->SetInsertPoint(finalEmptyBlock);
347    s2pInstance->clearOutputStreamSet();
348    iBuilder->CreateBr(endBlock);
349   
350    iBuilder->SetInsertPoint(endBlock);
351
352    wcInstance->CreateDoBlockCall();
353   
354    Value * lineCount = iBuilder->CreateExtractElement(iBuilder->CreateBlockAlignedLoad(wcInstance->getOutputStream((int) 0)), iBuilder->getInt32(0));
355    Value * wordCount = iBuilder->CreateExtractElement(iBuilder->CreateBlockAlignedLoad(wcInstance->getOutputStream(1)), iBuilder->getInt32(0));
356    Value * charCount = iBuilder->CreateExtractElement(iBuilder->CreateBlockAlignedLoad(wcInstance->getOutputStream(2)), iBuilder->getInt32(0));
357   
358    iBuilder->CreateCall(report_counts_routine, std::vector<Value *>({lineCount, wordCount, charCount, bufferSize, fileIdx}));
359   
360    iBuilder->CreateRetVoid();
361   
362    return main;
363}
364
365
366typedef void (*wcFunctionType)(char * byte_data, size_t filesize, size_t fileIdx);
367
368static ExecutionEngine * wcEngine = nullptr;
369
370wcFunctionType wcCodeGen(void) {
371                           
372    Module * M = new Module("wc", getGlobalContext());
373   
374    IDISA::IDISA_Builder * idb = GetIDISA_Builder(M);
375
376    wcPipelineBuilder pipelineBuilder(M, idb);
377
378    Encoding encoding(Encoding::Type::UTF_8, 8);
379   
380    pablo::PabloFunction * function = wc_gen(encoding);
381   
382
383    pipelineBuilder.CreateKernels(function);
384
385    llvm::Function * main_IR = pipelineBuilder.ExecuteKernels();
386   
387    if (DumpGeneratedIR) {
388        M->dump();
389    }
390   
391    //verifyModule(*M, &dbgs());
392    //std::cerr << "ExecuteKernels(); done\n";
393    wcEngine = wcJIT_to_ExecutionEngine(M);
394   
395    wcEngine->finalizeObject();
396    //std::cerr << "finalizeObject(); done\n";
397
398    delete idb;
399    return reinterpret_cast<wcFunctionType>(wcEngine->getPointerToFunction(main_IR));
400}
401
402void wc(wcFunctionType fn_ptr, const int64_t fileIdx) {
403    std::string fileName = inputFiles[fileIdx];
404    size_t fileSize;
405    char * fileBuffer;
406   
407    const path file(fileName);
408    if (exists(file)) {
409        if (is_directory(file)) {
410            return;
411        }
412    } else {
413        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
414        return;
415    }
416   
417    fileSize = file_size(file);
418    mapped_file_source mappedFile;
419    if (fileSize == 0) {
420        fileBuffer = nullptr;
421    }
422    else {
423        try {
424            mappedFile.open(fileName);
425        } catch (std::exception &e) {
426            std::cerr << "Error: Boost mmap of " << fileName << ": " << e.what() << std::endl;
427            return;
428        }
429        fileBuffer = const_cast<char *>(mappedFile.data());
430    }
431    fn_ptr(fileBuffer, fileSize, fileIdx);
432
433    mappedFile.close();
434   
435}
436
437
438
439
440int main(int argc, char *argv[]) {
441    HideUnrelatedOptions(wcFlags);
442
443    cl::ParseCommandLineOptions(argc, argv);
444    if (!(CountLines || CountWords || CountChars || CountBytes)) {
445        CountLines = true;
446        CountWords = true;
447        CountBytes = true;
448    }
449   
450    wcFunctionType fn_ptr = wcCodeGen();
451
452    int fileCount = inputFiles.size();
453    lineCount.resize(fileCount);
454    wordCount.resize(fileCount);
455    charCount.resize(fileCount);
456    byteCount.resize(fileCount);
457   
458    for (unsigned i = 0; i < inputFiles.size(); ++i) {
459        wc(fn_ptr, i);
460    }
461   
462    delete wcEngine;
463   
464    size_t maxCount = 0;
465    if (CountLines) maxCount = TotalLines;
466    if (CountWords) maxCount = TotalWords;
467    if (CountChars) maxCount = TotalChars;
468    if (CountBytes) maxCount = TotalBytes;
469   
470    int fieldWidth = std::to_string(maxCount).size() + 1;
471    if (fieldWidth < defaultFieldWidth) fieldWidth = defaultFieldWidth;
472
473    for (unsigned i = 0; i < inputFiles.size(); ++i) {
474        if (CountLines) {
475            std::cout << std::setw(fieldWidth) << lineCount[i];
476        }
477        if (CountWords) {
478            std::cout << std::setw(fieldWidth) << wordCount[i];
479        }
480        if (CountChars) {
481            std::cout << std::setw(fieldWidth) << charCount[i];
482        }
483        if (CountBytes) {
484            std::cout << std::setw(fieldWidth) << byteCount[i];
485        }
486        std::cout << " " << inputFiles[i] << std::endl;
487    }
488    if (inputFiles.size() > 1) {
489        if (CountLines) {
490            std::cout << std::setw(fieldWidth) << TotalLines;
491        }
492        if (CountWords) {
493            std::cout << std::setw(fieldWidth) << TotalWords;
494        }
495        if (CountChars) {
496            std::cout << std::setw(fieldWidth) << TotalChars;
497        }
498        if (CountBytes) {
499            std::cout << std::setw(fieldWidth) << TotalBytes;
500        }
501        std::cout << " total" << std::endl;
502    }
503
504    return 0;
505}
506
507                       
Note: See TracBrowser for help on using the repository browser.