source: icGREP/icgrep-devel/icgrep/wc.cpp @ 5020

Last change on this file since 5020 was 5020, checked in by cameron, 20 months ago

Set fieldwith based on maximum line/word/char/byte count reported.

File size: 18.9 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <string>
8#include <iostream>
9#include <iomanip>
10#include <fstream>
11#include <sstream>
12
13#include <llvm/IR/Function.h>
14#include <llvm/IR/Module.h>
15#include <llvm/ExecutionEngine/ExecutionEngine.h>
16#include <llvm/ExecutionEngine/MCJIT.h>
17#include <llvm/IRReader/IRReader.h>
18#include <llvm/IR/Verifier.h>
19#include <llvm/Support/Debug.h>
20
21#include <llvm/Support/CommandLine.h>
22#include <llvm/CodeGen/CommandFlags.h>
23#include <llvm/Support/SourceMgr.h>
24#include <llvm/Support/TargetSelect.h>
25#include <llvm/Support/Host.h>
26#include <llvm/Support/raw_ostream.h>
27
28#include <utf_encoding.h>
29#include <re/re_cc.h>
30#include <cc/cc_compiler.h>
31#include <pablo/function.h>
32#include <IDISA/idisa_builder.h>
33#include <IDISA/idisa_target.h>
34#include <kernels/instance.h>
35#include <kernels/kernel.h>
36#include <kernels/s2p_kernel.h>
37
38#include <pablo/pablo_compiler.h>
39#include <pablo/pablo_toolchain.h>
40
41// Dynamic processor detection
42#define ISPC_LLVM_VERSION ISPC_LLVM_3_6
43#include <util/ispc.cpp>
44
45#include <utf_encoding.h>
46
47// mmap system
48#include <boost/filesystem.hpp>
49#include <boost/iostreams/device/mapped_file.hpp>
50using namespace boost::iostreams;
51using namespace boost::filesystem;
52
53#include <fcntl.h>
54
55static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore);
56
57static cl::opt<bool> CountLines("l", cl::desc("Report the number of lines in each input file."), cl::init(false));
58static cl::opt<bool> CountWords("w", cl::desc("Report the number of lines in each input file."), cl::init(false));
59static cl::opt<bool> CountBytes("c", cl::desc("Report the number of bytes in each input file."), cl::init(false));
60static cl::opt<bool> CountChars("m", cl::desc("Report the number of characters in each input file."), cl::init(false));
61
62
63static cl::OptionCategory eIRDumpOptions("LLVM IR Dump Options", "These options control dumping of LLVM IR.");
64static cl::opt<bool> DumpGeneratedIR("dump-generated-IR", cl::init(false), cl::desc("Print LLVM IR generated by Pablo Compiler."), cl::cat(eIRDumpOptions));
65
66static cl::OptionCategory cMachineCodeOptimization("Machine Code Optimizations", "These options control back-end compilier optimization levels.");
67
68static cl::opt<char> OptLevel("O", cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] (default = '-O0')"),
69                              cl::cat(cMachineCodeOptimization), cl::Prefix, cl::ZeroOrMore, cl::init('0'));
70
71static cl::opt<unsigned> SegmentSize("segment-size", cl::desc("Segment Size"), cl::value_desc("positive integer"), cl::init(1));
72
73
74static int defaultFieldWidth = 7;  // default field width
75
76std::vector<uint64_t> lineCount;
77std::vector<uint64_t> wordCount;
78std::vector<uint64_t> charCount;
79std::vector<uint64_t> byteCount;
80
81uint64_t TotalLines = 0;
82uint64_t TotalWords = 0;
83uint64_t TotalChars = 0;
84uint64_t TotalBytes = 0;
85
86//
87//
88extern "C" {
89    void report_counts(uint64_t lines, uint64_t words, uint64_t chars, uint64_t bytes, uint64_t fileIdx) {
90        lineCount[fileIdx] = lines;
91        wordCount[fileIdx] = words;
92        charCount[fileIdx] = chars;
93        byteCount[fileIdx] = bytes;
94        TotalLines += lines;
95        TotalWords += words;
96        TotalChars += chars;
97        TotalBytes += bytes;
98    }
99}
100
101
102
103//
104//  Functions taken from toolchain.cpp and modified for casefold
105//  JIT_t_ExecutionEngine : remove object cache
106//  icgrep_Linking:   unneeded?
107//  all others: definitely unneeded
108//
109
110ExecutionEngine * JIT_to_ExecutionEngine (Module * m) {
111
112    InitializeNativeTarget();
113    InitializeNativeTargetAsmPrinter();
114    InitializeNativeTargetAsmParser();
115
116    PassRegistry * Registry = PassRegistry::getPassRegistry();
117    initializeCore(*Registry);
118    initializeCodeGen(*Registry);
119    initializeLowerIntrinsicsPass(*Registry);
120
121    std::string errMessage;
122    EngineBuilder builder(std::move(std::unique_ptr<Module>(m)));
123    builder.setErrorStr(&errMessage);
124    builder.setMCPU(sys::getHostCPUName());
125    CodeGenOpt::Level optLevel = CodeGenOpt::Level::None;
126    switch (OptLevel) {
127        case '0': optLevel = CodeGenOpt::None; break;
128        case '1': optLevel = CodeGenOpt::Less; break;
129        case '2': optLevel = CodeGenOpt::Default; break;
130        case '3': optLevel = CodeGenOpt::Aggressive; break;
131        default: errs() << OptLevel << " is an invalid optimization level.\n";
132    }
133    builder.setOptLevel(optLevel);
134
135    if ((strncmp(lGetSystemISA(), "avx2", 4) == 0)) {
136            std::vector<std::string> attrs;
137            attrs.push_back("avx2");
138            builder.setMAttrs(attrs);
139    }
140
141    // builder.selectTarget();
142
143    //builder.setOptLevel(mMaxWhileDepth ? CodeGenOpt::Level::Less : CodeGenOpt::Level::None);
144    ExecutionEngine * engine = builder.create();
145    if (engine == nullptr) {
146        throw std::runtime_error("Could not create ExecutionEngine: " + errMessage);
147    }
148    return engine;
149}
150
151
152pablo::PabloFunction * wc_gen(Encoding encoding) {
153    //  input: 8 basis bit streams
154    //  output: 3 count streams
155   
156    pablo::PabloFunction * function = pablo::PabloFunction::Create("wc", 8, 3);
157    cc::CC_Compiler ccc(*function, encoding);
158   
159    pablo::PabloBuilder pBuilder(ccc.getBuilder().getPabloBlock(), ccc.getBuilder());
160    const std::vector<pablo::Var *> u8_bits = ccc.getBasisBits();
161
162    if (CountLines) {
163        pablo::PabloAST * LF = ccc.compileCC(re::makeCC(0x0A));
164        function->setResult(0, pBuilder.createAssign("lineCount", pBuilder.createCount(LF)));
165    }
166    else function->setResult(0, pBuilder.createAssign("lineCount", pBuilder.createZeroes()));
167    // FIXME - we need to limit this to pablo.inFile() because null bytes past EOF are matched by wordChar
168    if (CountWords) {
169        pablo::PabloAST * WS = ccc.compileCC(re::makeCC(re::makeCC(0x09, 0x0D), re::makeCC(0x20)));
170       
171        pablo::PabloAST * wordChar = ccc.compileCC(re::makeCC(re::makeCC(re::makeCC(0x00, 0x08), re::makeCC(0xE, 0x1F)), re::makeCC(0x21, 0xFF)));
172        // WS_follow_or_start = 1 past WS or at start of file
173        pablo::PabloAST * WS_follow_or_start = pBuilder.createNot(pBuilder.createAdvance(pBuilder.createNot(WS), 1));
174        //
175        pablo::PabloAST * wordStart = pBuilder.createAnd(wordChar, WS_follow_or_start);
176        function->setResult(1, pBuilder.createAssign("wordCount", pBuilder.createCount(wordStart)));
177    }
178    else function->setResult(1, pBuilder.createAssign("wordCount", pBuilder.createZeroes()));
179    if (CountChars) {
180        //
181        // FIXME: This correctly counts characters assuming valid UTF-8 input.  But what if input is
182        // not UTF-8, or is not valid?
183        //
184        pablo::PabloAST * u8Begin = ccc.compileCC(re::makeCC(re::makeCC(0, 0x7F), re::makeCC(0xC2, 0xF4)));
185        function->setResult(2, pBuilder.createAssign("charCount", pBuilder.createCount(u8Begin)));
186    }
187    else function->setResult(2, pBuilder.createAssign("charCount", pBuilder.createZeroes()));
188    return function;
189}
190
191using namespace kernel;
192
193
194class PipelineBuilder {
195public:
196    PipelineBuilder(llvm::Module * m, IDISA::IDISA_Builder * b);
197   
198    ~PipelineBuilder();
199   
200    void CreateKernels(pablo::PabloFunction * function);
201    llvm::Function * ExecuteKernels();
202   
203private:
204    llvm::Module *                      mMod;
205    IDISA::IDISA_Builder *              iBuilder;
206    KernelBuilder *                     mS2PKernel;
207    KernelBuilder *                     mWC_Kernel;
208    llvm::Type *                        mBitBlockType;
209    int                                 mBlockSize;
210};
211
212
213Function *  PipelineBuilder::ExecuteKernels() {
214    Constant * report_counts_routine;
215    Type * const int64ty = iBuilder->getInt64Ty();
216    Type * const voidTy = Type::getVoidTy(mMod->getContext());
217    report_counts_routine = mMod->getOrInsertFunction("report_counts", voidTy, int64ty, int64ty, int64ty, int64ty, int64ty, nullptr);
218    Type * const inputType = PointerType::get(ArrayType::get(StructType::get(mMod->getContext(), std::vector<Type *>({ArrayType::get(mBitBlockType, 8)})), 1), 0);
219   
220    Function * const main = cast<Function>(mMod->getOrInsertFunction("Main", voidTy, inputType, int64ty, int64ty, nullptr));
221    main->setCallingConv(CallingConv::C);
222    Function::arg_iterator args = main->arg_begin();
223   
224    Value * const inputStream = &*(args++);
225    inputStream->setName("input");
226    Value * const bufferSize = &*(args++);
227    bufferSize->setName("bufferSize");
228    Value * const fileIdx = &*(args++);
229    fileIdx->setName("fileIdx");
230   
231    iBuilder->SetInsertPoint(BasicBlock::Create(mMod->getContext(), "entry", main,0));
232   
233    BasicBlock * entryBlock = iBuilder->GetInsertBlock();
234
235    BasicBlock * segmentCondBlock = nullptr;
236    BasicBlock * segmentBodyBlock = nullptr;
237    const unsigned segmentSize = SegmentSize;
238    if (segmentSize > 1) {
239        segmentCondBlock = BasicBlock::Create(mMod->getContext(), "segmentCond", main, 0);
240        segmentBodyBlock = BasicBlock::Create(mMod->getContext(), "segmentBody", main, 0);
241    }
242    BasicBlock * fullCondBlock = BasicBlock::Create(mMod->getContext(), "fullCond", main, 0);
243    BasicBlock * fullBodyBlock = BasicBlock::Create(mMod->getContext(), "fullBody", main, 0);
244    BasicBlock * finalBlock = BasicBlock::Create(mMod->getContext(), "final", main, 0);
245    BasicBlock * finalPartialBlock = BasicBlock::Create(mMod->getContext(), "partial", main, 0);
246    BasicBlock * finalEmptyBlock = BasicBlock::Create(mMod->getContext(), "empty", main, 0);
247    BasicBlock * endBlock = BasicBlock::Create(mMod->getContext(), "end", main, 0);
248
249    Instance * s2pInstance = mS2PKernel->instantiate(inputStream);
250    Instance * wcInstance = mWC_Kernel->instantiate(s2pInstance->getOutputStreamBuffer());
251
252    Value * initialBufferSize = nullptr;
253    BasicBlock * initialBlock = nullptr;
254   
255    if (segmentSize > 1) {
256        iBuilder->CreateBr(segmentCondBlock);
257        iBuilder->SetInsertPoint(segmentCondBlock);
258        PHINode * remainingBytes = iBuilder->CreatePHI(int64ty, 2, "remainingBytes");
259        remainingBytes->addIncoming(bufferSize, entryBlock);
260        Constant * const step = ConstantInt::get(int64ty, mBlockSize * segmentSize);
261        Value * segmentCondTest = iBuilder->CreateICmpULT(remainingBytes, step);
262        iBuilder->CreateCondBr(segmentCondTest, fullCondBlock, segmentBodyBlock);
263        iBuilder->SetInsertPoint(segmentBodyBlock);
264        for (unsigned i = 0; i < segmentSize; ++i) {
265            s2pInstance->CreateDoBlockCall();
266        }
267        for (unsigned i = 0; i < segmentSize; ++i) {
268            wcInstance->CreateDoBlockCall();
269        }
270        remainingBytes->addIncoming(iBuilder->CreateSub(remainingBytes, step), segmentBodyBlock);
271        iBuilder->CreateBr(segmentCondBlock);
272        initialBufferSize = remainingBytes;
273        initialBlock = segmentCondBlock;
274    } else {
275        initialBufferSize = bufferSize;
276        initialBlock = entryBlock;
277        iBuilder->CreateBr(fullCondBlock);
278    }
279
280    iBuilder->SetInsertPoint(fullCondBlock);
281    PHINode * remainingBytes = iBuilder->CreatePHI(int64ty, 2, "remainingBytes");
282    remainingBytes->addIncoming(initialBufferSize, initialBlock);
283
284    Constant * const step = ConstantInt::get(int64ty, mBlockSize);
285    Value * fullCondTest = iBuilder->CreateICmpULT(remainingBytes, step);
286    iBuilder->CreateCondBr(fullCondTest, finalBlock, fullBodyBlock);
287   
288    iBuilder->SetInsertPoint(fullBodyBlock);
289
290    s2pInstance->CreateDoBlockCall();
291    wcInstance->CreateDoBlockCall();
292
293    Value * diff = iBuilder->CreateSub(remainingBytes, step);
294
295    remainingBytes->addIncoming(diff, fullBodyBlock);
296    iBuilder->CreateBr(fullCondBlock);
297   
298    iBuilder->SetInsertPoint(finalBlock);
299    Value * emptyBlockCond = iBuilder->CreateICmpEQ(remainingBytes, ConstantInt::get(int64ty, 0));
300    iBuilder->CreateCondBr(emptyBlockCond, finalEmptyBlock, finalPartialBlock);
301   
302   
303    iBuilder->SetInsertPoint(finalPartialBlock);
304    s2pInstance->CreateDoBlockCall();
305    iBuilder->CreateBr(endBlock);
306   
307    iBuilder->SetInsertPoint(finalEmptyBlock);
308    s2pInstance->clearOutputStreamSet();
309    iBuilder->CreateBr(endBlock);
310   
311    iBuilder->SetInsertPoint(endBlock);
312
313    wcInstance->CreateDoBlockCall();
314   
315    Value * lineCount = iBuilder->CreateExtractElement(iBuilder->CreateBlockAlignedLoad(wcInstance->getOutputStream((int) 0)), iBuilder->getInt32(0));
316    Value * wordCount = iBuilder->CreateExtractElement(iBuilder->CreateBlockAlignedLoad(wcInstance->getOutputStream(1)), iBuilder->getInt32(0));
317    Value * charCount = iBuilder->CreateExtractElement(iBuilder->CreateBlockAlignedLoad(wcInstance->getOutputStream(2)), iBuilder->getInt32(0));
318   
319    iBuilder->CreateCall(report_counts_routine, std::vector<Value *>({lineCount, wordCount, charCount, bufferSize, fileIdx}));
320   
321    iBuilder->CreateRetVoid();
322   
323    return main;
324}
325
326
327typedef void (*wcFunctionType)(char * byte_data, size_t filesize, size_t fileIdx);
328
329static ExecutionEngine * wcEngine = nullptr;
330
331wcFunctionType wcCodeGen(void) {
332                           
333    Module * M = new Module("wc", getGlobalContext());
334   
335    IDISA::IDISA_Builder * idb = GetIDISA_Builder(M);
336
337    PipelineBuilder pipelineBuilder(M, idb);
338
339    Encoding encoding(Encoding::Type::UTF_8, 8);
340   
341    pablo::PabloFunction * function = wc_gen(encoding);
342   
343
344    pipelineBuilder.CreateKernels(function);
345
346    llvm::Function * main_IR = pipelineBuilder.ExecuteKernels();
347   
348    if (DumpGeneratedIR) {
349        M->dump();
350    }
351   
352    //verifyModule(*M, &dbgs());
353    //std::cerr << "ExecuteKernels(); done\n";
354    wcEngine = JIT_to_ExecutionEngine(M);
355   
356    wcEngine->finalizeObject();
357    //std::cerr << "finalizeObject(); done\n";
358
359    delete idb;
360    return reinterpret_cast<wcFunctionType>(wcEngine->getPointerToFunction(main_IR));
361}
362
363void wc(wcFunctionType fn_ptr, const int64_t fileIdx) {
364    std::string fileName = inputFiles[fileIdx];
365    size_t fileSize;
366    char * fileBuffer;
367   
368    const path file(fileName);
369    if (exists(file)) {
370        if (is_directory(file)) {
371            return;
372        }
373    } else {
374        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
375        return;
376    }
377   
378    fileSize = file_size(file);
379    mapped_file mappedFile;
380    if (fileSize == 0) {
381        fileBuffer = nullptr;
382    }
383    else {
384        try {
385            mappedFile.open(fileName, mapped_file::priv, fileSize, 0);
386        } catch (std::ios_base::failure e) {
387            std::cerr << "Error: Boost mmap of " << fileName << ": " << e.what() << std::endl;
388            return;
389        }
390        fileBuffer = mappedFile.data();
391    }
392    //std::cerr << "mFileSize =" << mFileSize << "\n";
393    //std::cerr << "fn_ptr =" << std::hex << reinterpret_cast<intptr_t>(fn_ptr) << "\n";
394
395    fn_ptr(fileBuffer, fileSize, fileIdx);
396
397    mappedFile.close();
398   
399}
400
401
402using namespace pablo;
403using namespace kernel;
404
405PipelineBuilder::PipelineBuilder(Module * m, IDISA::IDISA_Builder * b)
406: mMod(m)
407, iBuilder(b)
408, mBitBlockType(b->getBitBlockType())
409, mBlockSize(b->getBitBlockWidth()){
410
411}
412
413PipelineBuilder::~PipelineBuilder(){
414    delete mS2PKernel;
415    delete mWC_Kernel;
416}
417
418void PipelineBuilder::CreateKernels(PabloFunction * function){
419    mS2PKernel = new KernelBuilder(iBuilder, "s2p", SegmentSize);
420    mWC_Kernel = new KernelBuilder(iBuilder, "wc", SegmentSize);
421
422    generateS2PKernel(mMod, iBuilder, mS2PKernel);
423
424    pablo_function_passes(function);
425
426    PabloCompiler pablo_compiler(mMod, iBuilder);
427    try {
428        pablo_compiler.setKernel(mWC_Kernel);
429        pablo_compiler.compile(function);
430        delete function;
431        releaseSlabAllocatorMemory();
432    } catch (std::runtime_error e) {
433        delete function;
434        releaseSlabAllocatorMemory();
435        std::cerr << "Runtime error: " << e.what() << std::endl;
436        exit(1);
437    }
438   
439}
440
441
442int main(int argc, char *argv[]) {
443    StringMap<cl::Option*> Map;
444    cl::getRegisteredOptions(Map);
445    Map["time-passes"]->setHiddenFlag(cl::Hidden);
446    Map["disable-spill-fusing"]->setHiddenFlag(cl::Hidden);
447    Map["enable-misched"]->setHiddenFlag(cl::Hidden);
448    Map["enable-tbaa"]->setHiddenFlag(cl::Hidden);
449    Map["exhaustive-register-search"]->setHiddenFlag(cl::Hidden);
450    Map["join-liveintervals"]->setHiddenFlag(cl::Hidden);
451    Map["limit-float-precision"]->setHiddenFlag(cl::Hidden);
452    Map["mc-x86-disable-arith-relaxation"]->setHiddenFlag(cl::Hidden);
453    Map["limit-float-precision"]->setHiddenFlag(cl::Hidden);
454    Map["print-after-all"]->setHiddenFlag(cl::Hidden);
455    Map["print-before-all"]->setHiddenFlag(cl::Hidden);
456    Map["print-machineinstrs"]->setHiddenFlag(cl::Hidden);
457    Map["regalloc"]->setHiddenFlag(cl::Hidden);
458    Map["rng-seed"]->setHiddenFlag(cl::Hidden);
459    Map["stackmap-version"]->setHiddenFlag(cl::Hidden);
460    Map["x86-asm-syntax"]->setHiddenFlag(cl::Hidden);
461    Map["verify-debug-info"]->setHiddenFlag(cl::Hidden);
462    Map["verify-dom-info"]->setHiddenFlag(cl::Hidden);
463    Map["verify-loop-info"]->setHiddenFlag(cl::Hidden);
464    Map["verify-regalloc"]->setHiddenFlag(cl::Hidden);
465    Map["verify-scev"]->setHiddenFlag(cl::Hidden);
466    Map["x86-recip-refinement-steps"]->setHiddenFlag(cl::Hidden);
467    Map["rewrite-map-file"]->setHiddenFlag(cl::Hidden);
468
469    cl::ParseCommandLineOptions(argc, argv);
470    if (!(CountLines || CountWords || CountChars || CountBytes)) {
471        CountLines = true;
472        CountWords = true;
473        CountBytes = true;
474    }
475   
476    wcFunctionType fn_ptr = wcCodeGen();
477
478    int fileCount = inputFiles.size();
479    lineCount.resize(fileCount);
480    wordCount.resize(fileCount);
481    charCount.resize(fileCount);
482    byteCount.resize(fileCount);
483   
484    for (unsigned i = 0; i < inputFiles.size(); ++i) {
485        wc(fn_ptr, i);
486    }
487   
488    delete wcEngine;
489   
490    int maxCount = 0;
491    if (CountLines) maxCount = TotalLines;
492    if (CountWords) maxCount = TotalWords;
493    if (CountChars) maxCount = TotalChars;
494    if (CountBytes) maxCount = TotalBytes;
495   
496    int fieldWidth = std::to_string(maxCount).size() + 1;
497    if (fieldWidth < defaultFieldWidth) fieldWidth = defaultFieldWidth;
498
499    for (unsigned i = 0; i < inputFiles.size(); ++i) {
500        if (CountLines) {
501            std::cout << std::setw(fieldWidth) << lineCount[i];
502        }
503        if (CountWords) {
504            std::cout << std::setw(fieldWidth) << wordCount[i];
505        }
506        if (CountChars) {
507            std::cout << std::setw(fieldWidth) << charCount[i];
508        }
509        if (CountBytes) {
510            std::cout << std::setw(fieldWidth) << byteCount[i];
511        }
512        std::cout << " " << inputFiles[i] << std::endl;
513    }
514    if (inputFiles.size() > 1) {
515        if (CountLines) {
516            std::cout << std::setw(fieldWidth) << TotalLines;
517        }
518        if (CountWords) {
519            std::cout << std::setw(fieldWidth) << TotalWords;
520        }
521        if (CountChars) {
522            std::cout << std::setw(fieldWidth) << TotalChars;
523        }
524        if (CountBytes) {
525            std::cout << std::setw(fieldWidth) << TotalBytes;
526        }
527        std::cout << " total" << std::endl;
528    }
529
530    return 0;
531}
532
533                       
Note: See TracBrowser for help on using the repository browser.