source: icGREP/icgrep-devel/icgrep/wc.cpp @ 5019

Last change on this file since 5019 was 5019, checked in by cameron, 3 years ago

wc initial version

File size: 18.1 KB
Line 
1/*
2 *  Copyright (c) 2015 International Characters.
3 *  This software is licensed to the public under the Open Software License 3.0.
4 *  icgrep is a trademark of International Characters.
5 */
6
7#include <string>
8#include <iostream>
9#include <iomanip>
10#include <fstream>
11#include <sstream>
12
13#include <llvm/IR/Function.h>
14#include <llvm/IR/Module.h>
15#include <llvm/ExecutionEngine/ExecutionEngine.h>
16#include <llvm/ExecutionEngine/MCJIT.h>
17#include <llvm/IRReader/IRReader.h>
18#include <llvm/IR/Verifier.h>
19#include <llvm/Support/Debug.h>
20
21#include <llvm/Support/CommandLine.h>
22#include <llvm/CodeGen/CommandFlags.h>
23#include <llvm/Support/SourceMgr.h>
24#include <llvm/Support/TargetSelect.h>
25#include <llvm/Support/Host.h>
26#include <llvm/Support/raw_ostream.h>
27
28#include <utf_encoding.h>
29#include <re/re_cc.h>
30#include <cc/cc_compiler.h>
31#include <pablo/function.h>
32#include <IDISA/idisa_builder.h>
33#include <IDISA/idisa_target.h>
34#include <kernels/instance.h>
35#include <kernels/kernel.h>
36#include <kernels/s2p_kernel.h>
37
38#include <pablo/pablo_compiler.h>
39#include <pablo/pablo_toolchain.h>
40
41// Dynamic processor detection
42#define ISPC_LLVM_VERSION ISPC_LLVM_3_6
43#include <util/ispc.cpp>
44
45#include <utf_encoding.h>
46
47// mmap system
48#include <boost/filesystem.hpp>
49#include <boost/iostreams/device/mapped_file.hpp>
50using namespace boost::iostreams;
51using namespace boost::filesystem;
52
53#include <fcntl.h>
54
55static cl::list<std::string> inputFiles(cl::Positional, cl::desc("<input file ...>"), cl::OneOrMore);
56
57static cl::opt<bool> CountLines("l", cl::desc("Report the number of lines in each input file."), cl::init(false));
58static cl::opt<bool> CountWords("w", cl::desc("Report the number of lines in each input file."), cl::init(false));
59static cl::opt<bool> CountBytes("c", cl::desc("Report the number of bytes in each input file."), cl::init(false));
60static cl::opt<bool> CountChars("m", cl::desc("Report the number of characters in each input file."), cl::init(false));
61
62
63static cl::OptionCategory eIRDumpOptions("LLVM IR Dump Options", "These options control dumping of LLVM IR.");
64static cl::opt<bool> DumpGeneratedIR("dump-generated-IR", cl::init(false), cl::desc("Print LLVM IR generated by Pablo Compiler."), cl::cat(eIRDumpOptions));
65
66static cl::OptionCategory cMachineCodeOptimization("Machine Code Optimizations", "These options control back-end compilier optimization levels.");
67
68static cl::opt<char> OptLevel("O", cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] (default = '-O0')"),
69                              cl::cat(cMachineCodeOptimization), cl::Prefix, cl::ZeroOrMore, cl::init('0'));
70
71static cl::opt<unsigned> SegmentSize("segment-size", cl::desc("Segment Size"), cl::value_desc("positive integer"), cl::init(1));
72
73
74static int fieldWidth = 9;  // default field width
75
76uint64_t TotalLines = 0;
77uint64_t TotalWords = 0;
78uint64_t TotalChars = 0;
79uint64_t TotalBytes = 0;
80
81//
82//
83extern "C" {
84    void report_counts(uint64_t lines, uint64_t words, uint64_t chars, uint64_t bytes, uint64_t fileIdx) {
85        if (CountLines) {
86            std::cout << std::setw(fieldWidth) << lines;
87            TotalLines += lines;
88        }
89        // FIXME Temporary Hack because null bytes at EOF are considered word chars.
90        if (CountWords) {
91            std::cout << std::setw(fieldWidth) << words - 1;
92            TotalWords += words - 1;
93        }
94        if (CountChars) {
95            std::cout << std::setw(fieldWidth) << chars ;
96            TotalChars += chars;
97        }
98        if (CountBytes) {
99            std::cout << std::setw(fieldWidth) << bytes;
100            TotalBytes += bytes;
101        }
102        std::cout << " " << inputFiles[fileIdx] << std::endl;
103    }
104}
105
106
107
108//
109//  Functions taken from toolchain.cpp and modified for casefold
110//  JIT_t_ExecutionEngine : remove object cache
111//  icgrep_Linking:   unneeded?
112//  all others: definitely unneeded
113//
114
115ExecutionEngine * JIT_to_ExecutionEngine (Module * m) {
116
117    InitializeNativeTarget();
118    InitializeNativeTargetAsmPrinter();
119    InitializeNativeTargetAsmParser();
120
121    PassRegistry * Registry = PassRegistry::getPassRegistry();
122    initializeCore(*Registry);
123    initializeCodeGen(*Registry);
124    initializeLowerIntrinsicsPass(*Registry);
125
126    std::string errMessage;
127    EngineBuilder builder(std::move(std::unique_ptr<Module>(m)));
128    builder.setErrorStr(&errMessage);
129    builder.setMCPU(sys::getHostCPUName());
130    CodeGenOpt::Level optLevel = CodeGenOpt::Level::None;
131    switch (OptLevel) {
132        case '0': optLevel = CodeGenOpt::None; break;
133        case '1': optLevel = CodeGenOpt::Less; break;
134        case '2': optLevel = CodeGenOpt::Default; break;
135        case '3': optLevel = CodeGenOpt::Aggressive; break;
136        default: errs() << OptLevel << " is an invalid optimization level.\n";
137    }
138    builder.setOptLevel(optLevel);
139
140    if ((strncmp(lGetSystemISA(), "avx2", 4) == 0)) {
141            std::vector<std::string> attrs;
142            attrs.push_back("avx2");
143            builder.setMAttrs(attrs);
144    }
145
146    // builder.selectTarget();
147
148    //builder.setOptLevel(mMaxWhileDepth ? CodeGenOpt::Level::Less : CodeGenOpt::Level::None);
149    ExecutionEngine * engine = builder.create();
150    if (engine == nullptr) {
151        throw std::runtime_error("Could not create ExecutionEngine: " + errMessage);
152    }
153    return engine;
154}
155
156
157pablo::PabloFunction * wc_gen(Encoding encoding) {
158    //  input: 8 basis bit streams
159    //  output: 3 count streams
160   
161    pablo::PabloFunction * function = pablo::PabloFunction::Create("wc", 8, 3);
162    cc::CC_Compiler ccc(*function, encoding);
163   
164    pablo::PabloBuilder pBuilder(ccc.getBuilder().getPabloBlock(), ccc.getBuilder());
165    const std::vector<pablo::Var *> u8_bits = ccc.getBasisBits();
166
167    if (CountLines) {
168        pablo::PabloAST * LF = ccc.compileCC(re::makeCC(0x0A));
169        function->setResult(0, pBuilder.createAssign("lineCount", pBuilder.createCount(LF)));
170    }
171    else function->setResult(0, pBuilder.createAssign("lineCount", pBuilder.createZeroes()));
172    // FIXME - we need to limit this to pablo.inFile() because null bytes past EOF are matched by wordChar
173    if (CountWords) {
174        pablo::PabloAST * WS = ccc.compileCC(re::makeCC(re::makeCC(0x09, 0x0D), re::makeCC(0x20)));
175       
176        pablo::PabloAST * wordChar = ccc.compileCC(re::makeCC(re::makeCC(re::makeCC(0x00, 0x08), re::makeCC(0xE, 0x1F)), re::makeCC(0x21, 0xFF)));
177        // WS_follow_or_start = 1 past WS or at start of file
178        pablo::PabloAST * WS_follow_or_start = pBuilder.createNot(pBuilder.createAdvance(pBuilder.createNot(WS), 1));
179        //
180        pablo::PabloAST * wordStart = pBuilder.createAnd(wordChar, WS_follow_or_start);
181        function->setResult(1, pBuilder.createAssign("wordCount", pBuilder.createCount(wordStart)));
182    }
183    else function->setResult(1, pBuilder.createAssign("wordCount", pBuilder.createZeroes()));
184    if (CountChars) {
185        //
186        // FIXME: This correctly counts characters assuming valid UTF-8 input.  But what if input is
187        // not UTF-8, or is not valid?
188        //
189        pablo::PabloAST * u8Begin = ccc.compileCC(re::makeCC(re::makeCC(0, 0x7F), re::makeCC(0xC2, 0xF4)));
190        function->setResult(2, pBuilder.createAssign("charCount", pBuilder.createCount(u8Begin)));
191    }
192    else function->setResult(2, pBuilder.createAssign("charCount", pBuilder.createZeroes()));
193    return function;
194}
195
196using namespace kernel;
197
198
199class PipelineBuilder {
200public:
201    PipelineBuilder(llvm::Module * m, IDISA::IDISA_Builder * b);
202   
203    ~PipelineBuilder();
204   
205    void CreateKernels(pablo::PabloFunction * function);
206    llvm::Function * ExecuteKernels();
207   
208private:
209    llvm::Module *                      mMod;
210    IDISA::IDISA_Builder *              iBuilder;
211    KernelBuilder *                     mS2PKernel;
212    KernelBuilder *                     mWC_Kernel;
213    llvm::Type *                        mBitBlockType;
214    int                                 mBlockSize;
215};
216
217
218Function *  PipelineBuilder::ExecuteKernels() {
219    Constant * report_counts_routine;
220    Type * const int64ty = iBuilder->getInt64Ty();
221    Type * const voidTy = Type::getVoidTy(mMod->getContext());
222    report_counts_routine = mMod->getOrInsertFunction("report_counts", voidTy, int64ty, int64ty, int64ty, int64ty, int64ty, nullptr);
223    Type * const inputType = PointerType::get(ArrayType::get(StructType::get(mMod->getContext(), std::vector<Type *>({ArrayType::get(mBitBlockType, 8)})), 1), 0);
224   
225    Function * const main = cast<Function>(mMod->getOrInsertFunction("Main", voidTy, inputType, int64ty, int64ty, nullptr));
226    main->setCallingConv(CallingConv::C);
227    Function::arg_iterator args = main->arg_begin();
228   
229    Value * const inputStream = &*(args++);
230    inputStream->setName("input");
231    Value * const bufferSize = &*(args++);
232    bufferSize->setName("bufferSize");
233    Value * const fileIdx = &*(args++);
234    fileIdx->setName("fileIdx");
235   
236    iBuilder->SetInsertPoint(BasicBlock::Create(mMod->getContext(), "entry", main,0));
237   
238    BasicBlock * entryBlock = iBuilder->GetInsertBlock();
239
240    BasicBlock * segmentCondBlock = nullptr;
241    BasicBlock * segmentBodyBlock = nullptr;
242    const unsigned segmentSize = SegmentSize;
243    if (segmentSize > 1) {
244        segmentCondBlock = BasicBlock::Create(mMod->getContext(), "segmentCond", main, 0);
245        segmentBodyBlock = BasicBlock::Create(mMod->getContext(), "segmentBody", main, 0);
246    }
247    BasicBlock * fullCondBlock = BasicBlock::Create(mMod->getContext(), "fullCond", main, 0);
248    BasicBlock * fullBodyBlock = BasicBlock::Create(mMod->getContext(), "fullBody", main, 0);
249    BasicBlock * finalBlock = BasicBlock::Create(mMod->getContext(), "final", main, 0);
250    BasicBlock * finalPartialBlock = BasicBlock::Create(mMod->getContext(), "partial", main, 0);
251    BasicBlock * finalEmptyBlock = BasicBlock::Create(mMod->getContext(), "empty", main, 0);
252    BasicBlock * endBlock = BasicBlock::Create(mMod->getContext(), "end", main, 0);
253
254    Instance * s2pInstance = mS2PKernel->instantiate(inputStream);
255    Instance * wcInstance = mWC_Kernel->instantiate(s2pInstance->getOutputStreamBuffer());
256
257    Value * initialBufferSize = nullptr;
258    BasicBlock * initialBlock = nullptr;
259   
260    if (segmentSize > 1) {
261        iBuilder->CreateBr(segmentCondBlock);
262        iBuilder->SetInsertPoint(segmentCondBlock);
263        PHINode * remainingBytes = iBuilder->CreatePHI(int64ty, 2, "remainingBytes");
264        remainingBytes->addIncoming(bufferSize, entryBlock);
265        Constant * const step = ConstantInt::get(int64ty, mBlockSize * segmentSize);
266        Value * segmentCondTest = iBuilder->CreateICmpULT(remainingBytes, step);
267        iBuilder->CreateCondBr(segmentCondTest, fullCondBlock, segmentBodyBlock);
268        iBuilder->SetInsertPoint(segmentBodyBlock);
269        for (unsigned i = 0; i < segmentSize; ++i) {
270            s2pInstance->CreateDoBlockCall();
271        }
272        for (unsigned i = 0; i < segmentSize; ++i) {
273            wcInstance->CreateDoBlockCall();
274        }
275        remainingBytes->addIncoming(iBuilder->CreateSub(remainingBytes, step), segmentBodyBlock);
276        iBuilder->CreateBr(segmentCondBlock);
277        initialBufferSize = remainingBytes;
278        initialBlock = segmentCondBlock;
279    } else {
280        initialBufferSize = bufferSize;
281        initialBlock = entryBlock;
282        iBuilder->CreateBr(fullCondBlock);
283    }
284
285    iBuilder->SetInsertPoint(fullCondBlock);
286    PHINode * remainingBytes = iBuilder->CreatePHI(int64ty, 2, "remainingBytes");
287    remainingBytes->addIncoming(initialBufferSize, initialBlock);
288
289    Constant * const step = ConstantInt::get(int64ty, mBlockSize);
290    Value * fullCondTest = iBuilder->CreateICmpULT(remainingBytes, step);
291    iBuilder->CreateCondBr(fullCondTest, finalBlock, fullBodyBlock);
292   
293    iBuilder->SetInsertPoint(fullBodyBlock);
294
295    s2pInstance->CreateDoBlockCall();
296    wcInstance->CreateDoBlockCall();
297
298    Value * diff = iBuilder->CreateSub(remainingBytes, step);
299
300    remainingBytes->addIncoming(diff, fullBodyBlock);
301    iBuilder->CreateBr(fullCondBlock);
302   
303    iBuilder->SetInsertPoint(finalBlock);
304    Value * emptyBlockCond = iBuilder->CreateICmpEQ(remainingBytes, ConstantInt::get(int64ty, 0));
305    iBuilder->CreateCondBr(emptyBlockCond, finalEmptyBlock, finalPartialBlock);
306   
307   
308    iBuilder->SetInsertPoint(finalPartialBlock);
309    s2pInstance->CreateDoBlockCall();
310    iBuilder->CreateBr(endBlock);
311   
312    iBuilder->SetInsertPoint(finalEmptyBlock);
313    s2pInstance->clearOutputStreamSet();
314    iBuilder->CreateBr(endBlock);
315   
316    iBuilder->SetInsertPoint(endBlock);
317
318    wcInstance->CreateDoBlockCall();
319   
320    Value * lineCount = iBuilder->CreateExtractElement(iBuilder->CreateBlockAlignedLoad(wcInstance->getOutputStream((int) 0)), iBuilder->getInt32(0));
321    Value * wordCount = iBuilder->CreateExtractElement(iBuilder->CreateBlockAlignedLoad(wcInstance->getOutputStream(1)), iBuilder->getInt32(0));
322    Value * charCount = iBuilder->CreateExtractElement(iBuilder->CreateBlockAlignedLoad(wcInstance->getOutputStream(2)), iBuilder->getInt32(0));
323   
324    iBuilder->CreateCall(report_counts_routine, std::vector<Value *>({lineCount, wordCount, charCount, bufferSize, fileIdx}));
325   
326    iBuilder->CreateRetVoid();
327   
328    return main;
329}
330
331
332typedef void (*wcFunctionType)(char * byte_data, size_t filesize, size_t fileIdx);
333
334static ExecutionEngine * wcEngine = nullptr;
335
336wcFunctionType wcCodeGen(void) {
337                           
338    Module * M = new Module("wc", getGlobalContext());
339   
340    IDISA::IDISA_Builder * idb = GetIDISA_Builder(M);
341
342    PipelineBuilder pipelineBuilder(M, idb);
343
344    Encoding encoding(Encoding::Type::UTF_8, 8);
345   
346    pablo::PabloFunction * function = wc_gen(encoding);
347   
348
349    pipelineBuilder.CreateKernels(function);
350
351    llvm::Function * main_IR = pipelineBuilder.ExecuteKernels();
352   
353    if (DumpGeneratedIR) {
354        M->dump();
355    }
356   
357    //verifyModule(*M, &dbgs());
358    //std::cerr << "ExecuteKernels(); done\n";
359    wcEngine = JIT_to_ExecutionEngine(M);
360   
361    wcEngine->finalizeObject();
362    //std::cerr << "finalizeObject(); done\n";
363
364    delete idb;
365    return reinterpret_cast<wcFunctionType>(wcEngine->getPointerToFunction(main_IR));
366}
367
368void wc(wcFunctionType fn_ptr, const int64_t fileIdx) {
369    std::string fileName = inputFiles[fileIdx];
370    size_t fileSize;
371    char * fileBuffer;
372   
373    const path file(fileName);
374    if (exists(file)) {
375        if (is_directory(file)) {
376            return;
377        }
378    } else {
379        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
380        return;
381    }
382   
383    fileSize = file_size(file);
384    mapped_file mappedFile;
385    if (fileSize == 0) {
386        fileBuffer = nullptr;
387    }
388    else {
389        try {
390            mappedFile.open(fileName, mapped_file::priv, fileSize, 0);
391        } catch (std::ios_base::failure e) {
392            std::cerr << "Error: Boost mmap of " << fileName << ": " << e.what() << std::endl;
393            return;
394        }
395        fileBuffer = mappedFile.data();
396    }
397    //std::cerr << "mFileSize =" << mFileSize << "\n";
398    //std::cerr << "fn_ptr =" << std::hex << reinterpret_cast<intptr_t>(fn_ptr) << "\n";
399
400    fn_ptr(fileBuffer, fileSize, fileIdx);
401
402    mappedFile.close();
403   
404}
405
406
407using namespace pablo;
408using namespace kernel;
409
410PipelineBuilder::PipelineBuilder(Module * m, IDISA::IDISA_Builder * b)
411: mMod(m)
412, iBuilder(b)
413, mBitBlockType(b->getBitBlockType())
414, mBlockSize(b->getBitBlockWidth()){
415
416}
417
418PipelineBuilder::~PipelineBuilder(){
419    delete mS2PKernel;
420    delete mWC_Kernel;
421}
422
423void PipelineBuilder::CreateKernels(PabloFunction * function){
424    mS2PKernel = new KernelBuilder(iBuilder, "s2p", SegmentSize);
425    mWC_Kernel = new KernelBuilder(iBuilder, "wc", SegmentSize);
426
427    generateS2PKernel(mMod, iBuilder, mS2PKernel);
428
429    pablo_function_passes(function);
430
431    PabloCompiler pablo_compiler(mMod, iBuilder);
432    try {
433        pablo_compiler.setKernel(mWC_Kernel);
434        pablo_compiler.compile(function);
435        delete function;
436        releaseSlabAllocatorMemory();
437    } catch (std::runtime_error e) {
438        delete function;
439        releaseSlabAllocatorMemory();
440        std::cerr << "Runtime error: " << e.what() << std::endl;
441        exit(1);
442    }
443   
444}
445
446
447int main(int argc, char *argv[]) {
448    StringMap<cl::Option*> Map;
449    cl::getRegisteredOptions(Map);
450    Map["time-passes"]->setHiddenFlag(cl::Hidden);
451    Map["disable-spill-fusing"]->setHiddenFlag(cl::Hidden);
452    Map["enable-misched"]->setHiddenFlag(cl::Hidden);
453    Map["enable-tbaa"]->setHiddenFlag(cl::Hidden);
454    Map["exhaustive-register-search"]->setHiddenFlag(cl::Hidden);
455    Map["join-liveintervals"]->setHiddenFlag(cl::Hidden);
456    Map["limit-float-precision"]->setHiddenFlag(cl::Hidden);
457    Map["mc-x86-disable-arith-relaxation"]->setHiddenFlag(cl::Hidden);
458    Map["limit-float-precision"]->setHiddenFlag(cl::Hidden);
459    Map["print-after-all"]->setHiddenFlag(cl::Hidden);
460    Map["print-before-all"]->setHiddenFlag(cl::Hidden);
461    Map["print-machineinstrs"]->setHiddenFlag(cl::Hidden);
462    Map["regalloc"]->setHiddenFlag(cl::Hidden);
463    Map["rng-seed"]->setHiddenFlag(cl::Hidden);
464    Map["stackmap-version"]->setHiddenFlag(cl::Hidden);
465    Map["x86-asm-syntax"]->setHiddenFlag(cl::Hidden);
466    Map["verify-debug-info"]->setHiddenFlag(cl::Hidden);
467    Map["verify-dom-info"]->setHiddenFlag(cl::Hidden);
468    Map["verify-loop-info"]->setHiddenFlag(cl::Hidden);
469    Map["verify-regalloc"]->setHiddenFlag(cl::Hidden);
470    Map["verify-scev"]->setHiddenFlag(cl::Hidden);
471    Map["x86-recip-refinement-steps"]->setHiddenFlag(cl::Hidden);
472    Map["rewrite-map-file"]->setHiddenFlag(cl::Hidden);
473
474    cl::ParseCommandLineOptions(argc, argv);
475    if (!(CountLines || CountWords || CountChars || CountBytes)) {
476        CountLines = true;
477        CountWords = true;
478        CountBytes = true;
479    }
480   
481    wcFunctionType fn_ptr = wcCodeGen();
482
483    for (unsigned i = 0; i != inputFiles.size(); ++i) {
484        wc(fn_ptr, i);
485    }
486   
487    delete wcEngine;
488   
489    if (inputFiles.size() > 1) {
490        if (CountLines) {
491            std::cout << std::setw(fieldWidth) << TotalLines;
492        }
493        if (CountWords) {
494            std::cout << std::setw(fieldWidth) << TotalWords;
495        }
496        if (CountChars) {
497            std::cout << std::setw(fieldWidth) << TotalChars;
498        }
499        if (CountBytes) {
500            std::cout << std::setw(fieldWidth) << TotalBytes;
501        }
502        std::cout << " total" << std::endl;
503    }
504
505    return 0;
506}
507
508                       
Note: See TracBrowser for help on using the repository browser.