Ignore:
Timestamp:
Apr 22, 2017, 4:03:25 PM (2 years ago)
Author:
nmedfort
Message:

Removed non-functional CUDA code from icgrep and consolidated grep and multigrep mode into a single function; allowed segment parallel pipeline to utilize process as its initial thread; modified MMapSourceKernel to map and perform mmap directly and advise the OS to drop consumed data streams.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/wc.cpp

    r5414 r5418  
    2525#include <pablo/pablo_compiler.h>
    2626#include <pablo/pablo_toolchain.h>
    27 #include <boost/filesystem.hpp>
    28 #include <boost/iostreams/device/mapped_file.hpp>
    29 
     27#include <fcntl.h>
    3028
    3129using namespace llvm;
     
    135133
    136134
    137 typedef void (*wcFunctionType)(char * byte_data, size_t filesize, size_t fileIdx);
     135typedef void (*WordCountFunctionType)(uint32_t fd, size_t fileIdx);
    138136
    139137void wcPipelineGen(ParabixDriver & pxDriver) {
     
    142140    Module * m = iBuilder->getModule();
    143141   
    144     Type * mBitBlockType = iBuilder->getBitBlockType();
    145     Constant * record_counts_routine;
    146     Type * const size_ty = iBuilder->getSizeTy();
     142    Type * const int32Ty = iBuilder->getInt32Ty();
     143    Type * const sizeTy = iBuilder->getSizeTy();
    147144    Type * const voidTy = iBuilder->getVoidTy();
    148     record_counts_routine = m->getOrInsertFunction("record_counts", voidTy, size_ty, size_ty, size_ty, size_ty, size_ty, nullptr);
    149     Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
    150    
    151     Function * const main = cast<Function>(m->getOrInsertFunction("Main", voidTy, inputType, size_ty, size_ty, nullptr));
     145
     146    FunctionType * const recordCountsType = FunctionType::get(voidTy, {sizeTy, sizeTy, sizeTy, sizeTy, sizeTy}, false);
     147    Constant * const recordCounts = m->getOrInsertFunction("record_counts", recordCountsType);
     148
     149    FunctionType * const mainType = FunctionType::get(voidTy, {int32Ty, sizeTy}, false);
     150    Function * const main = cast<Function>(m->getOrInsertFunction("Main", mainType));
    152151    main->setCallingConv(CallingConv::C);
    153     Function::arg_iterator args = main->arg_begin();
    154    
    155     Value * const inputStream = &*(args++);
    156     inputStream->setName("input");
    157     Value * const fileSize = &*(args++);
    158     fileSize->setName("fileSize");
     152    Function::arg_iterator args = main->arg_begin();   
     153    Value * const fileDecriptor = &*(args++);
     154    fileDecriptor->setName("fileDecriptor");
    159155    Value * const fileIdx = &*(args++);
    160156    fileIdx->setName("fileIdx");
     157
    161158    iBuilder->SetInsertPoint(BasicBlock::Create(m->getContext(), "entry", main,0));
    162159
    163     StreamSetBuffer * ByteStream = pxDriver.addExternalBuffer(make_unique<ExternalFileBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)), inputStream);
    164 
    165     StreamSetBuffer * BasisBits = pxDriver.addBuffer(make_unique<SingleBlockBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1)));
     160    StreamSetBuffer * const ByteStream = pxDriver.addBuffer(make_unique<SourceFileBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)));
     161
     162    StreamSetBuffer * const BasisBits = pxDriver.addBuffer(make_unique<SingleBlockBuffer>(iBuilder, iBuilder->getStreamSetTy(8, 1)));
    166163
    167164    KernelBuilder * mmapK = pxDriver.addKernelInstance(make_unique<MMapSourceKernel>(iBuilder));
    168     mmapK->setInitialArguments({fileSize});
     165    mmapK->setInitialArguments({fileDecriptor});
    169166    pxDriver.makeKernelCall(mmapK, {}, {ByteStream});
    170167
     
    178175    pxDriver.generatePipelineIR();
    179176   
    180     Value * lineCount = wck->createGetAccumulatorCall("lineCount");
    181     Value * wordCount = wck->createGetAccumulatorCall("wordCount");
    182     Value * charCount = wck->createGetAccumulatorCall("charCount");
    183 
    184     iBuilder->CreateCall(record_counts_routine, std::vector<Value *>({lineCount, wordCount, charCount, fileSize, fileIdx}));
     177    Value * const fileSize = mmapK->getAccumulator("fileSize");
     178    Value * const lineCount = wck->getAccumulator("lineCount");
     179    Value * const wordCount = wck->getAccumulator("wordCount");
     180    Value * const charCount = wck->getAccumulator("charCount");
     181
     182    iBuilder->CreateCall(recordCounts, {lineCount, wordCount, charCount, fileSize, fileIdx});
    185183   
    186184    iBuilder->CreateRetVoid();
     
    190188
    191189
    192 wcFunctionType wcCodeGen(void) {
     190WordCountFunctionType wcCodeGen() {
    193191    Module * M = new Module("wc", getGlobalContext());
    194192    IDISA::IDISA_Builder * idb = IDISA::GetIDISA_Builder(M);
     
    197195    wcPipelineGen(pxDriver);
    198196
    199     wcFunctionType main = reinterpret_cast<wcFunctionType>(pxDriver.getPointerToMain());
     197    WordCountFunctionType main = reinterpret_cast<WordCountFunctionType>(pxDriver.getPointerToMain());
    200198    delete idb;
    201199    return main;
    202200}
    203201
    204 void wc(wcFunctionType fn_ptr, const int64_t fileIdx) {
     202void wc(WordCountFunctionType fn_ptr, const int64_t fileIdx) {
    205203    std::string fileName = inputFiles[fileIdx];
    206     size_t fileSize;
    207     char * fileBuffer;
    208    
    209     const boost::filesystem::path file(fileName);
    210     if (exists(file)) {
    211         if (is_directory(file)) {
    212             return;
    213         }
     204    const int fd = open(fileName.c_str(), O_RDONLY);
     205    if (LLVM_UNLIKELY(fd == -1)) {
     206        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
    214207    } else {
    215         std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
    216         return;
    217     }
    218    
    219     fileSize = file_size(file);
    220     boost::iostreams::mapped_file_source mappedFile;
    221     if (fileSize == 0) {
    222         fileBuffer = nullptr;
    223     }
    224     else {
    225         try {
    226             mappedFile.open(fileName);
    227         } catch (std::exception &e) {
    228             std::cerr << "Error: Boost mmap of " << fileName << ": " << e.what() << std::endl;
    229             return;
    230         }
    231         fileBuffer = const_cast<char *>(mappedFile.data());
    232     }
    233     fn_ptr(fileBuffer, fileSize, fileIdx);
    234 
    235     mappedFile.close();
    236    
    237 }
    238 
    239 
     208        fn_ptr(fd, fileIdx);
     209        close(fd);
     210    }
     211}
    240212
    241213int main(int argc, char *argv[]) {
     
    247219        CountWords = true;
    248220        CountBytes = true;
    249     }
    250     else {
     221    } else {
    251222        CountLines = false;
    252223        CountWords = false;
     
    263234    }
    264235   
    265    
    266     wcFunctionType fn_ptr = wcCodeGen();
    267 
    268     int fileCount = inputFiles.size();
     236    WordCountFunctionType wordCountFunctionPtr = wcCodeGen();
     237
     238    const auto fileCount = inputFiles.size();
    269239    lineCount.resize(fileCount);
    270240    wordCount.resize(fileCount);
     
    272242    byteCount.resize(fileCount);
    273243   
    274     for (unsigned i = 0; i < inputFiles.size(); ++i) {
    275         wc(fn_ptr, i);
     244    for (unsigned i = 0; i < fileCount; ++i) {
     245        wc(wordCountFunctionPtr, i);
    276246    }
    277247   
Note: See TracChangeset for help on using the changeset viewer.