Ignore:
Timestamp:
Apr 22, 2017, 4:03:25 PM (2 years ago)
Author:
nmedfort
Message:

Removed non-functional CUDA code from icgrep and consolidated grep and multigrep mode into a single function; allowed segment parallel pipeline to utilize process as its initial thread; modified MMapSourceKernel to map and perform mmap directly and advise the OS to drop consumed data streams.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/editd/editd.cpp

    r5402 r5418  
    2222#include <kernels/streamset.h>
    2323#include <kernels/mmap_kernel.h>
     24#include <kernels/stdin_kernel.h>
    2425#include <kernels/s2p_kernel.h>
    2526#include <editd/editdscan_kernel.h>
    2627#include <kernels/pipeline.h>
    2728#include <editd/pattern_compiler.h>
    28 #include <boost/filesystem.hpp>
    29 #include <boost/iostreams/device/mapped_file.hpp>
     29#include <sys/stat.h>
     30#include <fcntl.h>
    3031#include <mutex>
    3132#ifdef CUDA_ENABLED
     
    187188}
    188189
    189 void buildPatternKernel(PabloKernel & kernel, IDISA::IDISA_Builder * iBuilder, const std::vector<std::string> & patterns) {
    190     PabloBuilder entry(kernel.getEntryBlock());
    191 
    192     Var * pat = kernel.getInputStreamVar("pat");
     190void buildPatternKernel(PabloKernel * const kernel, const std::vector<std::string> & patterns) {
     191    PabloBuilder entry(kernel->getEntryBlock());
     192
     193    Var * pat = kernel->getInputStreamVar("pat");
    193194
    194195    PabloAST * basisBits[4];
     
    199200    basisBits[3] = entry.createExtract(pat, 3, "G");
    200201
    201     re::Pattern_Compiler pattern_compiler(kernel);
     202    re::Pattern_Compiler pattern_compiler(*kernel);
    202203    pattern_compiler.compile(patterns, entry, basisBits, editDistance, optPosition, stepSize);
    203204
    204     pablo_function_passes(&kernel);
     205    pablo_function_passes(kernel);
    205206}
    206207
     
    224225
    225226
    226     ExternalFileBuffer ChStream(iBuilder, iBuilder->getStreamSetTy(4));
    227     SingleBlockBuffer MatchResults(iBuilder, iBuilder->getStreamSetTy(editDistance + 1));
    228 
    229     MMapSourceKernel mmapK(iBuilder);
    230     mmapK.setInitialArguments({fileSize});
    231     pxDriver.addKernelCall(mmapK, {}, {&ChStream});
    232 
    233     PabloKernel editdk(iBuilder, "editd",
    234                         {Binding{iBuilder->getStreamSetTy(4), "pat"}},
    235                         {Binding{iBuilder->getStreamSetTy(editDistance + 1), "E"}});
    236 
    237     ChStream.setStreamSetBuffer(inputStream);
    238     MatchResults.allocateBuffer();
    239 
    240     buildPatternKernel(editdk, iBuilder, patterns);
    241     pxDriver.addKernelCall(editdk, {&ChStream}, {&MatchResults});
    242 
    243     kernel::editdScanKernel editdScanK(iBuilder, editDistance);
    244     pxDriver.addKernelCall(editdScanK, {&MatchResults}, {});
     227    auto ChStream = pxDriver.addBuffer(make_unique<SourceFileBuffer>(iBuilder, iBuilder->getStreamSetTy(4)));
     228
     229    auto mmapK = pxDriver.addKernelInstance(make_unique<kernel::FileSourceKernel>(iBuilder, inputType));
     230    mmapK->setInitialArguments({inputStream, fileSize});
     231
     232    pxDriver.makeKernelCall(mmapK, {}, {ChStream});
     233
     234    auto MatchResults = pxDriver.addBuffer(make_unique<SingleBlockBuffer>(iBuilder, iBuilder->getStreamSetTy(editDistance + 1)));
     235
     236    auto editdk = pxDriver.addKernelInstance(make_unique<PabloKernel>(
     237        iBuilder, "editd", std::vector<Binding>{{iBuilder->getStreamSetTy(4), "pat"}}, std::vector<Binding>{{iBuilder->getStreamSetTy(editDistance + 1), "E"}}
     238        ));
     239
     240    buildPatternKernel(reinterpret_cast<PabloKernel *>(editdk), patterns);
     241    pxDriver.makeKernelCall(editdk, {ChStream}, {MatchResults});
     242
     243    auto editdScanK = pxDriver.addKernelInstance(make_unique<editdScanKernel>(iBuilder, editDistance));
     244    pxDriver.makeKernelCall(editdScanK, {MatchResults}, {});
    245245       
    246246    pxDriver.generatePipelineIR();
     
    251251}
    252252
    253 void buildPreprocessKernel(PabloKernel & kernel, IDISA::IDISA_Builder * iBuilder) {
    254     cc::CC_Compiler ccc(&kernel, kernel.getInputStreamVar("basis"));
     253void buildPreprocessKernel(PabloKernel * const kernel) {
     254    cc::CC_Compiler ccc(kernel, kernel->getInputStreamVar("basis"));
    255255
    256256    PabloBuilder & pb = ccc.getBuilder();
     
    261261    PabloAST * G = ccc.compileCC(re::makeCC(re::makeCC(0x47), re::makeCC(0x67)), pb);
    262262
    263     Var * const pat = kernel.getOutputStreamVar("pat");
     263    Var * const pat = kernel->getOutputStreamVar("pat");
    264264
    265265    pb.createAssign(pb.createExtract(pat, 0), A);
     
    268268    pb.createAssign(pb.createExtract(pat, 3), G);
    269269
    270     pablo_function_passes(&kernel);
     270    pablo_function_passes(kernel);
    271271}
    272272
     
    277277    Type * mBitBlockType = iBuilder->getBitBlockType();
    278278   
    279     Type * const size_ty = iBuilder->getSizeTy();
     279
    280280    Type * const voidTy = iBuilder->getVoidTy();
    281     Type * const inputType = PointerType::get(ArrayType::get(ArrayType::get(mBitBlockType, 8), 1), 0);
     281    Type * const int32Ty = iBuilder->getInt32Ty();
    282282    Type * const outputType = PointerType::get(ArrayType::get(mBitBlockType, 4), 0);
    283283   
    284     Function * const main = cast<Function>(m->getOrInsertFunction("Main", voidTy, inputType, size_ty, outputType, nullptr));
     284    Function * const main = cast<Function>(m->getOrInsertFunction("Main", voidTy, int32Ty, outputType, nullptr));
    285285    main->setCallingConv(CallingConv::C);
    286286    Function::arg_iterator args = main->arg_begin();
    287287   
    288     Value * const inputStream = &*(args++);
    289     inputStream->setName("input");
    290     Value * const fileSize = &*(args++);
    291     fileSize->setName("fileSize");
     288    Value * const fileDescriptor = &*(args++);
     289    fileDescriptor->setName("fileDescriptor");
    292290    Value * const outputStream = &*(args++);
    293291    outputStream->setName("output");
     292
    294293    iBuilder->SetInsertPoint(BasicBlock::Create(m->getContext(), "entry", main));
    295294
    296     ExternalFileBuffer ByteStream(iBuilder, iBuilder->getStreamSetTy(1, 8));
    297     SingleBlockBuffer BasisBits(iBuilder, iBuilder->getStreamSetTy(8));
    298     ExternalFileBuffer CCResults(iBuilder, iBuilder->getStreamSetTy(4));
    299 
    300     MMapSourceKernel mmapK(iBuilder);
    301     mmapK.setInitialArguments({fileSize});
    302     pxDriver.addKernelCall(mmapK, {}, {&ByteStream});
    303    
    304     S2PKernel s2pk(iBuilder);
    305     pxDriver.addKernelCall(s2pk, {&ByteStream}, {&BasisBits});
    306 
    307     PabloKernel ccck(iBuilder, "ccc",
    308                 {{iBuilder->getStreamSetTy(8), "basis"}},
    309                 {{iBuilder->getStreamSetTy(4), "pat"}});
    310 
    311     buildPreprocessKernel(ccck, iBuilder);
    312     pxDriver.addKernelCall(ccck, {&BasisBits}, {&CCResults});
    313          
    314     ByteStream.setStreamSetBuffer(inputStream);
    315 
    316     BasisBits.allocateBuffer();
    317 
    318     CCResults.setStreamSetBuffer(outputStream);
    319    
     295    auto ByteStream = pxDriver.addBuffer(make_unique<SourceFileBuffer>(iBuilder, iBuilder->getStreamSetTy(1, 8)));
     296
     297    auto mmapK = pxDriver.addKernelInstance(make_unique<kernel::MMapSourceKernel>(iBuilder, 1));
     298    mmapK->setInitialArguments({fileDescriptor});
     299    pxDriver.makeKernelCall(mmapK, {}, {ByteStream});
     300
     301    auto BasisBits = pxDriver.addBuffer(make_unique<SingleBlockBuffer>(iBuilder, iBuilder->getStreamSetTy(8)));
     302    auto s2pk = pxDriver.addKernelInstance(make_unique<S2PKernel>(iBuilder));
     303    pxDriver.makeKernelCall(s2pk, {ByteStream}, {BasisBits});
     304
     305    auto CCResults = pxDriver.addExternalBuffer(make_unique<ExternalFileBuffer>(iBuilder, iBuilder->getStreamSetTy(4)), outputStream);
     306
     307
     308
     309    auto ccck = pxDriver.addKernelInstance(make_unique<PabloKernel>(
     310        iBuilder, "ccc", std::vector<Binding>{{iBuilder->getStreamSetTy(8), "basis"}}, std::vector<Binding>{{iBuilder->getStreamSetTy(4), "pat"}}
     311        ));
     312    buildPreprocessKernel(reinterpret_cast<PabloKernel *>(ccck));
     313    pxDriver.makeKernelCall(ccck, {BasisBits}, {CCResults});
     314             
    320315    pxDriver.generatePipelineIR();
    321316
     
    326321
    327322
    328 typedef void (*preprocessFunctionType)(char * byte_data, size_t filesize, char * output_data);
     323typedef void (*preprocessFunctionType)(const int fd, char * output_data);
    329324
    330325preprocessFunctionType preprocessCodeGen() {                           
     
    352347}
    353348
    354 char * chStream;
    355 int size;
     349static char * chStream;
     350static size_t size;
     351
     352size_t file_size(const int fd) {
     353    struct stat st;
     354    if (LLVM_UNLIKELY(fstat(fd, &st) != 0)) {
     355        st.st_size = 0;
     356    }
     357    return st.st_size;
     358}
     359
    356360
    357361char * preprocess(preprocessFunctionType fn_ptr) {
    358362    std::string fileName = inputFiles[0];
    359     size_t fileSize;
    360     char * fileBuffer;
    361    
    362     const boost::filesystem::path file(fileName);
    363     if (exists(file)) {
    364         if (is_directory(file)) {
    365             exit(0);
    366         }
    367     } else {
     363    const int fd = open(inputFiles[0].c_str(), O_RDONLY);
     364    if (LLVM_UNLIKELY(fd == -1)) {
    368365        std::cerr << "Error: cannot open " << fileName << " for processing. Skipped.\n";
    369         exit(0);
    370     }
    371    
    372     fileSize = file_size(file);
    373     boost::iostreams::mapped_file_source mappedFile;
    374     if (fileSize == 0) {
    375         fileBuffer = nullptr;
    376     }
    377     else {
    378         try {
    379             mappedFile.open(fileName);
    380         } catch (std::exception &e) {
    381             std::cerr << "Error: Boost mmap of " << fileName << ": " << e.what() << std::endl;
    382             exit(0);
    383         }
    384         fileBuffer = const_cast<char *>(mappedFile.data());
    385     }
    386 
    387     int ret = posix_memalign((void**)&chStream, 32, fileSize);
     366        exit(-1);
     367    }
     368    size = file_size(fd);
     369    int ret = posix_memalign((void**)&chStream, 32, size);
    388370    if (ret) {
    389371        std::cerr << "Cannot allocate memory for output.\n";
    390         exit(-1);
    391     }
    392 
    393     fn_ptr(fileBuffer, fileSize, chStream);
    394     size = fileSize;
    395 
    396     mappedFile.close();
    397 
     372        exit(-2);
     373    }
     374    fn_ptr(fd, chStream);
     375    close(fd);
    398376    return chStream;   
    399377}
    400378
    401 void editd(editdFunctionType fn_ptr, char * inputStream, int size) {
     379void editd(editdFunctionType fn_ptr, char * inputStream, size_t size) {
    402380 
    403381    if (size == 0) {
     
    411389std::mutex count_mutex;
    412390size_t groupCount;
    413 void *DoEditd(void *threadid)
     391void * DoEditd(void *)
    414392{
    415393    size_t groupIdx;
Note: See TracChangeset for help on using the changeset viewer.