Changeset 5963 for icGREP/icgrep-devel


Ignore:
Timestamp:
Apr 11, 2018, 7:31:18 AM (16 months ago)
Author:
cameron
Message:

Fixes for internal buffer search

Location:
icGREP/icgrep-devel/icgrep/grep
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.cpp

    r5957 r5963  
    8484inline static size_t ceil_log2(const size_t v) {
    8585    assert ("log2(0) is undefined!" && v != 0);
    86     assert ("sizeof(size_t) == sizeof(ulong)" && sizeof(size_t) == sizeof(ulong));
     86    assert ("sizeof(size_t) == sizeof(long)" && sizeof(size_t) == sizeof(long));
    8787    return (sizeof(size_t) * CHAR_BIT) - __builtin_clzl(v - 1UL);
    8888}
    8989
    90 void SearchableBuffer::addSearchCandidate(char * C_string_ptr, size_t length) {
     90void SearchableBuffer::addSearchCandidate(const char * C_string_ptr) {
     91    size_t length = strlen(C_string_ptr)+1;
    9192    if (mSpace_used + length >= mAllocated_capacity) {
    9293        size_t new_capacity = size_t{1} << (ceil_log2(mSpace_used + length + 1));
    9394        AlignedAllocator<char, BUFFER_ALIGNMENT> alloc;
    94         char * new_buffer = alloc.allocate(new_capacity, 0);
     95        char * new_buffer = mAllocator.allocate(new_capacity, 0);
    9596        memcpy(new_buffer, mBuffer_base, mSpace_used);
    9697        memset(&new_buffer[mSpace_used], 0, new_capacity-mSpace_used);
     
    101102        mAllocated_capacity = new_capacity;
    102103    }
    103     memcpy((void * ) &mBuffer_base[mSpace_used], C_string_ptr, length+1);
    104     mSpace_used += length+1;
     104    memcpy((void * ) &mBuffer_base[mSpace_used], C_string_ptr, length);
     105    mSpace_used += length;
    105106    assert("Search candidate not null terminated" && (mBuffer_base[mSpace_used] == '\0'));
    106107    mEntries++;
     
    108109
    109110SearchableBuffer::SearchableBuffer() :
    110     mAllocated_capacity(INITIAL_CAPACITY), mBuffer_base(mInitial_buffer) {
     111    mAllocated_capacity(INITIAL_CAPACITY),
     112    mSpace_used(0),
     113    mEntries(0),
     114    mBuffer_base(mInitial_buffer) {
    111115    memset(mBuffer_base, 0, INITIAL_CAPACITY);
    112116}
     
    114118SearchableBuffer::~SearchableBuffer() {
    115119    if (mBuffer_base != mInitial_buffer) {
    116         AlignedAllocator<char, BUFFER_ALIGNMENT> alloc;
    117         alloc.deallocate(mBuffer_base, 0);
     120        mAllocator.deallocate(mBuffer_base, 0);
    118121    }
    119122}
     
    745748    Module * M = idb->getModule();
    746749   
     750    const unsigned encodingBits = 8;
    747751    const unsigned segmentSize = codegen::BufferSegments * codegen::SegmentSize * codegen::ThreadNum;
    748752   
    749753    re::CC * breakCC = nullptr;
    750754    if (mGrepRecordBreak == GrepRecordBreakKind::Null) {
    751         breakCC = re::makeByte(0xA);
     755        breakCC = re::makeByte(0x0);
    752756    } else {// if (mGrepRecordBreak == GrepRecordBreakKind::LF)
    753757        breakCC = re::makeByte(0x0A);
     
    758762        matchingRE = re::exclude_CC(matchingRE, breakCC);
    759763        matchingRE = resolveAnchors(matchingRE, breakCC);
     764        matchingRE = toUTF8(matchingRE);
    760765    }
    761766    if (excludedRE != nullptr) {
     
    764769        excludedRE = re::exclude_CC(excludedRE, breakCC);
    765770        excludedRE = resolveAnchors(excludedRE, breakCC);
     771        excludedRE = toUTF8(excludedRE);
    766772    }
    767773    Function * mainFunc = cast<Function>(M->getOrInsertFunction("Main", idb->getVoidTy(), idb->getInt8PtrTy(), idb->getSizeTy(), nullptr));
     
    787793    mGrepDriver->makeKernelCall(breakK, {BasisBits}, {RecordBreakStream});
    788794   
    789     StreamSetBuffer * MatchingRecords = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    790795   
    791796    std::vector<std::string> externalStreamNames;
     797    StreamSetBuffer * MatchingRecords = nullptr;
    792798    if (matchingRE != nullptr) {
    793799        StreamSetBuffer * MatchResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    794800        kernel::Kernel * includeK = mGrepDriver->addKernelInstance<kernel::ICGrepKernel>(idb, matchingRE, externalStreamNames);
    795801        mGrepDriver->makeKernelCall(includeK, {BasisBits}, {MatchResults});
     802        MatchingRecords = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    796803        kernel::Kernel * matchedLinesK = mGrepDriver->addKernelInstance<kernel::MatchedLinesKernel>(idb);
    797804        mGrepDriver->makeKernelCall(matchedLinesK, {MatchResults, RecordBreakStream}, {MatchingRecords});
    798805    }
    799    
    800806    if (excludedRE != nullptr) {
    801807        StreamSetBuffer * ExcludedResults = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
     
    811817            mGrepDriver->makeKernelCall(invertK, {ExcludedRecords, RecordBreakStream}, {nonExcluded});
    812818            StreamSetBuffer * included = MatchingRecords;
    813             kernel::Kernel * streamsMergeK = mGrepDriver->addKernelInstance<kernel::StreamsMerge>(idb, 1, 2);
    814             mGrepDriver->makeKernelCall(streamsMergeK, {included, nonExcluded}, {MatchingRecords});
     819            MatchingRecords = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
     820            kernel::Kernel * streamsIntersectK = mGrepDriver->addKernelInstance<kernel::StreamsIntersect>(idb, 1, 2);
     821            mGrepDriver->makeKernelCall(streamsIntersectK, {included, nonExcluded}, {MatchingRecords});
    815822        }
    816823        else {
     824            MatchingRecords = mGrepDriver->addBuffer<CircularBuffer>(idb, idb->getStreamSetTy(1, 1), segmentSize);
    817825            mGrepDriver->makeKernelCall(invertK, {ExcludedRecords, RecordBreakStream}, {MatchingRecords});
    818826        }
    819827    }
    820 
    821828    kernel::Kernel * scanMatchK = mGrepDriver->addKernelInstance<kernel::ScanMatchKernel>(idb);
    822829    scanMatchK->setInitialArguments({ConstantInt::get(idb->getIntAddrTy(), reinterpret_cast<intptr_t>(accum))});
  • icGREP/icgrep-devel/icgrep/grep/grep_engine.h

    r5954 r5963  
     1
    12/*
    23 *  Copyright (c) 2017 International Characters.
     
    1314#include <sstream>
    1415#include <atomic>
     16#include <util/aligned_allocator.h>
    1517
    1618namespace re { class CC; }
     
    182184#define INITIAL_CAPACITY 1024
    183185   
    184     class SearchableBuffer  {
    185         SearchableBuffer();
    186         void addSearchCandidate(char * string_ptr, size_t length);
    187         size_t getCandidateCount() {return mEntries;}
    188         char * getBufferBase() {return mBuffer_base;}
    189         size_t getBufferSize() {return mSpace_used;}
    190         ~SearchableBuffer();
    191     private:
    192         static const unsigned BUFFER_ALIGNMENT = MAX_SIMD_WIDTH_SUPPORTED/8;
    193         size_t mAllocated_capacity;
    194         char * mBuffer_base;
    195         alignas(BUFFER_ALIGNMENT) char mInitial_buffer[INITIAL_CAPACITY];
    196         size_t mSpace_used;
    197         size_t mEntries;
    198     };
     186class SearchableBuffer  {
     187public:
     188    SearchableBuffer();
     189    void addSearchCandidate(const char * string_ptr);
     190    size_t getCandidateCount() {return mEntries;}
     191    char * getBufferBase() {return mBuffer_base;}
     192    size_t getBufferSize() {return mSpace_used;}
     193    ~SearchableBuffer();
     194private:
     195    static const unsigned BUFFER_ALIGNMENT = MAX_SIMD_WIDTH_SUPPORTED/8;
     196    AlignedAllocator<char, BUFFER_ALIGNMENT> mAllocator;
     197    size_t mAllocated_capacity;
     198    size_t mSpace_used;
     199    size_t mEntries;
     200    char * mBuffer_base;
     201    alignas(BUFFER_ALIGNMENT) char mInitial_buffer[INITIAL_CAPACITY];
     202};
    199203
    200204}
Note: See TracChangeset for help on using the changeset viewer.