Changeset 5493


Ignore:
Timestamp:
Jun 6, 2017, 6:20:55 PM (4 months ago)
Author:
cameron
Message:

Restore check-ins from the last several days

Location:
icGREP/icgrep-devel/icgrep
Files:
4 added
27 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/CMakeLists.txt

    r5492 r5493  
    8080add_library(PabloADT ${PABLO_SRC})
    8181add_library(RegExpADT re/re_re.cpp re/re_cc.cpp re/re_rep.cpp re/re_diff.cpp re/re_intersect.cpp re/printer_re.cpp)
    82 add_library(RegExpCompiler re/re_parser.cpp re/re_nullable.cpp re/re_simplifier.cpp re/re_compiler.cpp re/re_analysis.cpp re/re_toolchain.cpp re/re_name_resolve.cpp re/re_parser_pcre.cpp re/re_parser_ere.cpp re/re_parser_bre.cpp re/re_parser_prosite.cpp re/re_utility.cpp)
     82add_library(RegExpCompiler re/re_parser.cpp re/re_nullable.cpp re/re_simplifier.cpp re/re_star_normal.cpp re/re_compiler.cpp re/re_analysis.cpp re/re_toolchain.cpp re/re_name_resolve.cpp re/re_parser_pcre.cpp re/re_parser_ere.cpp re/re_parser_bre.cpp re/re_parser_prosite.cpp re/re_utility.cpp)
    8383add_library(CCADT cc/cc_compiler.cpp utf8_encoder.cpp utf16_encoder.cpp UCD/CaseFolding_txt.cpp cc/alphabet.cpp cc/multiplex_CCs.cpp)
    8484add_library(UCDlib UCD/unicode_set.cpp UCD/ucd_compiler.cpp UCD/PropertyObjects.cpp UCD/resolve_properties.cpp UCD/UnicodeNameData.cpp)
     
    159159SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
    160160
    161 include(CheckFunctionExists)
    162 CHECK_FUNCTION_EXISTS(aligned_alloc STDLIB_HAS_ALIGNED_ALLOC)
    163 IF (STDLIB_HAS_ALIGNED_ALLOC)
    164 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSTDLIB_HAS_ALIGNED_ALLOC")
     161UNSET(HAS_EXECINFO CACHE)
     162UNSET(HAS_MACH_VM_TYPES CACHE)
     163include(CheckIncludeFileCXX)
     164CHECK_INCLUDE_FILE_CXX(mach/vm_types.h HAS_MACH_VM_TYPES)
     165IF (HAS_MACH_VM_TYPES)
     166  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_MACH_VM_TYPES")
    165167ELSE()
    166 CHECK_FUNCTION_EXISTS(posix_memalign STDLIB_HAS_POSIX_MEMALIGN)
    167 IF (!STDLIB_HAS_POSIX_MEMALIGN)
    168 MESSAGE(FATAL_ERROR "stdlib.h does not contain either aligned_alloc or posix_memalign" )
    169 ENDIF()
    170 ENDIF()
    171 
    172 include(CheckIncludeFileCXX)
    173 find_package(Libunwind)
    174 IF (LIBUNWIND_FOUND)
    175 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_LIBUNWIND")
    176 include_directories(${LIBUNWIND_INCLUDE_DIR})
    177 target_link_libraries(CodeGen ${LIBUNWIND_LIBRARIES})
    178 ELSE()
    179 CHECK_INCLUDE_FILE_CXX(execinfo.h HAS_EXECINFO)
    180 IF (HAS_EXECINFO)
    181 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_EXECINFO")
    182 ENDIF()
     168  UNSET(LIBUNWIND_FOUND CACHE)
     169  find_package(Libunwind)
     170  IF (LIBUNWIND_FOUND)
     171    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_LIBUNWIND")
     172    include_directories(${LIBUNWIND_INCLUDE_DIR})
     173    target_link_libraries(CodeGen ${LIBUNWIND_LIBRARIES})
     174  ELSE()
     175    CHECK_INCLUDE_FILE_CXX(execinfo.h HAS_EXECINFO)
     176    IF (HAS_EXECINFO)
     177       SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_EXECINFO")
     178    ENDIF()
     179  ENDIF()
    183180ENDIF()
    184181
     
    202199SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_DEBUG} -O1 -g -fno-omit-frame-pointer -fno-optimize-sibling-calls")
    203200
    204 CHECK_INCLUDE_FILE_CXX(sanitizer/asan_interface.h HAS_ADDRESS_SANITIZER)
     201UNSET(HAS_ADDRESS_SANITIZER)
     202CHECK_INCLUDE_FILE_CXX("sanitizer/asan_interface.h" HAS_ADDRESS_SANITIZER)
    205203IF (HAS_ADDRESS_SANITIZER)
    206204SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DHAS_ADDRESS_SANITIZER -fsanitize=address")
    207205ENDIF()
    208206
    209 
    210207add_test(
    211208  NAME greptest
    212209  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA
    213   COMMAND python2 greptest.py ${CMAKE_BINARY_DIR}/icgrep)
     210  COMMAND timeout 1000 python greptest.py ${CMAKE_BINARY_DIR}/icgrep)
    214211
    215212add_test(
    216213  NAME proptest
    217214  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA
    218   COMMAND python2 greptest.py -d ${PROJECT_SOURCE_DIR}/../QA -t ${PROJECT_SOURCE_DIR}/../QA/proptest.xml ${CMAKE_BINARY_DIR}/icgrep)
     215  COMMAND timeout 2000 python greptest.py -d ${PROJECT_SOURCE_DIR}/../QA -t ${PROJECT_SOURCE_DIR}/../QA/proptest.xml ${CMAKE_BINARY_DIR}/icgrep)
    219216
    220217add_test(
    221218  NAME abc_test
    222219  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA
    223   COMMAND ./run_abc ${CMAKE_BINARY_DIR}/icgrep)
     220  COMMAND timeout 200 ./run_abc ${CMAKE_BINARY_DIR}/icgrep)
    224221
    225222add_test(
    226223  NAME u8u16_test
    227224  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA/u8u16
    228   COMMAND ./run_all "${CMAKE_BINARY_DIR}/u8u16 -segment-size=16 -enable-segment-pipeline-parallel")
     225  COMMAND timeout 15 ./run_all "${CMAKE_BINARY_DIR}/u8u16 -segment-size=16 -enable-segment-pipeline-parallel")
    229226
    230227add_test(
    231228  NAME lz4d_test
    232229  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA/lz4d
    233   COMMAND ./run_all ${CMAKE_BINARY_DIR}/lz4d)
     230  COMMAND timeout 15 ./run_all ${CMAKE_BINARY_DIR}/lz4d)
    234231
    235232add_test(
    236233  NAME editd_test
    237234  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA/editd
    238   COMMAND python run_all.py -e ${CMAKE_BINARY_DIR})
     235  COMMAND timeout 50 python run_all.py -e ${CMAKE_BINARY_DIR})
    239236
    240237add_test(
    241238  NAME base64_test
    242239  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA/base64
    243   COMMAND python base64test.py ${CMAKE_BINARY_DIR}/base64)
     240  COMMAND timeout 15 python base64test.py ${CMAKE_BINARY_DIR}/base64)
     241
     242add_custom_target (greptest
     243  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA
     244  COMMAND timeout 600 python greptest.py "${CMAKE_BINARY_DIR}/icgrep -ea")
     245
     246add_custom_target (proptest
     247  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA
     248  COMMAND timeout 1000 python greptest.py -d ${PROJECT_SOURCE_DIR}/../QA -t ${PROJECT_SOURCE_DIR}/../QA/proptest.xml "${CMAKE_BINARY_DIR}/icgrep -ea")
    244249
    245250add_custom_target (u8u16_test
    246251  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA/u8u16
    247   COMMAND ./run_all "${CMAKE_BINARY_DIR}/u8u16 -segment-size=16 -enable-segment-pipeline-parallel")
     252  COMMAND timeout 15 ./run_all "${CMAKE_BINARY_DIR}/u8u16 -ea -segment-size=16 -enable-segment-pipeline-parallel")
    248253
    249254add_custom_target (lz4d_test
    250255  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA/lz4d
    251   COMMAND ./run_all ${CMAKE_BINARY_DIR}/lz4d)
     256  COMMAND timeout 15 ./run_all "${CMAKE_BINARY_DIR}/lz4d -ea")
    252257
    253258add_custom_target (editd_test
    254259  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA/editd
    255   COMMAND python run_all.py -e ${CMAKE_BINARY_DIR})
     260  COMMAND timeout 50 python run_all.py -e ${CMAKE_BINARY_DIR})
    256261
    257262add_custom_target (base64_test
    258263  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA/base64
    259   COMMAND python base64test.py ${CMAKE_BINARY_DIR}/base64)
     264  COMMAND timeout 20 python base64test.py "${CMAKE_BINARY_DIR}/base64 -ea")
    260265
    261266add_custom_target (abc_test
    262   COMMAND python2 abc_testgen.py
    263   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA)
     267  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/../QA
     268  COMMAND timeout 200 ./run_abc "${CMAKE_BINARY_DIR}/icgrep -ea")
    264269
    265270add_custom_target (check
  • icGREP/icgrep-devel/icgrep/IR_Gen/CBuilder.cpp

    r5489 r5493  
    2121#include <unistd.h>
    2222#include <stdio.h>
    23 #ifdef HAS_ADDRESS_SANITIZER
    24 #include <sanitizer/asan_interface.h>
    25 #endif
    26 #ifdef HAS_LIBUNWIND
     23#if defined(HAS_MACH_VM_TYPES)
     24#include <mach/vm_types.h>
     25extern void _thread_stack_pcs(vm_address_t *buffer, unsigned max, unsigned *nb, unsigned skip);
     26static_assert(sizeof(vm_address_t) == sizeof(uintptr_t), "");
     27#elif defined(HAS_LIBUNWIND)
    2728#define UNW_LOCAL_ONLY
    2829#include <libunwind.h>
    29 using unw_word_t = unw_word_t;
    30 #elif HAS_EXECINFO
     30static_assert(sizeof(unw_word_t) <= sizeof(uintptr_t), "");
     31#elif defined(HAS_EXECINFO)
    3132#include <execinfo.h>
    32 #include <boost/integer.hpp>
    33 using unw_word_t = boost::uint_t<sizeof(void *) * CHAR_BIT>::exact;
    34 #else
    35 using unw_word_t = uint64_t;
     33static_assert(sizeof(void *) == sizeof(uintptr_t), "");
    3634#endif
    3735
     
    247245    size = CreateZExtOrTrunc(size, sizeTy);
    248246    CallInst * const ptr = CreateCall(f, size);
    249     ptr->setTailCall();
    250247    CreateAssert(ptr, "CreateMalloc: returned null pointer");
    251248    return ptr;
     
    259256    IntegerType * const sizeTy = getSizeTy();
    260257    PointerType * const voidPtrTy = getVoidPtrTy();
    261     #ifdef STDLIB_HAS_ALIGNED_ALLOC
    262     Function * f = m->getFunction("aligned_alloc");
    263     if (LLVM_UNLIKELY(f == nullptr)) {
    264         FunctionType * const fty = FunctionType::get(voidPtrTy, {sizeTy, sizeTy}, false);
    265         f = Function::Create(fty, Function::ExternalLinkage, "aligned_alloc", m);
    266         f->setCallingConv(CallingConv::C);
    267         f->setDoesNotAlias(0);
    268     }
    269     #else
    270     Function * f = m->getFunction("posix_memalign");
    271     if (LLVM_UNLIKELY(f == nullptr)) {
    272         FunctionType * const fty = FunctionType::get(getInt32Ty(), {voidPtrTy->getPointerTo(), sizeTy, sizeTy}, false);
    273         f = Function::Create(fty, Function::ExternalLinkage, "posix_memalign", m);
    274         f->setCallingConv(CallingConv::C);
    275         f->setDoesNotAlias(1);
    276     }
    277     #endif
     258
    278259    size = CreateZExtOrTrunc(size, sizeTy);
    279260    ConstantInt * const align = ConstantInt::get(sizeTy, alignment);
    280261    if (codegen::EnableAsserts) {
    281         CreateAssert(CreateICmpEQ(CreateURem(size, align), ConstantInt::get(sizeTy, 0)),
    282                      "CreateAlignedMalloc: size must be an integral multiple of alignment.");
    283     }
    284     #ifdef STDLIB_HAS_ALIGNED_ALLOC
    285     CallInst * const ptr = CreateCall(f, {align, size});
    286     ptr->setTailCall();
    287     #else
    288     Value * ptr = CreateAlloca(voidPtrTy);
    289     CallInst * success = CreateCall(f, {ptr, align, size});
    290     success->setTailCall();
    291     if (codegen::EnableAsserts) {
    292         CreateAssert(CreateICmpEQ(success, getInt32(0)),
    293                      "CreateAlignedMalloc: posix_memalign reported bad allocation");
    294     }
    295     ptr = CreateLoad(ptr);
    296     #endif
     262        CreateAssertZero(CreateURem(size, align), "CreateAlignedMalloc: size must be an integral multiple of alignment.");
     263    }
     264
     265    Value * ptr = nullptr;
     266    if (hasAlignedAlloc()) {
     267        Function * f = m->getFunction("aligned_alloc");
     268        if (LLVM_UNLIKELY(f == nullptr)) {
     269            FunctionType * const fty = FunctionType::get(voidPtrTy, {sizeTy, sizeTy}, false);
     270            f = Function::Create(fty, Function::ExternalLinkage, "aligned_alloc", m);
     271            f->setCallingConv(CallingConv::C);
     272            f->setDoesNotAlias(0);
     273        }
     274        ptr = CreateCall(f, {align, size});
     275    } else if (hasPosixMemalign()) {
     276        Function * f = m->getFunction("posix_memalign");
     277        if (LLVM_UNLIKELY(f == nullptr)) {
     278            FunctionType * const fty = FunctionType::get(getInt32Ty(), {voidPtrTy->getPointerTo(), sizeTy, sizeTy}, false);
     279            f = Function::Create(fty, Function::ExternalLinkage, "posix_memalign", m);
     280            f->setCallingConv(CallingConv::C);
     281            f->setDoesNotAlias(0);
     282            f->setDoesNotAlias(1);
     283        }
     284        ptr = CreateAlloca(voidPtrTy);
     285        CallInst * success = CreateCall(f, {ptr, align, size});
     286        if (codegen::EnableAsserts) {
     287            CreateAssertZero(success, "CreateAlignedMalloc: posix_memalign reported bad allocation");
     288        }
     289        ptr = CreateLoad(ptr);
     290    } else {
     291        report_fatal_error("stdlib.h does not contain either aligned_alloc or posix_memalign");
     292    }
    297293    CreateAssert(ptr, "CreateAlignedMalloc: returned null pointer.");
    298294    return ptr;
    299295}
    300296
    301 Value * CBuilder::CreateRealloc(Value * const ptr, Value * size) {
     297inline bool CBuilder::hasAlignedAlloc() const {
     298    return mDriver && mDriver->hasExternalFunction("aligned_alloc");
     299}
     300
     301
     302inline bool CBuilder::hasPosixMemalign() const {
     303    return mDriver && mDriver->hasExternalFunction("posix_memalign");
     304}
     305
     306Value * CBuilder::CreateRealloc(Value * const ptr, Value * const size) {
    302307    Module * const m = getModule();
    303308    IntegerType * const sizeTy = getSizeTy();
     
    311316        f->setDoesNotAlias(1);
    312317    }
    313     Value * const addr = CreatePointerCast(ptr, voidPtrTy);
    314     size = CreateZExtOrTrunc(size, sizeTy);
    315     CallInst * const ci = CreateCall(f, {addr, size});
    316     ci->setTailCall();
     318    CallInst * const ci = CreateCall(f, {CreatePointerCast(ptr, voidPtrTy), CreateZExtOrTrunc(size, sizeTy)});
     319
    317320    return CreatePointerCast(ci, ptr->getType());
    318321}
    319322
    320 void CBuilder::CreateFree(Value * ptr) {
     323void CBuilder::CreateFree(Value * const ptr) {
    321324    assert (ptr->getType()->isPointerTy());
    322325    Module * const m = getModule();
     
    328331        f->setCallingConv(CallingConv::C);
    329332    }
    330     ptr = CreatePointerCast(ptr, voidPtrTy);
    331     CreateCall(f, ptr)->setTailCall();
     333    CreateCall(f, CreatePointerCast(ptr, voidPtrTy));
    332334}
    333335
     
    485487        Value * const addrValue = CreatePtrToInt(addr, intPtrTy);
    486488        Value * const pageOffset = CreateURem(addrValue, ConstantInt::get(intPtrTy, getpagesize()));
    487         CreateAssert(CreateICmpEQ(pageOffset, ConstantInt::getNullValue(intPtrTy)), "CreateMUnmap: addr must be a multiple of the page size");
     489        CreateAssertZero(pageOffset, "CreateMUnmap: addr must be a multiple of the page size");
    488490        Value * const boundCheck = CreateICmpULT(addrValue, CreateSub(ConstantInt::getAllOnesValue(intPtrTy), CreateZExtOrTrunc(len, intPtrTy)));
    489491        CreateAssert(boundCheck, "CreateMUnmap: addresses in [addr, addr+len) are outside the valid address space range");
     
    494496
    495497PointerType * CBuilder::getVoidPtrTy() const {
    496     return TypeBuilder<void *, true>::get(getContext());
     498    return TypeBuilder<void *, false>::get(getContext());
    497499}
    498500
     
    641643}
    642644
    643 void __report_failure(const char * msg, const unw_word_t * trace, const uint32_t n) {
     645void __report_failure(const char * msg, const uintptr_t * trace, const uint32_t n) {
    644646    raw_fd_ostream out(STDERR_FILENO, false);
    645     #if defined(HAS_LIBUNWIND) || defined(HAS_EXECINFO)
    646647    if (trace) {
    647648        SmallVector<char, 4096> tmp;
     
    667668        out << trace_string.str();
    668669    }
    669     #endif
    670670    out.changeColor(raw_fd_ostream::WHITE, true);
    671671    out << "Assertion `" << msg << "' failed.\n";
    672672    out.resetColor();
    673673    out.flush();
    674 }
    675 
    676 void CBuilder::CreateAssert(Value * assertion, StringRef failureMessage) {
     674
     675}
     676
     677#if defined(HAS_MACH_VM_TYPES)
     678
     679/*
     680 * Copyright (c) 1999, 2007 Apple Inc. All rights reserved.
     681 *
     682 * @APPLE_LICENSE_HEADER_START@
     683 *
     684 * This file contains Original Code and/or Modifications of Original Code
     685 * as defined in and that are subject to the Apple Public Source License
     686 * Version 2.0 (the 'License'). You may not use this file except in
     687 * compliance with the License. Please obtain a copy of the License at
     688 * http://www.opensource.apple.com/apsl/ and read it before using this
     689 * file.
     690 *
     691 * The Original Code and all software distributed under the License are
     692 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
     693 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
     694 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
     695 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
     696 * Please see the License for the specific language governing rights and
     697 * limitations under the License.
     698 *
     699 * @APPLE_LICENSE_HEADER_END@
     700 */
     701
     702#include <pthread.h>
     703#include <mach/mach.h>
     704#include <mach/vm_statistics.h>
     705#include <stdlib.h>
     706
     707#if defined(__i386__) || defined(__x86_64__)
     708#define FP_LINK_OFFSET 1
     709#elif defined(__ppc__) || defined(__ppc64__)
     710#define FP_LINK_OFFSET 2
     711#else
     712#error  ********** Unimplemented architecture
     713#endif
     714
     715#define INSTACK(a)      ((uintptr_t)(a) >= stackbot && (uintptr_t)(a) <= stacktop)
     716#if defined(__ppc__) || defined(__ppc64__) || defined(__x86_64__)
     717#define ISALIGNED(a)    ((((uintptr_t)(a)) & 0xf) == 0)
     718#elif defined(__i386__)
     719#define ISALIGNED(a)    ((((uintptr_t)(a)) & 0xf) == 8)
     720#endif
     721
     722__private_extern__  __attribute__((noinline))
     723void
     724_thread_stack_pcs(vm_address_t *buffer, unsigned max, unsigned *nb, unsigned skip)
     725{
     726    void *frame, *next;
     727    pthread_t self = pthread_self();
     728    uintptr_t stacktop = (uintptr_t)(pthread_get_stackaddr_np(self));
     729    uintptr_t stackbot = stacktop - (uintptr_t)(pthread_get_stacksize_np(self));
     730   
     731    *nb = 0;
     732   
     733    /* make sure return address is never out of bounds */
     734    stacktop -= (FP_LINK_OFFSET + 1) * sizeof(void *);
     735   
     736    /*
     737     * The original implementation called the first_frame_address() function,
     738     * which returned the stack frame pointer.  The problem was that in ppc,
     739     * it was a leaf function, so no new stack frame was set up with
     740     * optimization turned on (while a new stack frame was set up without
     741     * optimization).  We now inline the code to get the stack frame pointer,
     742     * so we are consistent about the stack frame.
     743     */
     744#if defined(__i386__) || defined(__x86_64__)
     745    frame = __builtin_frame_address(0);
     746#elif defined(__ppc__) || defined(__ppc64__)
     747    /* __builtin_frame_address IS BROKEN IN BEAKER: RADAR #2340421 */
     748    __asm__ volatile("mr %0, r1" : "=r" (frame));
     749#endif
     750    if(!INSTACK(frame) || !ISALIGNED(frame))
     751        return;
     752#if defined(__ppc__) || defined(__ppc64__)
     753    /* back up the stack pointer up over the current stack frame */
     754    next = *(void **)frame;
     755    if(!INSTACK(next) || !ISALIGNED(next) || next <= frame)
     756        return;
     757    frame = next;
     758#endif
     759    while (skip--) {
     760        next = *(void **)frame;
     761        if(!INSTACK(next) || !ISALIGNED(next) || next <= frame)
     762            return;
     763        frame = next;
     764    }
     765    while (max--) {
     766        buffer[*nb] = *(vm_address_t *)(((void **)frame) + FP_LINK_OFFSET);
     767        (*nb)++;
     768        next = *(void **)frame;
     769        if(!INSTACK(next) || !ISALIGNED(next) || next <= frame)
     770            return;
     771        frame = next;
     772    }
     773}
     774#endif
     775
     776void CBuilder::__CreateAssert(Value * const assertion, StringRef failureMessage) {
    677777    if (LLVM_UNLIKELY(codegen::EnableAsserts)) {
    678778        Module * const m = getModule();
     779        if (LLVM_UNLIKELY(isa<ConstantInt>(assertion))) {
     780            if (LLVM_UNLIKELY(cast<ConstantInt>(assertion)->isZero())) {
     781                report_fatal_error(failureMessage);
     782            } else {
     783                return;
     784            }
     785        }
     786        Type * const stackTy = TypeBuilder<uintptr_t, false>::get(getContext());
     787        PointerType * const stackPtrTy = stackTy->getPointerTo();
     788        PointerType * const int8PtrTy = getInt8PtrTy();
    679789        Function * function = m->getFunction("assert");
    680         IntegerType * const int1Ty = getInt1Ty();
    681790        if (LLVM_UNLIKELY(function == nullptr)) {
    682791            auto ip = saveIP();
    683             PointerType * const int8PtrTy = getInt8PtrTy();
    684             IntegerType * const unwWordTy = TypeBuilder<unw_word_t, false>::get(getContext());
    685             PointerType * const unwWordPtrTy = unwWordTy->getPointerTo();
    686             FunctionType * fty = FunctionType::get(getVoidTy(), { int1Ty, int8PtrTy, unwWordPtrTy, getInt32Ty() }, false);
     792            IntegerType * const int1Ty = getInt1Ty();
     793            FunctionType * fty = FunctionType::get(getVoidTy(), { int1Ty, int8PtrTy, stackPtrTy, getInt32Ty() }, false);
    687794            function = Function::Create(fty, Function::PrivateLinkage, "assert", m);
    688795            function->setDoesNotThrow();
     
    710817            restoreIP(ip);
    711818        }
    712         if (assertion->getType() != int1Ty) {
    713             assertion = CreateICmpNE(assertion, Constant::getNullValue(assertion->getType()));
    714         }
    715         SmallVector<unw_word_t, 64> stack;
    716         #ifdef HAS_LIBUNWIND
     819
     820        SmallVector<uint64_t, 64> stack;
     821        #if defined(HAS_MACH_VM_TYPES)
     822        for (;;) {
     823            unsigned int n;
     824            _thread_stack_pcs(reinterpret_cast<vm_address_t *>(stack.data()), stack.capacity(), &n, 1);
     825            if (LLVM_UNLIKELY(n < stack.capacity() || stack[n - 1] == 0)) {
     826                while (n >= 1 && stack[n - 1] == 0) {
     827                    n -= 1;
     828                }
     829                stack.set_size(n);
     830                break;
     831            }
     832            stack.reserve(n * 2);
     833        }
     834        #elif defined(HAS_LIBUNWIND)
    717835        unw_context_t context;
    718836        // Initialize cursor to current frame for local unwinding.
     
    727845                break;
    728846            }
    729             stack.push_back(pc);
     847            stack.push_back(static_cast<uint64_t>(pc));
    730848        }
    731849        #elif defined(HAS_EXECINFO)
     
    739857        }
    740858        #endif
    741         IntegerType * const stackTy = TypeBuilder<unw_word_t, false>::get(getContext());
    742         PointerType * const stackPtrTy = stackTy->getPointerTo();
    743         GlobalVariable * ip_trace = nullptr;
    744         const auto n = stack.size();
    745         for (GlobalVariable & gv : m->getGlobalList()) {
    746             Type * const ty = gv.getValueType();
    747             if (ty->isArrayTy() && ty->getArrayElementType() == stackTy && ty->getArrayNumElements() == n) {
    748                 const ConstantDataArray * const array = cast<ConstantDataArray>(gv.getOperand(0));
    749                 bool found = true;
    750                 for (auto i = n - 1; i != 0; --i) {
    751                     if (LLVM_LIKELY(array->getElementAsInteger(i) != stack[i])) {
    752                         found = false;
     859        Value * trace = nullptr;
     860        ConstantInt * depth = nullptr;
     861        if (stack.empty()) {
     862            trace = ConstantPointerNull::get(stackPtrTy);
     863            depth = getInt32(0);
     864        } else {
     865            const auto n = stack.size() - 1;
     866            for (GlobalVariable & gv : m->getGlobalList()) {
     867                Type * const ty = gv.getValueType();
     868                if (ty->isArrayTy() && ty->getArrayElementType() == stackTy && ty->getArrayNumElements() == n) {
     869                    const ConstantDataArray * const array = cast<ConstantDataArray>(gv.getOperand(0));
     870                    bool found = true;
     871                    for (size_t i = 0; i < n; ++i) {
     872                        if (LLVM_LIKELY(array->getElementAsInteger(i) != stack[i + 1])) {
     873                            found = false;
     874                            break;
     875                        }
     876                    }
     877                    if (LLVM_UNLIKELY(found)) {
     878                        trace = &gv;
    753879                        break;
    754880                    }
    755881                }
    756                 if (LLVM_UNLIKELY(found)) {
    757                     ip_trace = &gv;
    758                     break;
    759                 }
    760882            }
    761         }
    762         if (LLVM_LIKELY(ip_trace == nullptr)) {
    763             Constant * const initializer = ConstantDataArray::get(getContext(), stack);
    764             ip_trace = new GlobalVariable(*m, initializer->getType(), true, GlobalVariable::InternalLinkage, initializer);
    765         }
    766         Value * const trace = CreatePointerCast(ip_trace, stackPtrTy);
    767         IRBuilder<>::CreateCall(function, {assertion, GetString(failureMessage), trace, getInt32(n)});
     883            if (LLVM_LIKELY(trace == nullptr)) {
     884                Constant * const initializer = ConstantDataArray::get(getContext(), ArrayRef<uint64_t>(stack.data() + 1, n));
     885                trace = new GlobalVariable(*m, initializer->getType(), true, GlobalVariable::InternalLinkage, initializer);
     886            }
     887            trace = CreatePointerCast(trace, stackPtrTy);
     888            depth = getInt32(n);
     889        }
     890        IRBuilder<>::CreateCall(function, {assertion, GetString(failureMessage), trace, depth});
    768891    }
    769892}
     
    833956    CreateAssert(value, "CreateCeilLog2: value cannot be zero");
    834957    Value * m = CreateCountReverseZeroes(CreateSub(value, ConstantInt::get(ty, 1)));
    835     return CreateSub(ConstantInt::get(m->getType(), ty->getBitWidth() - 1), m);
     958    return CreateSub(ConstantInt::get(m->getType(), ty->getBitWidth()), m);
    836959}
    837960
     
    860983
    861984#define CHECK_ADDRESS(Ptr) \
    862     if (codegen::EnableAsserts) { \
    863         Function * const isPoisoned = LinkFunction("__asan_region_is_poisoned", __asan_region_is_poisoned); \
    864         auto arg = isPoisoned->arg_begin(); \
    865         Value * const addr = CreatePointerCast(Ptr, arg->getType()); \
    866         Constant * const size = ConstantInt::get((++arg)->getType(), Ptr->getType()->getPointerElementType()->getPrimitiveSizeInBits() / 8); \
     985    if (LLVM_UNLIKELY(hasAddressSanitizer())) { \
     986        Module * const m = getModule(); \
     987        PointerType * const voidPtrTy = getVoidPtrTy(); \
     988        IntegerType * const sizeTy = getSizeTy(); \
     989        Function * isPoisoned = m->getFunction("__asan_region_is_poisoned"); \
     990        if (LLVM_UNLIKELY(isPoisoned == nullptr)) { \
     991            isPoisoned = Function::Create(FunctionType::get(voidPtrTy, {voidPtrTy, sizeTy}, false), Function::ExternalLinkage, "__asan_region_is_poisoned", m); \
     992            isPoisoned->setCallingConv(CallingConv::C); \
     993            isPoisoned->setDoesNotAlias(0); \
     994            isPoisoned->setDoesNotAlias(1); \
     995        } \
     996        Value * const addr = CreatePointerCast(Ptr, voidPtrTy); \
     997        ConstantInt * const size = ConstantInt::get(sizeTy, Ptr->getType()->getPointerElementType()->getPrimitiveSizeInBits() / 8); \
    867998        Value * check = CreateCall(isPoisoned, { addr, size }); \
    868999        check = CreateICmpEQ(check, ConstantPointerNull::get(cast<PointerType>(isPoisoned->getReturnType()))); \
     
    8981029
    8991030#endif
     1031
     1032inline bool CBuilder::hasAddressSanitizer() const {
     1033    return codegen::EnableAsserts && mDriver && mDriver->hasExternalFunction("__asan_region_is_poisoned");
     1034}
     1035
     1036LoadInst * CBuilder::CreateAlignedLoad(Value * Ptr, unsigned Align, const char * Name) {
     1037    if (codegen::EnableAsserts) {
     1038        DataLayout DL(getModule());
     1039        IntegerType * const intPtrTy = cast<IntegerType>(DL.getIntPtrType(Ptr->getType()));
     1040        Value * alignmentOffset = CreateURem(CreatePtrToInt(Ptr, intPtrTy), ConstantInt::get(intPtrTy, Align));
     1041        CreateAssertZero(alignmentOffset, "CreateAlignedLoad: pointer is misaligned");
     1042    }
     1043    LoadInst * LI = CreateLoad(Ptr, Name);
     1044    LI->setAlignment(Align);
     1045    return LI;
     1046}
     1047
     1048LoadInst * CBuilder::CreateAlignedLoad(Value * Ptr, unsigned Align, const Twine & Name) {
     1049    if (codegen::EnableAsserts) {
     1050        DataLayout DL(getModule());
     1051        IntegerType * const intPtrTy = cast<IntegerType>(DL.getIntPtrType(Ptr->getType()));
     1052        Value * alignmentOffset = CreateURem(CreatePtrToInt(Ptr, intPtrTy), ConstantInt::get(intPtrTy, Align));
     1053        CreateAssertZero(alignmentOffset, "CreateAlignedLoad: pointer is misaligned");
     1054    }
     1055    LoadInst * LI = CreateLoad(Ptr, Name);
     1056    LI->setAlignment(Align);
     1057    return LI;
     1058}
     1059
     1060LoadInst * CBuilder::CreateAlignedLoad(Value * Ptr, unsigned Align, bool isVolatile, const Twine & Name) {
     1061    if (codegen::EnableAsserts) {
     1062        DataLayout DL(getModule());
     1063        IntegerType * const intPtrTy = cast<IntegerType>(DL.getIntPtrType(Ptr->getType()));
     1064        Value * alignmentOffset = CreateURem(CreatePtrToInt(Ptr, intPtrTy), ConstantInt::get(intPtrTy, Align));
     1065        CreateAssertZero(alignmentOffset, "CreateAlignedLoad: pointer is misaligned");
     1066    }
     1067    LoadInst * LI = CreateLoad(Ptr, isVolatile, Name);
     1068    LI->setAlignment(Align);
     1069    return LI;
     1070}
     1071
     1072StoreInst * CBuilder::CreateAlignedStore(Value * Val, Value * Ptr, unsigned Align, bool isVolatile) {
     1073    if (codegen::EnableAsserts) {
     1074        DataLayout DL(getModule());
     1075        IntegerType * const intPtrTy = cast<IntegerType>(DL.getIntPtrType(Ptr->getType()));
     1076        Value * alignmentOffset = CreateURem(CreatePtrToInt(Ptr, intPtrTy), ConstantInt::get(intPtrTy, Align));
     1077        CreateAssertZero(alignmentOffset, "CreateAlignedStore: pointer is misaligned");
     1078    }
     1079    StoreInst *SI = CreateStore(Val, Ptr, isVolatile);
     1080    SI->setAlignment(Align);
     1081    return SI;
     1082}
    9001083
    9011084CBuilder::CBuilder(LLVMContext & C)
  • icGREP/icgrep-devel/icgrep/IR_Gen/CBuilder.h

    r5489 r5493  
    2525
    2626class CBuilder : public llvm::IRBuilder<> {
     27    using Predicate = llvm::CmpInst::Predicate;
    2728public:
    2829
     
    177178    virtual llvm::StoreInst *  CreateAtomicStoreRelease(llvm::Value * val, llvm::Value * ptr);
    178179
    179     void CreateAssert(llvm::Value * assertion, llvm::StringRef failureMessage);
     180    void CreateAssert(llvm::Value * assertion, llvm::StringRef failureMessage) {
     181        if (LLVM_UNLIKELY(assertion->getType()->isVectorTy())) {
     182            assertion = CreateBitCast(assertion, llvm::IntegerType::get(getContext(), assertion->getType()->getPrimitiveSizeInBits()));
     183        }
     184        return __CreateAssert(CreateICmpNE(assertion, llvm::Constant::getNullValue(assertion->getType())), failureMessage);
     185    }
     186
     187    void CreateAssertZero(llvm::Value * assertion, llvm::StringRef failureMessage) {
     188        if (LLVM_UNLIKELY(assertion->getType()->isVectorTy())) {
     189            assertion = CreateBitCast(assertion, llvm::IntegerType::get(getContext(), assertion->getType()->getPrimitiveSizeInBits()));
     190        }
     191        return __CreateAssert(CreateICmpEQ(assertion, llvm::Constant::getNullValue(assertion->getType())), failureMessage);
     192    }
    180193
    181194    void CreateExit(const int exitCode);
     
    218231
    219232    #ifdef HAS_ADDRESS_SANITIZER
    220     virtual llvm::LoadInst * CreateLoad(llvm::Value *Ptr, const char *Name);
    221 
    222     virtual llvm::LoadInst * CreateLoad(llvm::Value *Ptr, const llvm::Twine &Name = "");
    223 
    224     virtual llvm::LoadInst * CreateLoad(llvm::Type *Ty, llvm::Value *Ptr, const llvm::Twine &Name = "");
    225 
    226     virtual llvm::LoadInst * CreateLoad(llvm::Value *Ptr, bool isVolatile, const llvm::Twine &Name = "");
    227 
    228     virtual llvm::StoreInst * CreateStore(llvm::Value *Val, llvm::Value *Ptr, bool isVolatile = false);
    229 
    230     llvm::LoadInst * CreateAlignedLoad(llvm::Value *Ptr, unsigned Align, const char *Name) {
    231         llvm::LoadInst * LI = CreateLoad(Ptr, Name);
    232         LI->setAlignment(Align);
    233         return LI;
    234     }
    235 
    236     llvm::LoadInst * CreateAlignedLoad(llvm::Value *Ptr, unsigned Align, const llvm::Twine &Name = "") {
    237         llvm::LoadInst * LI = CreateLoad(Ptr, Name);
    238         LI->setAlignment(Align);
    239         return LI;
    240     }
    241 
    242     llvm::LoadInst * CreateAlignedLoad(llvm::Value *Ptr, unsigned Align, bool isVolatile, const llvm::Twine &Name = "") {
    243         llvm::LoadInst * LI = CreateLoad(Ptr, isVolatile, Name);
    244         LI->setAlignment(Align);
    245         return LI;
    246     }
    247 
    248     llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Ptr, unsigned Align, bool isVolatile = false) {
    249         llvm::StoreInst *SI = CreateStore(Val, Ptr, isVolatile);
    250         SI->setAlignment(Align);
    251         return SI;
    252     }
     233    virtual llvm::LoadInst * CreateLoad(llvm::Value * Ptr, const char * Name);
     234
     235    virtual llvm::LoadInst * CreateLoad(llvm::Value * Ptr, const llvm::Twine & Name = "");
     236
     237    virtual llvm::LoadInst * CreateLoad(llvm::Type * Ty, llvm::Value * Ptr, const llvm::Twine & Name = "");
     238
     239    virtual llvm::LoadInst * CreateLoad(llvm::Value * Ptr, bool isVolatile, const llvm::Twine & Name = "");
     240
     241    virtual llvm::StoreInst * CreateStore(llvm::Value * Val, llvm::Value * Ptr, bool isVolatile = false);
    253242    #endif
     243
     244    llvm::LoadInst * CreateAlignedLoad(llvm::Value * Ptr, unsigned Align, const char * Name);
     245
     246    llvm::LoadInst * CreateAlignedLoad(llvm::Value * Ptr, unsigned Align, const llvm::Twine & Name = "");
     247
     248    llvm::LoadInst * CreateAlignedLoad(llvm::Value * Ptr, unsigned Align, bool isVolatile, const llvm::Twine & Name = "");
     249
     250    llvm::StoreInst * CreateAlignedStore(llvm::Value * Val, llvm::Value * Ptr, unsigned Align, bool isVolatile = false);
    254251
    255252    void setDriver(Driver * const driver) {
     
    258255
    259256protected:
     257
     258    bool hasAlignedAlloc() const;
     259
     260    bool hasPosixMemalign() const;
     261
     262    bool hasAddressSanitizer() const;
     263
     264    void __CreateAssert(llvm::Value * assertion, llvm::StringRef failureMessage);
    260265
    261266    llvm::Function * LinkFunction(llvm::StringRef name, llvm::FunctionType * type, void * functionPtr) const;
     
    267272    llvm::IntegerType * const       mSizeType;
    268273    llvm::StructType *              mFILEtype;
    269     Driver *                        mDriver;
     274    Driver *                        mDriver;   
    270275    llvm::LLVMContext               mContext;
    271276    const std::string               mTriple;
  • icGREP/icgrep-devel/icgrep/IR_Gen/idisa_builder.cpp

    r5489 r5493  
    372372}
    373373
    374 LoadInst * IDISA_Builder::CreateBlockAlignedLoad(Value * const ptr) {
    375     const auto alignment = mBitBlockWidth / 8;
    376     if (codegen::EnableAsserts) {
    377         DataLayout DL(getModule());
    378         IntegerType * const intPtrTy = getIntPtrTy(DL);
    379         Value * alignmentOffset = CreateURem(CreatePtrToInt(ptr, intPtrTy), ConstantInt::get(intPtrTy, alignment));
    380         Value * alignmentCheck = CreateICmpEQ(alignmentOffset, ConstantInt::getNullValue(intPtrTy));
    381         CreateAssert(alignmentCheck, "CreateBlockAlignedLoad: pointer is unaligned");
    382     }
    383     return CreateAlignedLoad(ptr, alignment);
    384 }
    385 
    386 void IDISA_Builder::CreateBlockAlignedStore(Value * const value, Value * const ptr) {
    387     const auto alignment = mBitBlockWidth / 8;
    388     if (codegen::EnableAsserts) {
    389         DataLayout DL(getModule());
    390         IntegerType * const intPtrTy = getIntPtrTy(DL);
    391         Value * alignmentOffset = CreateURem(CreatePtrToInt(ptr, intPtrTy), ConstantInt::get(intPtrTy, alignment));
    392         Value * alignmentCheck = CreateICmpEQ(alignmentOffset, ConstantInt::getNullValue(intPtrTy));
    393         CreateAssert(alignmentCheck, "CreateBlockAlignedStore: pointer is not aligned");
    394     }
    395     CreateAlignedStore(value, ptr, alignment);
    396 }
    397 
    398374IDISA_Builder::IDISA_Builder(llvm::LLVMContext & C, unsigned vectorWidth, unsigned stride)
    399375: CBuilder(C)
  • icGREP/icgrep-devel/icgrep/IR_Gen/idisa_builder.h

    r5489 r5493  
    4646    }
    4747   
    48     llvm::LoadInst * CreateBlockAlignedLoad(llvm::Value * const ptr);
    49     llvm::LoadInst * CreateBlockAlignedLoad(llvm::Value * const ptr, llvm::Value * const index);
    50     llvm::LoadInst * CreateBlockAlignedLoad(llvm::Value * const ptr, std::initializer_list<llvm::Value *> indices);
     48    llvm::LoadInst * CreateBlockAlignedLoad(llvm::Value * const ptr) {
     49        return CreateAlignedLoad(ptr, mBitBlockWidth / 8);
     50    }
    5151
    52     void CreateBlockAlignedStore(llvm::Value * const value, llvm::Value * const ptr);
    53     void CreateBlockAlignedStore(llvm::Value * const value, llvm::Value * const ptr, llvm::Value * const index);
    54     void CreateBlockAlignedStore(llvm::Value * const value, llvm::Value * const ptr, std::initializer_list<llvm::Value *> indices);
     52    llvm::LoadInst * CreateBlockAlignedLoad(llvm::Value * const ptr, llvm::Value * const index) {
     53        return CreateBlockAlignedLoad(CreateGEP(ptr, index));
     54    }
     55
     56    llvm::LoadInst * CreateBlockAlignedLoad(llvm::Value * const ptr, std::initializer_list<llvm::Value *> indices) {
     57        return CreateBlockAlignedLoad(CreateGEP(ptr, indices));
     58    }
     59
     60    llvm::StoreInst * CreateBlockAlignedStore(llvm::Value * const value, llvm::Value * const ptr) {
     61        return CreateAlignedStore(value, ptr, mBitBlockWidth / 8);
     62    }
     63
     64    llvm::StoreInst * CreateBlockAlignedStore(llvm::Value * const value, llvm::Value * const ptr, llvm::Value * const index) {
     65        return CreateBlockAlignedStore(value, CreateGEP(ptr, index));
     66    }
     67
     68    llvm::StoreInst * CreateBlockAlignedStore(llvm::Value * const value, llvm::Value * const ptr, std::initializer_list<llvm::Value *> indices) {
     69        return CreateBlockAlignedStore(value, CreateGEP(ptr, indices));
     70    }
     71
     72    llvm::Value * CreateBlockAlignedMalloc(llvm::Value * size) {
     73        return CreateAlignedMalloc(size, mBitBlockWidth / 8);
     74    }
    5575
    5676    llvm::VectorType * fwVectorType(const unsigned fw);
     
    145165    llvm::Constant * const      mZeroInitializer;
    146166    llvm::Constant * const      mOneInitializer;
    147 
    148167    llvm::Constant *            mPrintRegisterFunction;
    149168};
    150169
    151 inline llvm::LoadInst * IDISA_Builder::CreateBlockAlignedLoad(llvm::Value * const ptr, llvm::Value * const index) {
    152     return CreateBlockAlignedLoad(CreateGEP(ptr, index));
    153 }
    154 
    155 inline llvm::LoadInst * IDISA_Builder::CreateBlockAlignedLoad(llvm::Value * const ptr, std::initializer_list<llvm::Value *> indices) {
    156     return CreateBlockAlignedLoad(CreateGEP(ptr, indices));
    157 }
    158 
    159 inline void IDISA_Builder::CreateBlockAlignedStore(llvm::Value * const value, llvm::Value * const ptr, llvm::Value * const index) {
    160     CreateBlockAlignedStore(value, CreateGEP(ptr, index));
    161 }
    162 
    163 inline void IDISA_Builder::CreateBlockAlignedStore(llvm::Value * const value, llvm::Value * const ptr, std::initializer_list<llvm::Value *> indices) {
    164     CreateBlockAlignedStore(value, CreateGEP(ptr, indices));
    165 }
    166 
    167170}
    168171#endif // IDISA_BUILDER_H
  • icGREP/icgrep-devel/icgrep/grep_engine.cpp

    r5491 r5493  
    3636#include <errno.h>
    3737#include <mutex>
    38 
    3938#ifdef CUDA_ENABLED
    4039#include <preprocess.cpp>
     
    8382    pthread_exit(nullptr);
    8483}
    85 
    86 
    8784
    8885void GrepEngine::doGrep(const std::string & fileName) const{
  • icGREP/icgrep-devel/icgrep/kernels/kernel.cpp

    r5486 r5493  
    11301130               std::vector<Binding> && scalar_outputs,
    11311131               std::vector<Binding> && internal_scalars)
    1132 : KernelInterface(std::move(annotateKernelNameWithDebugFlags(std::move(kernelName)))
     1132: KernelInterface(annotateKernelNameWithDebugFlags(std::move(kernelName))
    11331133                  , std::move(stream_inputs), std::move(stream_outputs)
    11341134                  , std::move(scalar_parameters), std::move(scalar_outputs)
  • icGREP/icgrep-devel/icgrep/kernels/kernel_builder.cpp

    r5446 r5493  
    187187}
    188188
    189 void KernelBuilder::storeOutputStreamBlock(const std::string & name, Value * streamIndex, Value * toStore) {
     189StoreInst * KernelBuilder::storeOutputStreamBlock(const std::string & name, Value * streamIndex, Value * toStore) {
    190190    return CreateBlockAlignedStore(toStore, getOutputStreamBlockPtr(name, streamIndex));
    191191}
     
    197197}
    198198
    199 void KernelBuilder::storeOutputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex, Value * toStore) {
     199StoreInst * KernelBuilder::storeOutputStreamPack(const std::string & name, Value * streamIndex, Value * packIndex, Value * toStore) {
    200200    return CreateBlockAlignedStore(toStore, getOutputStreamPackPtr(name, streamIndex, packIndex));
    201201}
  • icGREP/icgrep-devel/icgrep/kernels/kernel_builder.h

    r5489 r5493  
    6666    llvm::Value * getOutputStreamBlockPtr(const std::string & name, llvm::Value * streamIndex);
    6767
    68     void storeOutputStreamBlock(const std::string & name, llvm::Value * streamIndex, llvm::Value * toStore);
     68    llvm::StoreInst * storeOutputStreamBlock(const std::string & name, llvm::Value * streamIndex, llvm::Value * toStore);
    6969
    7070    llvm::Value * getOutputStreamPackPtr(const std::string & name, llvm::Value * streamIndex, llvm::Value * packIndex);
    7171
    72     void storeOutputStreamPack(const std::string & name, llvm::Value * streamIndex, llvm::Value * packIndex, llvm::Value * toStore);
     72    llvm::StoreInst * storeOutputStreamPack(const std::string & name, llvm::Value * streamIndex, llvm::Value * packIndex, llvm::Value * toStore);
    7373
    7474    llvm::Value * getOutputStreamSetCount(const std::string & name);
  • icGREP/icgrep-devel/icgrep/kernels/lz4_index_decoder.cpp

    r5440 r5493  
    7171 */
    7272Value * LZ4IndexDecoderKernel::getWordOffset(const std::unique_ptr<kernel::KernelBuilder> & iBuilder) {
    73     Value * wordWidthMask = iBuilder->getInt32(wordWidth - 1);
    74     return iBuilder->CreateAnd(
    75             iBuilder->CreateLoad(sOffset),
    76             wordWidthMask
    77             );
    78 }
    79 
     73    Value * offset = iBuilder->CreateLoad(sOffset);
     74    IntegerType * type = cast<IntegerType>(offset->getType());
     75    Constant * mask = ConstantInt::get(type, wordWidth - 1);
     76    return iBuilder->CreateAnd(offset, mask);
     77}
    8078
    8179/**
     
    8381 */
    8482Value * LZ4IndexDecoderKernel::getWordStartOffset(const std::unique_ptr<KernelBuilder> & iBuilder) {
    85     Value * wordWidthMask = iBuilder->getInt32(wordWidth - 1);
    86     return iBuilder->CreateAnd(
    87             iBuilder->CreateLoad(sOffset),
    88             iBuilder->CreateNot(wordWidthMask)
    89             );
    90 }
    91 
     83    Value * offset = iBuilder->CreateLoad(sOffset);
     84    IntegerType * type = cast<IntegerType>(offset->getType());
     85    Constant * mask = ConstantExpr::getNeg(ConstantInt::get(type, wordWidth));
     86    return iBuilder->CreateAnd(offset, mask);
     87}
    9288
    9389/**
     
    136132 */
    137133void LZ4IndexDecoderKernel::loadCurrentExtender(const std::unique_ptr<KernelBuilder> & iBuilder) {
    138     iBuilder->CreateStore(
    139             iBuilder->CreateExtractElement(extenders,
    140                 iBuilder->CreateLShr(
    141                     iBuilder->CreateLoad(sOffset),
    142                     iBuilder->getInt32(std::log2(wordWidth))
    143                     )
    144                 ),
    145             sExtender);
     134    Value * offset = iBuilder->CreateLoad(sOffset);
     135    IntegerType * type = cast<IntegerType>(offset->getType());
     136    ConstantInt * shift = ConstantInt::get(type, std::log2(wordWidth));
     137    Value * shiftedOffset = iBuilder->CreateLShr(offset, shift);
     138    Value * extender = iBuilder->CreateExtractElement(extenders, shiftedOffset);
     139    iBuilder->CreateStore(extender, sExtender);
    146140}
    147141
     
    195189    extenders = iBuilder->CreateBitCast(
    196190            iBuilder->loadInputStreamBlock("extenders", iBuilder->getInt32(0)),
    197             VectorType::get(iBuilder->getSizeTy(), iBuilder->getBitBlockWidth() / wordWidth),
     191            VectorType::get(iBuilder->getSizeTy(), iBuilder->getBitBlockWidth() / iBuilder->getSizeTy()->getBitWidth()),
    198192            "extenders");
    199193    // Create a series of stack variables which will be promoted by mem2reg.
  • icGREP/icgrep-devel/icgrep/kernels/source_kernel.cpp

    r5474 r5493  
    66#include <kernels/kernel_builder.h>
    77#include <kernels/streamset.h>
     8#include <llvm/IR/Module.h>
    89#include <sys/stat.h>
    910#include <fcntl.h>
     
    208209
    209210    Value * L = iBuilder->CreateGEP(originalPtr, pageSize);
    210 
    211 //    iBuilder->CallPrintInt("L", L);
    212 
    213211    Value * B = iBuilder->CreateGEP(buffer, capacity);
    214 
    215 //    iBuilder->CallPrintInt("B", B);
    216 
    217212    Value * const canAppend = iBuilder->CreateICmpULT(L, B);
    218213    iBuilder->CreateLikelyCondBr(canAppend, readData, waitOnConsumers);
     
    224219    // Then determine how much data has been consumed and how much needs to be copied back, noting
    225220    // that our "unproduced" data must be block aligned.
    226     const auto blockAlignment = iBuilder->getBitBlockWidth() / 8;
    227     Constant * const alignmentMask = ConstantExpr::getNot(iBuilder->getSize(blockAlignment - 1));
     221    const size_t blockAlignment = iBuilder->getBitBlockWidth() / 8;
     222    Constant * const alignmentMask = iBuilder->getSize(-blockAlignment);
    228223    Value * const consumed = iBuilder->CreateAnd(iBuilder->getConsumedItemCount("sourceBuffer"), alignmentMask);
    229224    Value * const remaining = iBuilder->CreateSub(bufferedSize, consumed);
     
    231226    Value * const consumedMajority = iBuilder->CreateICmpULT(iBuilder->CreateGEP(buffer, remaining), unconsumedPtr);
    232227
    233 //    iBuilder->CallPrintInt("consumedMajority", consumedMajority);
    234 
    235228    BasicBlock * const copyBack = iBuilder->CreateBasicBlock("CopyBack");
    236229    BasicBlock * const expandAndCopyBack = iBuilder->CreateBasicBlock("ExpandAndCopyBack");
    237230    BasicBlock * const calculateLogicalAddress = iBuilder->CreateBasicBlock("CalculateLogicalAddress");
     231
    238232    // Have we consumed enough data that we can safely copy back the unconsumed data without needing
    239233    // a temporary buffer? (i.e., B + remaining < L + consumed)
    240234    iBuilder->CreateLikelyCondBr(consumedMajority, copyBack, expandAndCopyBack);
    241235    iBuilder->SetInsertPoint(copyBack);
     236
    242237    // If so, just copy the data ...
    243     iBuilder->CreateMemCpy(buffer, unconsumedPtr, remaining, blockAlignment);
     238    iBuilder->CreateMemCpy(buffer, unconsumedPtr, remaining, 1);
    244239    iBuilder->CreateBr(calculateLogicalAddress);
     240
    245241    // Otherwise, allocate a buffer with twice the capacity and copy the unconsumed data back into it
    246242    iBuilder->SetInsertPoint(expandAndCopyBack);
    247243    Value * const expandedCapacity = iBuilder->CreateShl(capacity, 1);
    248     Value * const expandedBuffer = iBuilder->CreateCacheAlignedMalloc(expandedCapacity);
    249     Value * const expandedPtr = iBuilder->CreatePointerCast(expandedBuffer, codeUnitPtrTy);
    250     iBuilder->CreateMemCpy(expandedPtr, unconsumedPtr, remaining, blockAlignment);
     244    Value * const expandedBuffer = iBuilder->CreatePointerCast(iBuilder->CreateCacheAlignedMalloc(expandedCapacity), codeUnitPtrTy);
     245    iBuilder->CreateMemCpy(expandedBuffer, unconsumedPtr, remaining, 1);
    251246    iBuilder->CreateFree(buffer);
    252247    iBuilder->setScalarField("buffer", expandedBuffer);
    253248    iBuilder->setScalarField("capacity", expandedCapacity);
    254249    iBuilder->CreateBr(calculateLogicalAddress);
     250
    255251    // Update the logical address for this buffer....
    256252    iBuilder->SetInsertPoint(calculateLogicalAddress);
    257253    PHINode * const baseAddress = iBuilder->CreatePHI(codeUnitPtrTy, 2);
    258254    baseAddress->addIncoming(buffer, copyBack);
    259     baseAddress->addIncoming(expandedPtr, expandAndCopyBack);
     255    baseAddress->addIncoming(expandedBuffer, expandAndCopyBack);
    260256    Value * const modifiedPtr = iBuilder->CreateGEP(baseAddress, remaining);
    261     Value * const logicalAddress = iBuilder->CreateGEP(modifiedPtr, iBuilder->CreateNeg(produced));
     257    Value * const logicalAddress = iBuilder->CreateGEP(modifiedPtr, iBuilder->CreateNeg(iBuilder->CreateAnd(produced, alignmentMask)));
    262258    iBuilder->setBaseAddress("sourceBuffer", logicalAddress);
    263259    iBuilder->CreateBr(readData);
     260
    264261    // Regardless of whether we're simply appending data or had to allocate a new buffer, read a new page
    265262    // of data into the input source buffer. If we fail to read a full segment ...
  • icGREP/icgrep-devel/icgrep/kernels/streamset.cpp

    r5486 r5493  
    150150    Type * i8ptr = iBuilder->getInt8PtrTy();
    151151    unsigned alignment = iBuilder->getBitBlockWidth() / 8;
    152     unsigned numStreams = getType()->getArrayNumElements();
    153     auto elemTy = getType()->getArrayElementType();
    154     unsigned fieldWidth = isa<ArrayType>(elemTy) ? elemTy->getArrayNumElements() : 1;
     152    uint64_t numStreams = 1;
     153    if (isa<ArrayType>(mBaseType)) {
     154        numStreams = mBaseType->getArrayNumElements();
     155    }
     156    const auto fieldWidth = mBaseType->getArrayElementType()->getScalarSizeInBits();
    155157    Value * blockCopyBytes = iBuilder->CreateMul(blocksToCopy, iBuilder->getSize(iBuilder->getBitBlockWidth() * numStreams * fieldWidth/8));
    156158    iBuilder->CreateMemMove(iBuilder->CreateBitCast(targetBlockPtr, i8ptr), iBuilder->CreateBitCast(sourceBlockPtr, i8ptr), blockCopyBytes, alignment);
     
    161163    const unsigned alignment = iBuilder->getBitBlockWidth() / 8;
    162164    Constant * const blockSize = iBuilder->getSize(iBuilder->getBitBlockWidth());
    163     const unsigned numStreams = getType()->getArrayNumElements();
    164     const auto elemTy = getType()->getArrayElementType();
    165     const auto fieldWidth = isa<ArrayType>(elemTy) ? elemTy->getArrayNumElements() : 1;
     165    uint64_t numStreams = 1;
     166    if (isa<ArrayType>(mBaseType)) {
     167        numStreams = mBaseType->getArrayNumElements();
     168    }
     169    const auto fieldWidth = mBaseType->getArrayElementType()->getScalarSizeInBits();
    166170    if (numStreams == 1) {
    167171        Value * copyBits = iBuilder->CreateMul(itemsToCopy, iBuilder->getSize(fieldWidth));
  • icGREP/icgrep-devel/icgrep/kernels/until_n.cpp

    r5451 r5493  
    7979    PHINode * blockGroupBase = kb->CreatePHI(kb->getSizeTy(), 2);
    8080    blockGroupBase->addIncoming(kb->getSize(0), entry);
    81     Value * groupPackPtr = kb->CreatePointerCast(kb->CreateGEP(sourceBitstream, {blockGroupBase}), iPackPtrTy);
     81    Value * groupPackPtr = kb->CreatePointerCast(kb->CreateGEP(sourceBitstream, blockGroupBase), iPackPtrTy);
    8282    Value * blockGroupLimit = kb->CreateAdd(blockGroupBase, blocksPerGroup);
    8383    blockGroupLimit = kb->CreateSelect(kb->CreateICmpULT(blockGroupLimit, blocksToDo), blockGroupLimit, blocksToDo);
     
    114114    seenSoFarPhi->addIncoming(seenSoFar, doScan);
    115115    Value * nonZeroPack = kb->CreateZExtOrTrunc(kb->CreateCountForwardZeroes(groupMaskPhi), kb->getSizeTy());
    116     Value * scanMask = kb->CreateLoad(kb->CreateGEP(groupPackPtr, {nonZeroPack}));
     116    Value * scanMask = kb->CreateLoad(kb->CreateGEP(groupPackPtr, nonZeroPack));
    117117    Value * packCount = kb->CreateZExtOrTrunc(kb->CreatePopcount(scanMask), kb->getSizeTy());
    118118    Value * newTotalSeen = kb->CreateAdd(packCount, seenSoFarPhi);
  • icGREP/icgrep-devel/icgrep/lz4d.cpp

    r5486 r5493  
    4444static cl::opt<bool> overwriteOutput("f", cl::desc("Overwrite existing output file."), cl::init(false), cl::cat(lz4dFlags));
    4545
    46 
    4746typedef void (*MainFunctionType)(char * byte_data, size_t filesize, bool hasBlockChecksum);
    4847
     
    5150    Module * M = iBuilder->getModule();
    5251
    53     Type * const size_ty = iBuilder->getSizeTy();
    54     Type * const bool_ty = iBuilder->getIntNTy(sizeof(bool) * 8);
     52    Type * const sizeTy = iBuilder->getSizeTy();
     53    Type * const boolTy = iBuilder->getIntNTy(sizeof(bool) * 8);
    5554    Type * const voidTy = iBuilder->getVoidTy();
    5655    Type * const inputType = iBuilder->getInt8PtrTy();
    5756   
    58     Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, size_ty, bool_ty, nullptr));
     57    Function * const main = cast<Function>(M->getOrInsertFunction("Main", voidTy, inputType, sizeTy, boolTy, nullptr));
    5958    main->setCallingConv(CallingConv::C);
    6059    Function::arg_iterator args = main->arg_begin();
     
    7069    // Output buffer should be at least one whole LZ4 block (4MB) large in case of uncompressed blocks.
    7170    // And the size (in bytes) also needs to be a power of two.
    72     const unsigned decompressBufBlocks = 4U * 1024 * 1024 / codegen::BlockSize;
     71    const unsigned decompressBufBlocks = (4194304U) / codegen::BlockSize;
    7372
    7473    iBuilder->SetInsertPoint(BasicBlock::Create(M->getContext(), "entry", main, 0));
     
    111110}
    112111
    113 
    114 MainFunctionType codeGen() {
    115     ParabixDriver pxDriver("lz4d");
    116     generatePipeline(pxDriver);
    117     return reinterpret_cast<MainFunctionType>(pxDriver.getMain());
    118 }
    119 
    120 
    121112int main(int argc, char *argv[]) {
    122113    // This boilerplate provides convenient stack traces and clean LLVM exit
     
    148139    char *fileBuffer = const_cast<char *>(mappedFile.data()) + lz4Frame.getBlocksStart();
    149140
     141    if (codegen::SegmentSize < 2) {
     142        codegen::SegmentSize = 2;
     143    }
     144
    150145    ParabixDriver pxDriver("lz4d");
    151146    generatePipeline(pxDriver);
  • icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp

    r5486 r5493  
    1616#include <pablo/pe_var.h>
    1717#include <kernels/kernel_builder.h>
    18 #include <llvm/Support/raw_ostream.h>
     18#include <toolchain/toolchain.h>
     19#include <array>
    1920
    2021using namespace llvm;
     
    2425inline static unsigned ceil_log2(const unsigned v) {
    2526    assert ("log2(0) is undefined!" && v != 0);
    26     return 32 - __builtin_clz(v - 1);
     27    return (sizeof(unsigned) * CHAR_BIT) - __builtin_clz(v - 1);
    2728}
    2829
    2930inline static unsigned floor_log2(const unsigned v) {
    3031    assert ("log2(0) is undefined!" && v != 0);
    31     return 31 - __builtin_clz(v);
     32    return ((sizeof(unsigned) * CHAR_BIT) - 1) - __builtin_clz(v);
    3233}
    3334
     
    8889        kernel->addScalar(iBuilder->getSizeTy(), "CarryBlockIndex");
    8990    }
    90 
     91}
     92
     93bool isDynamicallyAllocatedType(const Type * const ty) {
     94    if (isa<StructType>(ty) && ty->getStructNumElements() == 3) {
     95        return (ty->getStructElementType(1)->isPointerTy() && ty->getStructElementType(2)->isPointerTy() && ty->getStructElementType(0)->isIntegerTy());
     96    }
     97    return false;
     98}
     99
     100bool containsDynamicallyAllocatedType(const Type * const ty) {
     101    if (isa<StructType>(ty)) {
     102        for (unsigned i = 0; i < ty->getStructNumElements(); ++i) {
     103            if (isDynamicallyAllocatedType(ty->getStructElementType(i))) {
     104                return true;
     105            }
     106        }
     107    }
     108    return false;
     109}
     110
     111void freeDynamicallyAllocatedMemory(const std::unique_ptr<kernel::KernelBuilder> & idb, Value * const frame) {
     112    StructType * const ty = cast<StructType>(frame->getType()->getPointerElementType());
     113    std::array<Value *, 3> indices;
     114    indices[0] = idb->getInt32(0);
     115    for (unsigned i = 0; i < ty->getStructNumElements(); ++i) {
     116        if (isDynamicallyAllocatedType(ty->getStructElementType(i))) {
     117            indices[1] = idb->getInt32(i);
     118            indices[2] = idb->getInt32(1);
     119            Value * const innerFrame = idb->CreateLoad(idb->CreateGEP(frame, ArrayRef<Value*>(indices.data(), 3)));
     120            if (containsDynamicallyAllocatedType(innerFrame->getType())) {
     121                indices[2] = indices[0];
     122                Value *  const count = idb->CreateLoad(idb->CreateGEP(frame, ArrayRef<Value*>(indices.data(), 3)));
     123                BasicBlock * const entry = idb->GetInsertBlock();
     124                BasicBlock * const cond = idb->CreateBasicBlock("freeCarryDataCond");
     125                BasicBlock * const body = idb->CreateBasicBlock("freeCarryDataLoop");
     126                BasicBlock * const exit = idb->CreateBasicBlock("freeCarryDataExit");
     127                idb->CreateBr(cond);
     128                idb->SetInsertPoint(cond);
     129                PHINode * const index = idb->CreatePHI(count->getType(), 2);
     130                index->addIncoming(ConstantInt::getNullValue(count->getType()), entry);
     131                Value * test = idb->CreateICmpNE(index, count);
     132                idb->CreateCondBr(test, body, exit);
     133                idb->SetInsertPoint(body);
     134                freeDynamicallyAllocatedMemory(idb, idb->CreateGEP(innerFrame, index));
     135                index->addIncoming(idb->CreateAdd(index, ConstantInt::get(count->getType(), 1)), body);
     136                idb->CreateBr(cond);
     137                idb->SetInsertPoint(exit);
     138            }
     139            idb->CreateFree(innerFrame);
     140            indices[2] = idb->getInt32(2);
     141            Value *  const summary = idb->CreateLoad(idb->CreateGEP(frame, ArrayRef<Value*>(indices.data(), 3)));
     142            idb->CreateFree(summary);
     143        }
     144    }
    91145}
    92146
     
    94148 * @brief releaseCarryData
    95149 ** ------------------------------------------------------------------------------------------------------------- */
    96 void CarryManager::releaseCarryData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, PabloKernel * const kernel) {
    97 
    98 
     150void CarryManager::releaseCarryData(const std::unique_ptr<kernel::KernelBuilder> & idb) {
     151    if (mHasNonCarryCollapsingLoops) {
     152        freeDynamicallyAllocatedMemory(idb, idb->getScalarFieldPtr("carries"));
     153    }
    99154}
    100155
     
    186241        mLoopIndicies.push_back(index);
    187242        index->addIncoming(iBuilder->getSize(0), entryBlock);
    188 
    189243        Value * capacityPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
    190         Value * capacity = iBuilder->CreateLoad(capacityPtr, false, "capacity");
     244        Value * capacity = iBuilder->CreateLoad(capacityPtr, "capacity");
    191245        Constant * const ONE = ConstantInt::get(capacity->getType(), 1);
    192246        Value * arrayPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(1)});
    193         Value * array = iBuilder->CreateLoad(arrayPtr, false, "array");
    194 
     247        Value * array = iBuilder->CreateLoad(arrayPtr, "array");
    195248        BasicBlock * const entry = iBuilder->GetInsertBlock();
    196249        BasicBlock * const resizeCarryState = iBuilder->CreateBasicBlock("ResizeCarryState");
     
    198251        BasicBlock * const createNew = iBuilder->CreateBasicBlock("CreateNew");
    199252        BasicBlock * const resumeKernel = iBuilder->CreateBasicBlock("ResumeKernel");
    200 
    201         iBuilder->CreateLikelyCondBr(iBuilder->CreateICmpULT(index, capacity), resumeKernel, resizeCarryState);
     253        iBuilder->CreateLikelyCondBr(iBuilder->CreateICmpNE(index, capacity), resumeKernel, resizeCarryState);
    202254
    203255        // RESIZE CARRY BLOCK
    204256        iBuilder->SetInsertPoint(resizeCarryState);
    205         const auto BlockWidth = carryTy->getPrimitiveSizeInBits() / 8;
     257        const auto BlockWidth = iBuilder->getBitBlockWidth() / 8;
    206258        const auto Log2BlockWidth = floor_log2(BlockWidth);
    207259        Constant * const carryStateWidth = ConstantExpr::getIntegerCast(ConstantExpr::getSizeOf(array->getType()->getPointerElementType()), iBuilder->getSizeTy(), false);
    208         Value * summaryPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(2)});
    209 
     260        Value * const summaryPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(2)});
    210261        Value * const hasCarryState = iBuilder->CreateICmpNE(array, ConstantPointerNull::get(cast<PointerType>(array->getType())));
    211 
    212262        iBuilder->CreateLikelyCondBr(hasCarryState, reallocExisting, createNew);
    213263
    214264        // REALLOCATE EXISTING
    215265        iBuilder->SetInsertPoint(reallocExisting);
    216 
    217266        Value * const capacitySize = iBuilder->CreateMul(capacity, carryStateWidth);
    218267        Value * const newCapacitySize = iBuilder->CreateShl(capacitySize, 1); // x 2
    219 
    220         Value * newArray = iBuilder->CreateRealloc(array, newCapacitySize);
    221 
     268        Value * const newArray = iBuilder->CreateCacheAlignedMalloc(newCapacitySize);
     269        iBuilder->CreateMemCpy(newArray, array, capacitySize, iBuilder->getCacheAlignment());
     270        iBuilder->CreateFree(array);
     271        iBuilder->CreateStore(newArray, arrayPtr);
    222272        Value * const startNewArrayPtr = iBuilder->CreateGEP(iBuilder->CreatePointerCast(newArray, int8PtrTy), capacitySize);
    223273        iBuilder->CreateMemZero(startNewArrayPtr, capacitySize, BlockWidth);
    224 
    225         iBuilder->CreateStore(newArray, arrayPtr);
    226 
    227         Value * const log2capacity = iBuilder->CreateAdd(iBuilder->CreateCeilLog2(capacity), ONE);
    228         Value * const summarySize = iBuilder->CreateShl(log2capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
    229         Value * const newLog2Capacity = iBuilder->CreateAdd(log2capacity, ONE);
    230         Value * const newSummarySize = iBuilder->CreateShl(newLog2Capacity, Log2BlockWidth + 1); // x 2(BlockWidth)
    231 
     274        Value * const newCapacity = iBuilder->CreateShl(capacity, 1);
     275        iBuilder->CreateStore(newCapacity, capacityPtr);
    232276        Value * const summary = iBuilder->CreateLoad(summaryPtr, false);
    233         Value * newSummary = iBuilder->CreateRealloc(summary, newSummarySize);
    234         Value * const startNewSummaryPtr = iBuilder->CreateGEP(iBuilder->CreatePointerCast(newArray, int8PtrTy), summarySize);
    235         iBuilder->CreateMemZero(startNewSummaryPtr, iBuilder->getSize(2 * BlockWidth), BlockWidth);
    236 
    237         Value * ptr1 = iBuilder->CreateGEP(newSummary, summarySize);
    238         ptr1 = iBuilder->CreatePointerCast(ptr1, carryPtrTy);
    239 
    240         Value * ptr2 = iBuilder->CreateGEP(newSummary, iBuilder->CreateAdd(summarySize, iBuilder->getSize(BlockWidth)));
    241         ptr2 = iBuilder->CreatePointerCast(ptr2, carryPtrTy);
    242 
    243         newSummary = iBuilder->CreatePointerCast(newSummary, carryPtrTy);
    244         iBuilder->CreateStore(newSummary, summaryPtr);
    245         Value * const newCapacity = iBuilder->CreateShl(ONE, log2capacity);
    246 
    247         iBuilder->CreateStore(newCapacity, capacityPtr);
    248 
     277        Value * const summarySize = iBuilder->CreateShl(iBuilder->CreateAdd(iBuilder->CreateCeilLog2(capacity), ONE), Log2BlockWidth + 1);
     278        Constant * const additionalSpace = iBuilder->getSize(2 * BlockWidth);
     279        Value * const newSummarySize = iBuilder->CreateAdd(summarySize, additionalSpace);
     280        Value * const newSummary = iBuilder->CreateBlockAlignedMalloc(newSummarySize);
     281        iBuilder->CreateMemCpy(newSummary, summary, summarySize, BlockWidth);
     282        iBuilder->CreateFree(summary);
     283        iBuilder->CreateStore(iBuilder->CreatePointerCast(newSummary, carryPtrTy), summaryPtr);
     284        Value * const startNewSummaryPtr = iBuilder->CreateGEP(iBuilder->CreatePointerCast(newSummary, int8PtrTy), summarySize);
     285        iBuilder->CreateMemZero(startNewSummaryPtr, additionalSpace, BlockWidth);
    249286        iBuilder->CreateBr(resumeKernel);
    250287
    251288        // CREATE NEW
    252289        iBuilder->SetInsertPoint(createNew);
    253 
    254290        Constant * const initialLog2Capacity = iBuilder->getInt64(4);
    255291        Constant * const initialCapacity = ConstantExpr::getShl(ONE, initialLog2Capacity);
     292        iBuilder->CreateStore(initialCapacity, capacityPtr);
    256293        Constant * const initialCapacitySize = ConstantExpr::getMul(initialCapacity, carryStateWidth);
    257 
    258         Value * initialArray = iBuilder->CreateAlignedMalloc(initialCapacitySize, iBuilder->getCacheAlignment());
     294        Value * initialArray = iBuilder->CreateCacheAlignedMalloc(initialCapacitySize);
    259295        iBuilder->CreateMemZero(initialArray, initialCapacitySize, BlockWidth);
    260296        initialArray = iBuilder->CreatePointerCast(initialArray, array->getType());
    261297        iBuilder->CreateStore(initialArray, arrayPtr);
    262 
    263298        Constant * initialSummarySize = ConstantExpr::getShl(ConstantExpr::getAdd(initialLog2Capacity, iBuilder->getInt64(1)), iBuilder->getInt64(Log2BlockWidth + 1));
    264         Value * initialSummary = iBuilder->CreateAlignedMalloc(initialSummarySize, BlockWidth);
     299        Value * initialSummary = iBuilder->CreateBlockAlignedMalloc(initialSummarySize);
    265300        iBuilder->CreateMemZero(initialSummary, initialSummarySize, BlockWidth);
    266301        initialSummary = iBuilder->CreatePointerCast(initialSummary, carryPtrTy);
    267302        iBuilder->CreateStore(initialSummary, summaryPtr);
    268 
    269         iBuilder->CreateStore(initialCapacity, capacityPtr);
    270 
    271303        iBuilder->CreateBr(resumeKernel);
    272304
    273305        // RESUME KERNEL
    274306        iBuilder->SetInsertPoint(resumeKernel);
    275         // Load the appropriate carry stat block
    276307        PHINode * phiArrayPtr = iBuilder->CreatePHI(array->getType(), 3);
    277308        phiArrayPtr->addIncoming(array, entry);
     
    316347        // NOTE: this requires that, for all loop iterations, i, and all block iterations, j, the carry in
    317348        // summary, CI_i,j, matches the carry out summary of the prior block iteration, CO_i,j - 1.
    318         // Otherwise we may end up with an incorrect result or being trapped in an infinite loop.
     349        // Otherwise we will end up with an incorrect result or being trapped in an infinite loop.
    319350
    320351        Value * capacityPtr = iBuilder->CreateGEP(mCurrentFrame, {iBuilder->getInt32(0), iBuilder->getInt32(0)});
     
    370401        iBuilder->SetInsertPoint(resume);
    371402
    372         IntegerType * ty = IntegerType::get(iBuilder->getContext(), carryTy->getPrimitiveSizeInBits());
    373         iBuilder->CreateAssert(iBuilder->CreateICmpEQ(iBuilder->CreateBitCast(finalBorrow, ty), ConstantInt::getNullValue(ty)), "finalBorrow != 0");
    374         iBuilder->CreateAssert(iBuilder->CreateICmpEQ(iBuilder->CreateBitCast(finalCarry, ty), ConstantInt::getNullValue(ty)), "finalCarry != 0");
     403        if (codegen::EnableAsserts) {
     404            iBuilder->CreateAssertZero(iBuilder->CreateOr(finalBorrow, finalCarry),
     405                                       "CarryManager: loop post-condition violated: final borrow and carry must be zero!");
     406        }
    375407
    376408        assert (!mLoopIndicies.empty());
     
    778810 * @brief analyse
    779811 ** ------------------------------------------------------------------------------------------------------------- */
    780 StructType * CarryManager::analyse(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const PabloBlock * const scope, const unsigned ifDepth, const unsigned loopDepth, const bool isNestedWithinNonCarryCollapsingLoop) {
     812StructType * CarryManager::analyse(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, const PabloBlock * const scope,
     813                                   const unsigned ifDepth, const unsigned loopDepth, const bool isNestedWithinNonCarryCollapsingLoop) {
    781814    assert ("scope cannot be null!" && scope);
    782     assert (mCarryScopes == 0 ? (scope == mKernel->getEntryBlock()) : (scope != mKernel->getEntryBlock()));
     815    assert ("entry scope (and only the entry scope) must be in scope 0"
     816            && (mCarryScopes == 0 ? (scope == mKernel->getEntryBlock()) : (scope != mKernel->getEntryBlock())));
    783817    assert (mCarryScopes < mCarryMetadata.size());
    784818    Type * const carryTy = iBuilder->getBitBlockType();
     
    821855        carryState = StructType::get(iBuilder->getContext());
    822856    } else {
    823         //if (ifDepth > 0 || (nonCarryCollapsingMode | isNestedWithinNonCarryCollapsingLoop)) {
    824857        if (dyn_cast_or_null<If>(scope->getBranch()) || nonCarryCollapsingMode || isNestedWithinNonCarryCollapsingLoop) {
    825858            if (LLVM_LIKELY(state.size() > 1)) {
     
    839872        // carry state pointer, and summary pointer struct.
    840873        if (LLVM_UNLIKELY(nonCarryCollapsingMode)) {
     874            mHasNonCarryCollapsingLoops = true;
    841875            carryState = StructType::get(iBuilder->getSizeTy(), carryState->getPointerTo(), carryTy->getPointerTo(), nullptr);
     876            assert (isDynamicallyAllocatedType(carryState));
    842877        }
    843878        cd.setNonCollapsingCarryMode(nonCarryCollapsingMode);
     
    859894, mIfDepth(0)
    860895, mHasLongAdvance(false)
     896, mHasNonCarryCollapsingLoops(false)
    861897, mHasLoop(false)
    862898, mLoopDepth(0)
  • icGREP/icgrep-devel/icgrep/pablo/carry_manager.h

    r5486 r5493  
    4848    void initializeCarryData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, PabloKernel * const kernel);
    4949
    50     void releaseCarryData(const std::unique_ptr<kernel::KernelBuilder> & iBuilder, PabloKernel * const kernel);
     50    void releaseCarryData(const std::unique_ptr<kernel::KernelBuilder> & idb);
    5151
    5252    void initializeCodeGen(const std::unique_ptr<kernel::KernelBuilder> & iBuilder);
     
    121121
    122122    bool                                            mHasLongAdvance;
    123 
     123    bool                                            mHasNonCarryCollapsingLoops;
    124124    bool                                            mHasLoop;
    125125    unsigned                                        mLoopDepth;
  • icGREP/icgrep-devel/icgrep/pablo/optimizers/distributivepass.cpp

    r5486 r5493  
    629629            cliques.emplace_back(std::move(Ai), std::move(*Bi));
    630630        }
    631         return std::move(cliques);
     631        return cliques;
    632632    }
    633633
  • icGREP/icgrep-devel/icgrep/pablo/pablo_compiler.cpp

    r5486 r5493  
    5252void PabloCompiler::releaseKernelData(const std::unique_ptr<kernel::KernelBuilder> &  iBuilder) {
    5353    assert ("PabloCompiler does not have a IDISA iBuilder" && iBuilder);
    54     mCarryManager->releaseCarryData(iBuilder, mKernel);
     54    mCarryManager->releaseCarryData(iBuilder);
    5555}
    5656
  • icGREP/icgrep-devel/icgrep/pablo/pablo_kernel.cpp

    r5486 r5493  
    176176                         std::vector<Binding> scalar_parameters,
    177177                         std::vector<Binding> scalar_outputs)
    178 : BlockOrientedKernel(std::move(annotateKernelNameWithDebugFlags(std::move(kernelName))),
     178: BlockOrientedKernel(annotateKernelNameWithDebugFlags(std::move(kernelName)),
    179179                      std::move(stream_inputs), std::move(stream_outputs),
    180180                      std::move(scalar_parameters), std::move(scalar_outputs),
  • icGREP/icgrep-devel/icgrep/re/re_assertion.h

    r5308 r5493  
    2828    void setAsserted(RE * r) {mAsserted = r;}
    2929
     30    static Assertion::Kind reverseKind(Assertion::Kind k);
     31    static Assertion::Sense negateSense(Assertion::Sense s);
    3032
    3133protected:
     
    3941    Sense mSense;
    4042};
     43
     44inline Assertion::Kind Assertion::reverseKind(Assertion::Kind k) {
     45    if (k == Assertion::Kind::Boundary) return k;
     46    return k == Assertion::Kind::Lookahead ? Assertion::Kind::Lookbehind : Assertion::Kind::Lookahead;
     47}
     48
     49inline Assertion::Sense Assertion::negateSense(Assertion::Sense s) {
     50    return s == Assertion::Sense::Positive ? Assertion::Sense::Negative : Assertion::Sense::Positive;
     51}
    4152
    4253inline Assertion * makeAssertion(RE * asserted, Assertion::Kind k, Assertion::Sense s) {
  • icGREP/icgrep-devel/icgrep/re/re_simplifier.cpp

    r5267 r5493  
    4141        re = makeRep(expr, rep->getLB(), rep->getUB());
    4242    } else if (Diff * diff = dyn_cast<Diff>(re)) {
    43         re = makeDiff(simplify(diff->getLH()), diff->getRH());
     43        re = makeDiff(simplify(diff->getLH()), simplify(diff->getRH()));
    4444    } else if (Intersect * e = dyn_cast<Intersect>(re)) {
    45         re = makeIntersect(simplify(e->getLH()), e->getRH());
     45        re = makeIntersect(simplify(e->getLH()), simplify(e->getRH()));
    4646    }
    4747    return re;
  • icGREP/icgrep-devel/icgrep/re/re_toolchain.cpp

    r5473 r5493  
    1010#include <re/re_compiler.h>            // for RE_Compiler
    1111#include <re/re_nullable.h>            // for RE_Nullable
     12#include <re/re_star_normal.h>         // for RE_Star_Normal
    1213#include <re/re_simplifier.h>          // for RE_Simplifier
    1314#include <re/printer_re.h>
     
    8182        std::cerr << "Simplifier:" << std::endl << Printer_RE::PrintRE(re_ast) << std::endl;
    8283    }
     84
     85    re_ast = re::RE_Star_Normal::star_normal(re_ast);
     86    if (PrintOptions.isSet(ShowAllREs) || PrintOptions.isSet(ShowSimplifiedREs)) {
     87        //Print to the terminal the AST that was transformed to the star normal form.
     88        std::cerr << "Star_Normal_Form:" << std::endl << Printer_RE::PrintRE(re_ast) << std::endl;
     89    }   
    8390    return re_ast;
    8491}
  • icGREP/icgrep-devel/icgrep/toolchain/NVPTXDriver.h

    r5474 r5493  
    2323    void finalizeObject() override;
    2424
     25    bool hasExternalFunction(const llvm::StringRef /* functionName */) const override { return false; }
     26
    2527    void * getMain() override; // "main" exists until the driver is deleted
    2628
  • icGREP/icgrep-devel/icgrep/toolchain/cpudriver.cpp

    r5489 r5493  
    2020#include <kernels/kernel.h>
    2121#include <llvm/IR/Verifier.h>
    22 
    2322#ifndef NDEBUG
    2423#define IN_DEBUG_MODE true
     
    7877        mEngine->setObjectCache(mCache);
    7978    }
    80 
    8179    mMainModule->setTargetTriple(mTarget->getTargetTriple().getTriple());
    8280
     
    136134        mEngine->addGlobalMapping(f, functionPtr);
    137135    } else if (LLVM_UNLIKELY(f->getType() != type->getPointerTo())) {
    138         report_fatal_error("Cannot link " + name + ": a function with a differing signature already exists with that name in " + mod->getName());
     136        report_fatal_error("Cannot link " + name + ": a function with a different signature already exists with that name in " + mod->getName());
    139137    }
    140138    return f;
     
    154152        PM.add(createPrintModulePass(*IROutputStream));
    155153    }
    156 
    157154    if (IN_DEBUG_MODE || LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::VerifyIR))) {
    158155        PM.add(createVerifierPass());
     
    163160    PM.add(createInstructionCombiningPass());    //Simple peephole optimizations and bit-twiddling.
    164161    PM.add(createCFGSimplificationPass());
     162
    165163    if (LLVM_UNLIKELY(codegen::DebugOptionIsSet(codegen::ShowIR))) {
    166164        if (LLVM_LIKELY(IROutputStream == nullptr)) {
     
    223221}
    224222
     223bool ParabixDriver::hasExternalFunction(llvm::StringRef functionName) const {
     224    return mEngine->getPointerToNamedFunction(functionName, false) != nullptr;
     225}
     226
    225227void * ParabixDriver::getMain() {
    226228    return mEngine->getPointerToNamedFunction("Main");
  • icGREP/icgrep-devel/icgrep/toolchain/cpudriver.h

    r5474 r5493  
    2121    void finalizeObject() override;
    2222
     23    bool hasExternalFunction(const llvm::StringRef functionName) const override;
     24
    2325    void * getMain() override; // "main" exists until the driver is deleted
    2426
  • icGREP/icgrep-devel/icgrep/toolchain/driver.h

    r5474 r5493  
    4040    llvm::Function * LinkFunction(kernel::Kernel & kb, llvm::StringRef name, ExternalFunctionType * functionPtr) const;
    4141
     42    virtual bool hasExternalFunction(const llvm::StringRef functionName) const = 0;
     43
    4244    virtual void finalizeObject() = 0;
    4345
  • icGREP/icgrep-devel/icgrep/toolchain/object_cache.cpp

    r5464 r5493  
    207207
    208208ParabixObjectCache::ParabixObjectCache()
    209 : mCachePath(std::move(getDefaultPath())) {
     209: mCachePath(getDefaultPath()) {
    210210
    211211}
Note: See TracChangeset for help on using the changeset viewer.