Changeset 5115


Ignore:
Timestamp:
Aug 5, 2016, 8:27:14 AM (3 years ago)
Author:
cameron
Message:

Use bitblock_add_with_carry in carry_manager; add AVX2 implementation

Location:
icGREP/icgrep-devel/icgrep
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • icGREP/icgrep-devel/icgrep/IDISA/idisa_avx_builder.cpp

    r5014 r5115  
    1010#include <llvm/IR/Intrinsics.h>
    1111#include <llvm/IR/Function.h>
     12#include <iostream>
    1213
    1314namespace IDISA {
     
    145146}
    146147   
     148std::pair<Value *, Value *> IDISA_AVX2_Builder::bitblock_add_with_carry(Value * e1, Value * e2, Value * carryin) {
     149    // using LONG_ADD
     150    Type * carryTy = carryin->getType();
     151    if (carryTy == mBitBlockType) {
     152        carryin = mvmd_extract(32, carryin, 0);
     153    }
     154    Value * carrygen = simd_and(e1, e2);
     155    Value * carryprop = simd_or(e1, e2);
     156    Value * digitsum = simd_add(64, e1, e2);
     157    Value * digitcarry = simd_or(carrygen, simd_and(carryprop, CreateNot(digitsum)));
     158    Value * carryMask = hsimd_signmask(64, digitcarry);
     159    Value * carryMask2 = CreateOr(CreateAdd(carryMask, carryMask), carryin);
     160    Value * bubble = simd_eq(64, digitsum, allOnes());
     161    Value * bubbleMask = hsimd_signmask(64, bubble);
     162    Value * incrementMask = CreateXor(CreateAdd(bubbleMask, carryMask2), bubbleMask);
     163    Value * increments = esimd_bitspread(64,incrementMask);
     164    Value * sum = simd_add(64, digitsum, increments);
     165    Value * carry_out = CreateLShr(incrementMask, mBitBlockWidth / 64);
     166    if (carryTy == mBitBlockType) {
     167        carry_out = bitCast(CreateZExt(carry_out, getIntNTy(mBitBlockWidth)));
     168    }
     169    return std::pair<Value *, Value *>(carry_out, bitCast(sum));
    147170}
     171   
     172}
  • icGREP/icgrep-devel/icgrep/IDISA/idisa_avx_builder.h

    r4996 r5115  
    3535    Value * hsimd_packh_in_lanes(unsigned lanes, unsigned fw, Value * a, Value * b) override;
    3636    Value * hsimd_packl_in_lanes(unsigned lanes, unsigned fw, Value * a, Value * b) override;
     37    std::pair<Value *, Value *> bitblock_add_with_carry(Value * a, Value * b, Value * carryin) override;
    3738
    3839    ~IDISA_AVX2_Builder() {};
  • icGREP/icgrep-devel/icgrep/IDISA/idisa_builder.cpp

    r5114 r5115  
    367367
    368368// full add producing {carryout, sum}
    369 std::pair<Value *, Value *> IDISA_Builder::bitblock_add(Value * a, Value * b, Value * carryin) {
     369std::pair<Value *, Value *> IDISA_Builder::bitblock_add_with_carry(Value * a, Value * b, Value * carryin) {
    370370    Value * carrygen = simd_and(a, b);
    371371    Value * carryprop = simd_or(a, b);
  • icGREP/icgrep-devel/icgrep/IDISA/idisa_builder.h

    r5114 r5115  
    121121    virtual Value * bitblock_any(Value * a);
    122122    // full add producing {carryout, sum}
    123     virtual std::pair<Value *, Value *> bitblock_add(Value * a, Value * b, Value * carryin);
     123    virtual std::pair<Value *, Value *> bitblock_add_with_carry(Value * a, Value * b, Value * carryin);
    124124    // full shift producing {shiftout, shifted}
    125125    virtual std::pair<Value *, Value *> bitblock_advance(Value * a, Value * shiftin, unsigned shift);
  • icGREP/icgrep-devel/icgrep/kernels/kernel.cpp

    r5111 r5115  
    119119        doBlockArgs.push_back(&*args++);
    120120    }
    121     Value * rslt = iBuilder->CreateCall(doBlockFunction, doBlockArgs);
     121    iBuilder->CreateCall(doBlockFunction, doBlockArgs);
    122122    iBuilder->CreateRetVoid();
    123123    iBuilder->restoreIP(savePoint);
  • icGREP/icgrep-devel/icgrep/pablo/carry_manager.cpp

    r5106 r5115  
    9292 ** ------------------------------------------------------------------------------------------------------------- */
    9393Value * CarryManager::addCarryInCarryOut(const unsigned localIndex, Value * const e1, Value * const e2) {
    94     Value * sum = nullptr;
    95     if (mBitBlockWidth == 128) {
    96         Value * carryq_value = getCarryIn(localIndex);
    97         //calculate carry through logical ops
    98         Value * carrygen = iBuilder->simd_and(e1, e2);
    99         Value * carryprop = iBuilder->simd_or(e1, e2);
    100         Value * digitsum = iBuilder->simd_add(64, e1, e2);
    101         Value * partial = iBuilder->simd_add(64, digitsum, carryq_value);
    102         Value * digitcarry = iBuilder->simd_or(carrygen, iBuilder->simd_and(carryprop, iBuilder->CreateNot(partial)));
    103         Value * mid_carry_in = iBuilder->simd_slli(128, iBuilder->CreateLShr(digitcarry, 63), 64);
    104         sum = iBuilder->simd_add(64, partial, iBuilder->CreateBitCast(mid_carry_in, mBitBlockType));
    105         Value * carry_out_strm = iBuilder->simd_or(carrygen, iBuilder->simd_and(carryprop, iBuilder->CreateNot(sum)));
    106         setCarryOut(localIndex, carry_out_strm);
    107     } else if (mBitBlockWidth >= 256) {
    108         // using LONG_ADD
    109         Value * carryq_value = getCarryIn(localIndex);
    110         Value * carryin = iBuilder->mvmd_extract(32, carryq_value, 0);
    111         Value * carrygen = iBuilder->simd_and(e1, e2);
    112         Value * carryprop = iBuilder->simd_or(e1, e2);
    113         Value * digitsum = iBuilder->simd_add(64, e1, e2);
    114         Value * digitcarry = iBuilder->simd_or(carrygen, iBuilder->simd_and(carryprop, iBuilder->CreateNot(digitsum)));
    115         Value * carryMask = iBuilder->hsimd_signmask(64, digitcarry);
    116         Value * carryMask2 = iBuilder->CreateOr(iBuilder->CreateAdd(carryMask, carryMask), carryin);
    117         Value * bubble = iBuilder->simd_eq(64, digitsum, iBuilder->allOnes());
    118         Value * bubbleMask = iBuilder->hsimd_signmask(64, bubble);
    119         Value * incrementMask = iBuilder->CreateXor(iBuilder->CreateAdd(bubbleMask, carryMask2), bubbleMask);
    120         Value * increments = iBuilder->esimd_bitspread(64,incrementMask);
    121         sum = iBuilder->simd_add(64, digitsum, increments);
    122         Value * carry_out_strm = iBuilder->CreateZExt(iBuilder->CreateLShr(incrementMask, mBitBlockWidth / 64), iBuilder->getIntNTy(mBitBlockWidth));
    123         setCarryOut(localIndex, iBuilder->bitCast(carry_out_strm));
    124     }
    125     else {
    126         Value * carryq_value = getCarryIn(localIndex);
    127         Value * carrygen = iBuilder->simd_and(e1, e2);
    128         Value * carryprop = iBuilder->simd_or(e1, e2);
    129         sum = iBuilder->simd_add(mBitBlockWidth, iBuilder->simd_add(mBitBlockWidth, e1, e2), carryq_value);
    130         Value * carry_out_strm = iBuilder->simd_or(carrygen, iBuilder->simd_and(carryprop, iBuilder->CreateNot(sum)));
    131         setCarryOut(localIndex, carry_out_strm);
    132     }
    133     return sum;
     94    std::pair<Value *, Value *> fullAdd = iBuilder->bitblock_add_with_carry(e1, e2, getCarryIn(localIndex));
     95    setCarryOut(localIndex, std::get<0>(fullAdd));
     96    return std::get<1>(fullAdd);
    13497}
    13598
     
    363326void CarryManager::setCarryOut(const unsigned localIndex, Value * carryOut) {
    364327    const unsigned index = addPosition(localIndex);
    365     if (mBitBlockWidth < 256) { // #ifndef USING_LONG_ADD
    366         Value * carry_bit = iBuilder->CreateLShr(iBuilder->CreateBitCast(carryOut, iBuilder->getIntNTy(mBitBlockWidth)), mBitBlockWidth-1);
    367         carryOut = iBuilder->CreateBitCast(carry_bit, mBitBlockType);
    368     }
    369328    assert (index < mCarryOutPack.size());
    370329    mCarryOutPack[index] = carryOut;
     
    441400 * @brief addToSummary
    442401 ** ------------------------------------------------------------------------------------------------------------- */
    443 inline Value * CarryManager::addToSummary(Value * const value) {
     402void CarryManager::addToSummary(Value * const value) {
    444403    const unsigned summaryIndex = summaryPack();
    445404    assert (summaryIndex < mCarryInPack.size());
     
    447406    assert (summary);
    448407    assert (value);
     408    if (LLVM_UNLIKELY(summary == value)) return;  //Nothing to add.
     409   
     410    Type * summaryTy = summary->getType();
     411    Type * valueTy = value->getType();
     412    if (LLVM_UNLIKELY(isa<Constant>(value))) {
     413        if (LLVM_LIKELY(cast<Constant>(value)->isZeroValue())) return;
     414        if (cast<Constant>(value)->isAllOnesValue()) {
     415            mCarryOutPack[summaryIndex] = Constant::getAllOnesValue(summaryTy);
     416            return;
     417        }
     418    }
     419    Value * v = value;
     420    if (valueTy != summaryTy) {
     421        // valueTy must be an integer type.
     422        unsigned summaryWidth = summaryTy->isIntegerTy() ? summaryTy->getIntegerBitWidth() : cast<VectorType>(summaryTy)->getBitWidth();       
     423        if (valueTy->getIntegerBitWidth() != summaryWidth) {
     424            v = iBuilder->CreateZExt(v, iBuilder->getIntNTy(summaryWidth));
     425        }
     426        if (!(summaryTy->isIntegerTy())) {
     427            v = iBuilder->CreateBitCast(v, summaryTy);
     428        }
     429    }
    449430    if (LLVM_UNLIKELY(isa<Constant>(summary))) {
    450431        if (LLVM_LIKELY(cast<Constant>(summary)->isZeroValue())) {
    451             summary = value;
    452             goto return_result;
     432            mCarryOutPack[summaryIndex] = v;
     433            return;
    453434        } else if (cast<Constant>(summary)->isAllOnesValue()) {
    454             goto return_result;
    455         }
    456     }
    457     if (LLVM_UNLIKELY(isa<Constant>(value))) {
    458         if (LLVM_LIKELY(cast<Constant>(value)->isZeroValue())) {
    459             goto return_result;
    460         } else if (cast<Constant>(summary)->isAllOnesValue()) {
    461             summary = value;
    462             goto return_result;
    463         }
    464     }
    465     if (LLVM_LIKELY(summary != value)) {
    466         summary = iBuilder->CreateOr(summary, value, "summary");
    467     }
    468 return_result:
    469     mCarryOutPack[summaryIndex] = summary;
    470     return summary;
     435            return;
     436        }
     437    }
     438    mCarryOutPack[summaryIndex] = iBuilder->CreateOr(summary, v, "summary");
    471439}
    472440
  • icGREP/icgrep-devel/icgrep/pablo/carry_manager.h

    r5063 r5115  
    115115    void storeCarryOut(const unsigned packIndex);
    116116   
    117     Value * addToSummary(Value * const value);
     117    void addToSummary(Value * const value);
    118118
    119119    bool hasSummary() const;
Note: See TracChangeset for help on using the changeset viewer.