Changeset 1991 for trunk/symbol_table


Ignore:
Timestamp:
Mar 31, 2012, 4:05:09 PM (7 years ago)
Author:
ksherdy
Message:

Updated Makefile.

Location:
trunk/symbol_table/src
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • trunk/symbol_table/src/hash_table.hpp

    r1989 r1991  
    1919#define MIN(a, b) ((a < b) ? a : b)
    2020#define MAX(a, b) ((a > b) ? a : b)
    21 #define MAX_TABLE_SIZE 65536 * 4
     21#define MAX_TABLE_SIZE 65536
    2222#define MAX_HASH_BITS 17   
    2323
     
    101101    }
    102102
    103     IDISA_ALWAYS_INLINE gid_type lookup_or_insert(uint8_t * raw_bytes, const uint32_t idx, const uint32_t raw_byte_lgth,
    104                                                   uint8_t * h0, uint8_t * h1, const uint32_t hash_bit_lgth) {
     103    IDISA_ALWAYS_INLINE void lookup_or_insert(uint8_t * raw_bytes, const uint32_t idx, const uint32_t raw_byte_lgth, /* IN, IN, IN */
     104                                                  uint8_t * h0, uint8_t * h1, const uint32_t hash_bit_lgth /*IN, IN, IN*/
     105                                                  gid_type & gid, uint8_t & bytes /*OUT, OUT*/) {
    105106
    106107        uint64_t bucket = get_bucket(h0,h1,idx,hash_bit_lgth);
     
    121122        }
    122123
    123         /* insert */
    124         gid_type gid = gid::next();
     124        /* OUT params (GID, symbol length) */
     125        gid = gid::next();
     126        bytes = raw_data_pool.insert(&raw_bytes[idx],raw_byte_lgth)
    125127
    126128        uint64_t x0 = bit_slice(h0, idx, hash_bit_lgth);
     
    128130
    129131        insert( bucket,
    130                 raw_data_pool.insert(&raw_bytes[idx],raw_byte_lgth),
     132                bytes,
    131133                raw_byte_lgth,
    132134                raw_data_pool.insert((uint8_t *)&x0, bits2bytes(hash_bit_lgth)),
     
    148150        #endif
    149151
    150         return gid;
    151152    }
    152153
  • trunk/symbol_table/src/id_symbol_table.hpp

    r1989 r1991  
    8181    // Groups & groups
    8282    void resolve(uint8_t buffer [], Groups groups [],  BitBlock starts [], BitBlock ends_gte_17 [],
    83                  BitBlock h0 [], BitBlock h1 [], uint32_t blocks, AoS_symbol aos [], const uint32_t symbols) {
     83                 BitBlock h0 [], BitBlock h1 [], uint32_t blocks, AoS_symbol aos []/*, const uint32_t symbols*/) {
     84
     85        uint32_t blk_base;
    8486
    8587        for(uint32_t blk=0;blk<blocks;blk++) {
     88
     89        blk_base = blk*BLOCK_SIZE;
    8690        ///////////////////////////////////////////////////////////////////////////////
    8791        // Byte Space Hash
    8892        ///////////////////////////////////////////////////////////////////////////////
    8993        if(bitblock::any(groups[blk].ends_1)) {
    90         do_block<hash_table <identity_strategy_t<uint8_t,1>, hash_strategy_t<1>, ALLOCATOR> >(hash_table_1, groups[blk].ends_1, &buffer[blk*BLOCK_SIZE], 1, &buffer[blk*BLOCK_SIZE], &buffer[blk*BLOCK_SIZE], bytes2bits(1), BLOCK_SIZE);
     94        do_block<hash_table <identity_strategy_t<uint8_t,1>, hash_strategy_t<1>, ALLOCATOR> >(hash_table_1, groups[blk].ends_1, &buffer[blk_base], 1, &buffer[blk_base], &buffer[blk_base], bytes2bits(1), BLOCK_SIZE, aos, blk_base);
    9195        }
    9296        if(bitblock::any(groups[blk].ends_2)) {
    93         do_block<hash_table <identity_strategy_t<uint16_t,2>, hash_strategy_t<2>, ALLOCATOR> >(hash_table_2, groups[blk].ends_2, &buffer[blk*BLOCK_SIZE], 2, &buffer[blk*BLOCK_SIZE], &buffer[blk*BLOCK_SIZE], bytes2bits(2), BLOCK_SIZE);
     97        do_block<hash_table <identity_strategy_t<uint16_t,2>, hash_strategy_t<2>, ALLOCATOR> >(hash_table_2, groups[blk].ends_2, &buffer[blk_base], 2, &buffer[blk_base], &buffer[blk_base], bytes2bits(2), BLOCK_SIZE, aos, blk_base);
    9498        }
    9599        if(bitblock::any(groups[blk].ends_3)) {
    96         do_block<hash_table <identity_strategy_t<uint16_t,3>, hash_strategy_t<3>, ALLOCATOR> >(hash_table_3, groups[blk].ends_3, &buffer[blk*BLOCK_SIZE], 3, &buffer[blk*BLOCK_SIZE], &buffer[blk*BLOCK_SIZE], bytes2bits(3), BLOCK_SIZE);
     100        do_block<hash_table <identity_strategy_t<uint16_t,3>, hash_strategy_t<3>, ALLOCATOR> >(hash_table_3, groups[blk].ends_3, &buffer[blk_base], 3, &buffer[blk_base], &buffer[blk_base], bytes2bits(3), BLOCK_SIZE, aos, blk_base);
    97101        }
    98102        if(bitblock::any(groups[blk].ends_4)) {
    99         do_block<hash_table <identity_strategy_t<uint32_t,4>, hash_strategy_t<4>, ALLOCATOR> >(hash_table_4, groups[blk].ends_4, &buffer[blk*BLOCK_SIZE], 4, &buffer[blk*BLOCK_SIZE], &buffer[blk*BLOCK_SIZE], bytes2bits(4), BLOCK_SIZE);
     103        do_block<hash_table <identity_strategy_t<uint32_t,4>, hash_strategy_t<4>, ALLOCATOR> >(hash_table_4, groups[blk].ends_4, &buffer[blk_base], 4, &buffer[blk_base], &buffer[blk_base], bytes2bits(4), BLOCK_SIZE, aos, blk_base);
    100104        }
    101105        if(bitblock::any(groups[blk].ends_5)) {
    102         do_block<hash_table <identity_strategy_t<uint32_t,5>, hash_strategy_t<5>, ALLOCATOR> >(hash_table_5, groups[blk].ends_5, &buffer[blk*BLOCK_SIZE], 5, &buffer[blk*BLOCK_SIZE], &buffer[blk*BLOCK_SIZE], bytes2bits(5), BLOCK_SIZE);
     106        do_block<hash_table <identity_strategy_t<uint32_t,5>, hash_strategy_t<5>, ALLOCATOR> >(hash_table_5, groups[blk].ends_5, &buffer[blk_base], 5, &buffer[blk_base], &buffer[blk_base], bytes2bits(5), BLOCK_SIZE, aos, blk_base);
    103107        }
    104108        if(bitblock::any(groups[blk].ends_6)) {
    105         do_block<hash_table <identity_strategy_t<uint32_t,6>, hash_strategy_t<6>, ALLOCATOR> >(hash_table_6, groups[blk].ends_6, &buffer[blk*BLOCK_SIZE], 6, &buffer[blk*BLOCK_SIZE], &buffer[blk*BLOCK_SIZE], bytes2bits(6), BLOCK_SIZE);
     109        do_block<hash_table <identity_strategy_t<uint32_t,6>, hash_strategy_t<6>, ALLOCATOR> >(hash_table_6, groups[blk].ends_6, &buffer[blk_base], 6, &buffer[blk_base], &buffer[blk_base], bytes2bits(6), BLOCK_SIZE, aos, blk_base);
    106110        }
    107111        if(bitblock::any(groups[blk].ends_7)) {
    108         do_block<hash_table <identity_strategy_t<uint32_t,7>, hash_strategy_t<7>, ALLOCATOR> >(hash_table_7, groups[blk].ends_7, &buffer[blk*BLOCK_SIZE], 7, &buffer[blk*BLOCK_SIZE], &buffer[blk*BLOCK_SIZE], bytes2bits(7), BLOCK_SIZE);
     112        do_block<hash_table <identity_strategy_t<uint32_t,7>, hash_strategy_t<7>, ALLOCATOR> >(hash_table_7, groups[blk].ends_7, &buffer[blk_base], 7, &buffer[blk_base], &buffer[blk_base], bytes2bits(7), BLOCK_SIZE, aos, blk_base);
    109113        }
    110114        ///////////////////////////////////////////////////////////////////////////////
     
    112116        ///////////////////////////////////////////////////////////////////////////////
    113117        if(bitblock::any(groups[blk].ends_8)) {
    114         do_block<hash_table <identity_strategy_t<uint64_t,8>, hash_strategy_d, ALLOCATOR> >(hash_table_8, groups[blk].ends_8, &buffer[blk*BLOCK_SIZE], 8, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 8, BLOCK_SIZE/8);
     118        do_block<hash_table <identity_strategy_t<uint64_t,8>, hash_strategy_d, ALLOCATOR> >(hash_table_8, groups[blk].ends_8, &buffer[blk_base], 8, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 8, BLOCK_SIZE/8, aos, blk_base);
    115119        }
    116120        if(bitblock::any(groups[blk].ends_9)) {
    117         do_block<hash_table<identity_strategy_t<uint64_t,9>, hash_strategy_d, ALLOCATOR> >(hash_table_9, groups[blk].ends_9, &buffer[blk*BLOCK_SIZE], 9, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 9, BLOCK_SIZE/8);
     121        do_block<hash_table<identity_strategy_t<uint64_t,9>, hash_strategy_d, ALLOCATOR> >(hash_table_9, groups[blk].ends_9, &buffer[blk_base], 9, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 9, BLOCK_SIZE/8, aos, blk_base);
    118122        }
    119123        if(bitblock::any(groups[blk].ends_10)) {
    120         do_block<hash_table<identity_strategy_t<uint64_t,10>, hash_strategy_d, ALLOCATOR> >(hash_table_10, groups[blk].ends_10, &buffer[blk*BLOCK_SIZE], 10, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 10, BLOCK_SIZE/8);
     124        do_block<hash_table<identity_strategy_t<uint64_t,10>, hash_strategy_d, ALLOCATOR> >(hash_table_10, groups[blk].ends_10, &buffer[blk_base], 10, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 10, BLOCK_SIZE/8, aos, blk_base);
    121125        }
    122126        if(bitblock::any(groups[blk].ends_11)) {
    123         do_block<hash_table<identity_strategy_t<uint64_t,11>, hash_strategy_d, ALLOCATOR> >(hash_table_11, groups[blk].ends_11, &buffer[blk*BLOCK_SIZE], 11, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 11, BLOCK_SIZE/8);
     127        do_block<hash_table<identity_strategy_t<uint64_t,11>, hash_strategy_d, ALLOCATOR> >(hash_table_11, groups[blk].ends_11, &buffer[blk_base], 11, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 11, BLOCK_SIZE/8, aos, blk_base);
    124128        }
    125129        if(bitblock::any(groups[blk].ends_12)) {
    126         do_block<hash_table<identity_strategy_t<uint64_t,12>, hash_strategy_d, ALLOCATOR> >(hash_table_12, groups[blk].ends_12, &buffer[blk*BLOCK_SIZE], 12, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 12, BLOCK_SIZE/8);
     130        do_block<hash_table<identity_strategy_t<uint64_t,12>, hash_strategy_d, ALLOCATOR> >(hash_table_12, groups[blk].ends_12, &buffer[blk_base], 12, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 12, BLOCK_SIZE/8, aos, blk_base);
    127131        }
    128132        if(bitblock::any(groups[blk].ends_13)) {
    129         do_block<hash_table<identity_strategy_t<uint64_t,13>, hash_strategy_d, ALLOCATOR> >(hash_table_13, groups[blk].ends_13, &buffer[blk*BLOCK_SIZE], 13, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 13, BLOCK_SIZE/8);
     133        do_block<hash_table<identity_strategy_t<uint64_t,13>, hash_strategy_d, ALLOCATOR> >(hash_table_13, groups[blk].ends_13, &buffer[blk_base], 13, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 13, BLOCK_SIZE/8, aos, blk_base);
    130134        }
    131135        if(bitblock::any(groups[blk].ends_14)) {
    132         do_block<hash_table<identity_strategy_t<uint64_t,14>, hash_strategy_d, ALLOCATOR> >(hash_table_14, groups[blk].ends_14, &buffer[blk*BLOCK_SIZE], 14, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 14, BLOCK_SIZE/8);
     136        do_block<hash_table<identity_strategy_t<uint64_t,14>, hash_strategy_d, ALLOCATOR> >(hash_table_14, groups[blk].ends_14, &buffer[blk_base], 14, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 14, BLOCK_SIZE/8, aos, blk_base);
    133137        }
    134138        if(bitblock::any(groups[blk].ends_15)) {
    135         do_block<hash_table<identity_strategy_t<uint64_t,15>, hash_strategy_d, ALLOCATOR> >(hash_table_15, groups[blk].ends_15, &buffer[blk*BLOCK_SIZE], 15, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 15, BLOCK_SIZE/8);
     139        do_block<hash_table<identity_strategy_t<uint64_t,15>, hash_strategy_d, ALLOCATOR> >(hash_table_15, groups[blk].ends_15, &buffer[blk_base], 15, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 15, BLOCK_SIZE/8, aos, blk_base);
    136140        }
    137141        if(bitblock::any(groups[blk].ends_16)) {
    138         do_block<hash_table<identity_strategy_t<BitBlock,16>, hash_strategy_d, ALLOCATOR> >(hash_table_16, groups[blk].ends_16, &buffer[blk*BLOCK_SIZE], 16, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 16, BLOCK_SIZE/8);
     142        do_block<hash_table<identity_strategy_t<BitBlock,16>, hash_strategy_d, ALLOCATOR> >(hash_table_16, groups[blk].ends_16, &buffer[blk_base], 16, (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 16, BLOCK_SIZE/8, aos, blk_base);
    139143        }
    140144        if(bitblock::any(ends_gte_17[blk])) {
    141         do_block<hash_table<identity_strategy_d, hash_strategy_d, ALLOCATOR> >(hash_table_gte_17, &starts[blk], &ends_gte_17[blk], &buffer[blk*BLOCK_SIZE], (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], BLOCK_SIZE/8);
     145        do_block<hash_table<identity_strategy_d, hash_strategy_d, ALLOCATOR> >(hash_table_gte_17, &starts[blk], &ends_gte_17[blk], &buffer[blk_base], (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], BLOCK_SIZE/8, aos, blk_base);
    142146        }
    143147
     
    177181template<class HASH_TABLE>
    178182IDISA_ALWAYS_INLINE void do_block(HASH_TABLE & h_table, BitBlock ends, uint8_t buffer [], const uint32_t lgth,
    179                                   uint8_t h0 [], uint8_t h1 [], const uint32_t h_lgth, const uint32_t h_block_size) {
     183                                  uint8_t h0 [], uint8_t h1 [], const uint32_t h_lgth, const uint32_t h_block_size,
     184                                  AoS_symbol aos [], const uint32_t block_base){
    180185
    181186    int32_t spos;
     
    188193
    189194        h_table.lookup_or_insert(buffer, spos, lgth,
    190                                  h0, h1, h_lgth);
     195                                 h0, h1, h_lgth,
     196                                 aos[block_base+spos].gid, aos[block_base+spos].raw_data);
    191197
    192198#ifdef ID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG
    193199        print_symbol_debug(buffer, spos, rscanner.get_pos(), lgth);
     200        aos[block_base+spos].gid;
    194201#endif
    195202        rscanner.scan_to_next();
     
    214221
    215222        h_table.lookup_or_insert(lb_buffer, lb_spos, lgth,
    216                                  lb_h0, lb_h1, h_lgth);
     223                                 lb_h0, lb_h1, h_lgth,
     224                                 aos[block_base+spos].gid, aos[block_base+spos].raw_data);
    217225
    218226#ifdef ID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG
     
    226234template<class HASH_TABLE>
    227235IDISA_ALWAYS_INLINE void do_block(HASH_TABLE & h_table, BitBlock starts [], BitBlock ends [], uint8_t buffer [],
    228                                   uint8_t h0 [], uint8_t h1 [], const uint32_t h_block_size) {
     236                                  uint8_t h0 [], uint8_t h1 [], const uint32_t h_block_size,
     237                                  AoS_symbol aos [], const uint32_t block_base) {
    229238
    230239
     
    243252        lgth = ends_rscanner.get_pos() - starts_rscanner.get_pos();
    244253
    245         h_table.lookup_or_insert(buffer, starts_rscanner.get_pos(), lgth, h0, h1, lgth);
     254        h_table.lookup_or_insert(buffer, starts_rscanner.get_pos(), lgth, h0, h1, lgth,aos[block_base+spos].gid, aos[block_base+spos].raw_data);
    246255
    247256#ifdef ID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG
     
    289298        uint8_t * lb_h1 = h1 - ((lgth / BLOCK_SIZE) + 1)*h_block_size;
    290299
    291         h_table.lookup_or_insert(lb_buffer, lb_spos, lgth, lb_h0, lb_h1, lgth);
     300        h_table.lookup_or_insert(lb_buffer, lb_spos, lgth, lb_h0, lb_h1, lgth,aos[block_base+spos].gid, aos[block_base+spos].raw_data);
    292301
    293302#ifdef ID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG
  • trunk/symbol_table/src/symbol_table.hpp

    r1979 r1991  
    2121    gid_type gid;
    2222    uint8_t * raw_data;
     23    uint32_t lgth;
    2324} AoS_symbol;
    2425
     
    2728    gid_type * gids;
    2829    uint8_t * raw_data;
     30    uint32_t * lgth;
    2931} SoA_symbol;
    3032
     
    3234public:
    3335    void resolve(uint8_t buffer [], Groups groups [],  BitBlock starts [], BitBlock ends_gte_17 [],
    34                  BitBlock h0 [], BitBlock h1 [], uint32_t blocks, AoS_symbol aos [], const uint32_t symbols);
     36                 BitBlock h0 [], BitBlock h1 [], uint32_t blocks, AoS_symbol aos []/*, const uint32_t symbols*/);
    3537
    3638    void resolve(uint8_t buffer [], Groups groups [],  BitBlock starts [], BitBlock ends_gte_17 [],
    37                  BitBlock h0 [], BitBlock h1 [], uint32_t blocks, SoA_symbol & soa, const uint32_t symbols);
     39                 BitBlock h0 [], BitBlock h1 [], uint32_t blocks, SoA_symbol & soa/*, const uint32_t symbols*/);
    3840
    3941protected:
Note: See TracChangeset for help on using the changeset viewer.