Changeset 2014


Ignore:
Timestamp:
Apr 10, 2012, 2:16:29 AM (7 years ago)
Author:
ksherdy
Message:

Added scanner move_to logic for non-boundary case arbitrary length symbols.

Location:
trunk/symbol_table/src
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • trunk/symbol_table/src/Makefile

    r2010 r2014  
    1414
    1515all: basis_bits.hpp buffer.hpp byte_pool.hpp  hash_strms.hpp  hash_table.hpp  id_group_strms.hpp  id_symbol_table.hpp  main.cpp  Makefile  marker_strms.hpp  symbol_table.hpp  transpose.hpp
    16         $(CC) -o main main.cpp $(AFLAGS) #-DID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG -DHASH_TABLE_HPP_DEBUG # -DBUFFER_PROFILING
     16        $(CC) -o main main.cpp $(AFLAGS) #-DID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG #-DHASH_TABLE_HPP_DEBUG # -DBUFFER_PROFILING
    1717
    1818clean:
  • trunk/symbol_table/src/id_symbol_table.hpp

    r2010 r2014  
    233233                }
    234234
    235                 }
    236 
    237 //      if(bitblock::any(ends_gte_17[blk])) {
    238 //      do_block<SYMBOL, hash_table<identity_strategy_d, hash_strategy_d, ALLOCATOR> >(hash_table_gte_17, &starts[blk], &ends_gte_17[blk], &buffer[blk], (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], BLOCK_SIZE/8, symbols);
    239 //      }
    240 
     235                if(bitblock::any(groups[blk].ends_gte_17)) {
     236                        do_block<SYMBOL, hash_table<identity_strategy_d, hash_strategy_d, ALLOCATOR> >
     237                                        (blk_offset,
     238                                         hash_table_gte_17,
     239                                         &starts[blk], &ends_gte_17[blk],
     240                                         &buffer[blk_offset],
     241                                         (uint8_t *)&h0[blk], (uint8_t *)&h1[blk], 17, BLOCK_SIZE/8,
     242                                         symbols);
     243                }
     244
     245                }
    241246    }
    242247
     
    265270    hash_table<identity_strategy_t<BitBlock,16>, hash_strategy_d, ALLOCATOR> hash_table_16;
    266271    hash_table<identity_strategy_d, hash_strategy_d, ALLOCATOR> hash_table_gte_17;
    267 
    268272};
    269273
     
    332336}
    333337
    334 // Variable Lengths - REVERSE SCAN LOGIC - Scan each BLOCK MSB to LSB (high to low memory address)
     338/////////
     339
     340// Variable Lengths, reverse scanner logic
    335341template<class SYMBOL, class HASH_TABLE>
    336 IDISA_ALWAYS_INLINE void do_block(HASH_TABLE & h_table, BitBlock starts [], BitBlock ends [], uint8_t buffer [],
    337                                   uint8_t h0 [], uint8_t h1 [], const uint32_t h_block_size,
    338                                   SYMBOL * aos, const uint32_t block_base) {
    339 
    340     int32_t spos;
    341     uint32_t lgth;
    342     ReverseScanner<BitBlock, scanword_t> ends_rscanner(ends);
    343     ReverseScanner<BitBlock, scanword_t> starts_rscanner(starts);
    344 
    345     ends_rscanner.scan_to_next();
    346     starts_rscanner.scan_to_next();
    347 
    348     while(starts_rscanner.get_pos() > ends_rscanner.get_pos()) { // synchronize, TODO - scan_from_position();
    349         starts_rscanner.scan_to_next();
    350     }
    351 
    352     while((!ends_rscanner.is_done()) && (!starts_rscanner.is_done())) {
     342void do_block(uint32_t blk_offset,
     343                                  HASH_TABLE & h_table,
     344                                  BitBlock starts [], BitBlock ends [],
     345                                  uint8_t buffer [],
     346                                  uint8_t h0 [], uint8_t h1 [], const uint32_t h_lgth, const uint32_t h_block_size,
     347                                  SYMBOL & symbols) {
     348
     349        gid_type gid;
     350        int32_t epos;
     351        int32_t spos;
     352        uint32_t lgth;
     353
     354        ReverseScanner<BitBlock, scanword_t> ends_rscanner(ends);
     355        ReverseScanner<BitBlock, scanword_t> starts_rscanner(starts);
     356
     357        ends_rscanner.scan_to_next();
     358        epos = ends_rscanner.get_pos();
     359        if(epos > 0) { starts_rscanner.move_to(epos); }
     360        starts_rscanner.scan_to_next();
    353361        spos = starts_rscanner.get_pos();
    354         lgth = ends_rscanner.get_pos() - spos;
    355         // Comment out LOOKBACK_SIZE . Nigel's code *should* not produce this case.
    356 //      aos[LOOKBACK_SIZE + block_base + spos].gid = h_table.lookup_or_insert(buffer, spos, lgth, h0, h1, lgth);
    357 //      aos[LOOKBACK_SIZE + block_base + spos].lgth = lgth;
    358 
    359 
    360 
    361 #ifdef ID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG
    362     print_symbol_debug(buffer, spos, ends_rscanner.get_pos(), lgth);
    363 #endif
    364 
    365         ends_rscanner.scan_to_next();
    366         starts_rscanner.scan_to_next();
    367 
    368         while(starts_rscanner.get_pos() > ends_rscanner.get_pos()) { // synchronize, TODO - scan_from_position();
    369             starts_rscanner.scan_to_next();
    370         }
    371 
    372     }
    373 
    374     // TODO - COMMENT OUT. Nigel's code *should* not produce this case.
    375     if((!ends_rscanner.is_done()) && (starts_rscanner.is_done())) { // block boundary case
    376 
    377         uint32_t lb_blk = 0;
    378         while(1) {
    379 
    380             starts_rscanner.init(--starts);
    381             starts_rscanner.scan_to_next();
    382 
    383             if(!starts_rscanner.is_done()) {
    384                 break;
    385             }
    386 
    387             if (lb_blk > LOOKBACK_BLOCKS) {
    388                 cerr << lb_blk << endl;
    389                 cerr << "Fatal Error.";
    390                 cerr << " Symbol length exceeds " << (LOOKBACK_SIZE) << " bytes." << endl;
    391                 cerr << " Symbol tail : ";
    392                 cerr << string((char *)&(buffer[starts_rscanner.get_pos()-(LOOKBACK_SIZE+BLOCK_SIZE)]), LOOKBACK_SIZE+BLOCK_SIZE) << endl;
    393                 abort();
    394             }
    395             lb_blk++;
     362        lgth = epos - spos;
     363
     364        while(!ends_rscanner.is_done() && !starts_rscanner.is_done()) {
     365               
     366                gid = h_table.lookup_or_insert(buffer, spos, lgth, h0, h1, h_lgth);
     367                symbols.gids[blk_offset + spos] = gid;
     368
     369                #ifdef ID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG
     370                        print_symbol_debug(buffer, spos, epos, lgth);
     371                #endif
     372
     373                ends_rscanner.scan_to_next();
     374                epos = ends_rscanner.get_pos();
     375                if(epos > 0) { starts_rscanner.move_to(epos); }
     376                starts_rscanner.scan_to_next();
     377                spos = starts_rscanner.get_pos();
     378                lgth = epos - spos;
    396379        }
    397380
    398         lgth = ends_rscanner.get_pos() + (BLOCK_SIZE - starts_rscanner.get_pos()) + lb_blk * BLOCK_SIZE;
    399 
    400         uint8_t * lb_buffer = buffer - ((lgth / BLOCK_SIZE) + 1)*BLOCK_SIZE;
    401         int32_t lb_spos = BLOCK_SIZE - ((-1*starts_rscanner.get_pos()) & (BLOCK_SIZE-1));
    402 
    403         uint8_t * lb_h0 = h0 - ((lgth / BLOCK_SIZE) + 1)*h_block_size;
    404         uint8_t * lb_h1 = h1 - ((lgth / BLOCK_SIZE) + 1)*h_block_size;
    405 
    406         // Comment out LOOKBACK_SIZE. Nigel's code *should* not produce this case.
    407 //      aos[LOOKBACK_SIZE + block_base + lb_spos].gid = h_table.lookup_or_insert(lb_buffer, lb_spos, lgth, lb_h0, lb_h1, lgth);
    408 //      aos[LOOKBACK_SIZE + block_base + lb_spos].lgth = lgth;
    409 
    410         if((lb_spos) < 0) {
    411             cerr << "ERROR" << endl;
    412             exit(1);
     381        ////////////////////////////////////////////////////////////////////
     382        // Start - Review boundary logic
     383        ////////////////////////////////////////////////////////////////////
     384
     385        BitBlock * lb_starts = starts;
     386        uint32_t blks = 0;
     387
     388        while(!ends_rscanner.is_done() && starts_rscanner.is_done()) {
     389        // TODO - FIX - BOUNDARY LOGIC FAILS for arbitrary lengths
     390
     391                starts_rscanner.init(--lb_starts);
     392                starts_rscanner.scan_to_next();
     393
     394                if(!starts_rscanner.is_done()) {
     395
     396                        ////////////////////////////////////////////////////////////////////
     397                        // Start - Review boundary logic
     398                        ////////////////////////////////////////////////////////////////////
     399                        spos = starts_rscanner.get_pos();
     400                        lgth = epos + (BLOCK_SIZE - spos) + (blks * BLOCK_SIZE);
     401
     402                        uint8_t * lb_buffer = buffer - ((lgth / BLOCK_SIZE) + 1)*BLOCK_SIZE;
     403                        int32_t lb_spos = blk_offset - ((BLOCK_SIZE - spos) + (blks * BLOCK_SIZE));//(BLOCK_SIZE - (-1*spos)) & (BLOCK_SIZE-1);
     404
     405                        uint8_t * lb_h0 = h0 - ((lgth / BLOCK_SIZE) + 1)*h_block_size;
     406                        uint8_t * lb_h1 = h1 - ((lgth / BLOCK_SIZE) + 1)*h_block_size;
     407
     408                        gid = h_table.lookup_or_insert(lb_buffer, lb_spos, lgth, lb_h0, lb_h1, h_lgth);
     409
     410                        symbols.gids[lb_spos] = gid;
     411                        ////////////////////////////////////////////////////////////////////
     412                        // End
     413                        ////////////////////////////////////////////////////////////////////
     414
     415                        break;
     416                }
     417
     418                blks++;
    413419        }
    414420
    415         //aos[block_base + lb_spos]->lgth = lgth;
    416 
    417 #ifdef ID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG
    418         print_symbol_debug(lb_buffer, lb_spos, ends_rscanner.get_pos(), lgth);
    419 #endif
    420 
    421     }
     421
     422
     423
     424
     425
     426
     427
     428
     429
     430
     431
     432
     433
     434        ////////////////////////////////////////////////////////////////////
     435        // End
     436        ////////////////////////////////////////////////////////////////////
    422437}
     438
     439/////////
    423440
    424441// Fixed Lengths - FORWARD SCAN LOGIC - Scan each BLOCK LSB to MSB (low to high memory address)
  • trunk/symbol_table/src/symbol_table.hpp

    r2001 r2014  
    6060
    6161template<class SYMBOL, class HASH_TABLE>
    62 IDISA_ALWAYS_INLINE void do_block(uint32_t blk_offset,
     62void do_block(uint32_t blk_offset,
    6363                                  HASH_TABLE & h_table,
    6464                                  BitBlock ends,
     
    6767                                  SYMBOL & symbols);
    6868
    69 
    70 //template<class HASH_TABLE>
    71 //IDISA_ALWAYS_INLINE void do_block(uint32_t blk, HASH_TABLE & h_table, BitBlock starts [], BitBlock ends [], uint8_t buffer [],
    72 //                                uint8_t h0 [], uint8_t h1 [], const uint32_t h_block_size);
     69template<class SYMBOL, class HASH_TABLE>
     70void do_block(uint32_t blk_offset,
     71                                  HASH_TABLE & h_table,
     72                                  BitBlock starts [], BitBlock ends [],
     73                                  uint8_t buffer [],
     74                                  uint8_t h0 [], uint8_t h1 [], const uint32_t h_lgth, const uint32_t h_block_size,
     75                                  SYMBOL & symbols);
    7376
    7477#endif // SYMBOL_TABLE_HPP
Note: See TracChangeset for help on using the changeset viewer.