Changeset 2024 for trunk/symbol_table


Ignore:
Timestamp:
Apr 14, 2012, 4:53:31 PM (7 years ago)
Author:
ksherdy
Message:

Added test, diff, clean targets.

Location:
trunk/symbol_table/src
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/symbol_table/src/Makefile

    r2014 r2024  
    1313endif
    1414
     15TEST_ROOT=../test
     16
    1517all: basis_bits.hpp buffer.hpp byte_pool.hpp  hash_strms.hpp  hash_table.hpp  id_group_strms.hpp  id_symbol_table.hpp  main.cpp  Makefile  marker_strms.hpp  symbol_table.hpp  transpose.hpp
    1618        $(CC) -o main main.cpp $(AFLAGS) #-DID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG #-DHASH_TABLE_HPP_DEBUG # -DBUFFER_PROFILING
    1719
     20test: main
     21        python $(TEST_ROOT)/run_tests.py main $(TEST_ROOT)/uniform
     22
     23diff:
     24        diff -rq --exclude=".svn" $(TEST_ROOT)/uniform $(TEST_ROOT)/uniform_rslts || exit 0
     25
    1826clean:
    19         rm -f main
     27        rm -Rf main $(TEST_ROOT)/uniform_rslts
     28
  • trunk/symbol_table/src/id_symbol_table.hpp

    r2017 r2024  
    336336}
    337337
    338 /////////
    339 
    340338// Variable Lengths, reverse scanner logic
     339// Precondition: A symbol end is marked iff a symbol start is marked within a buffer segment.
    341340template<class SYMBOL, class HASH_TABLE>
    342341void do_block(uint32_t blk_offset,
    343                                   HASH_TABLE & h_table,
    344                                   BitBlock starts [], BitBlock ends [],
    345                                   uint8_t buffer [],
    346                                   uint8_t h0 [], uint8_t h1 [], const uint32_t h_lgth, const uint32_t h_block_size,
    347                                   SYMBOL & symbols) {
     342                          HASH_TABLE & h_table,
     343                          BitBlock starts [], BitBlock ends [],
     344                          uint8_t buffer [],
     345                          uint8_t h0 [], uint8_t h1 [], const uint32_t h_lgth, const uint32_t h_block_size,
     346                          SYMBOL & symbols) {
     347
     348        BitBlock * starts_base = starts;
     349        uint8_t * h0_base = h0;
     350        uint8_t * h1_base = h1;
    348351
    349352        gid_type gid;
     
    351354        int32_t spos;
    352355        uint32_t lgth;
     356        uint32_t blk_count = 0;
    353357
    354358        ReverseScanner<BitBlock, scanword_t> ends_rscanner(ends);
     
    357361        ends_rscanner.scan_to_next();
    358362        epos = ends_rscanner.get_pos();
    359         if(epos > 0) { starts_rscanner.move_to(epos); }
    360         starts_rscanner.scan_to_next();
    361         spos = starts_rscanner.get_pos();
    362         lgth = epos - spos;
    363 
    364         while(!ends_rscanner.is_done() && !starts_rscanner.is_done()) {
     363
     364        while(!ends_rscanner.is_done()) {
    365365               
    366                 gid = h_table.lookup_or_insert(buffer, spos, lgth, h0, h1, h_lgth);
     366                starts_rscanner.move_to(epos);
     367                starts_rscanner.scan_to_next();
     368                spos = starts_rscanner.get_pos();
     369                lgth = epos - spos;
     370
     371                while(starts_rscanner.is_done()) { // boundary case
     372                    starts_base--;
     373                    h0_base--;
     374                    h1_base--;
     375
     376                    starts_rscanner.init(starts_base);
     377                    starts_rscanner.scan_to_next();
     378
     379                    if(!starts_rscanner.is_done()) { // found start
     380
     381                        lgth = epos + (BLOCK_SIZE - starts_rscanner.get_pos()) + (BLOCK_SIZE * blk_count);
     382                        spos = epos - lgth;
     383                        h0_base -= (h_block_size * blk_count);
     384                        h1_base -= (h_block_size * blk_count);
     385                        break;
     386                    }
     387                    blk_count++;
     388
     389                }
     390
     391                gid = h_table.lookup_or_insert(buffer, spos, lgth, h0_base, h1_base, h_lgth);
    367392                symbols.gids[blk_offset + spos] = gid;
    368393
     
    373398                ends_rscanner.scan_to_next();
    374399                epos = ends_rscanner.get_pos();
    375                 if(epos > 0) { starts_rscanner.move_to(epos); }
    376                 starts_rscanner.scan_to_next();
    377                 spos = starts_rscanner.get_pos();
    378                 lgth = epos - spos;
    379         }
    380 
    381         ////////////////////////////////////////////////////////////////////
    382         // Start - Review boundary logic
    383         ////////////////////////////////////////////////////////////////////
    384 
    385         BitBlock * lb_starts = starts;
    386         uint32_t blks = 0;
    387 
    388         while(!ends_rscanner.is_done() && starts_rscanner.is_done()) {
    389         // TODO - FIX - BOUNDARY LOGIC FAILS for arbitrary lengths
    390 
    391                 starts_rscanner.init(--lb_starts);
    392                 starts_rscanner.scan_to_next();
    393 
    394                 if(!starts_rscanner.is_done()) {
    395 
    396                         spos = starts_rscanner.get_pos();
    397                         lgth = epos + (BLOCK_SIZE - spos) + (blks * BLOCK_SIZE);
    398 
    399                         uint8_t * lb_buffer = buffer - ((lgth / BLOCK_SIZE) + 1)*BLOCK_SIZE;
    400                         int32_t lb_spos = (BLOCK_SIZE - (-1*spos)) & (BLOCK_SIZE-1);
    401 
    402                         uint8_t * lb_h0 = h0 - ((lgth / BLOCK_SIZE) + 1)*h_block_size;
    403                         uint8_t * lb_h1 = h1 - ((lgth / BLOCK_SIZE) + 1)*h_block_size;
    404 
    405                         gid = h_table.lookup_or_insert(lb_buffer, lb_spos, lgth, lb_h0, lb_h1, h_lgth);
    406 
    407                         symbols.gids[blk_offset + epos - lgth] = gid;
    408 
    409                         return; // found start
    410                 }
    411                 blks++;
    412400        }
    413401}
    414402
    415 ////////////////////////////////////////////////////////////////////
    416 // End
    417 ////////////////////////////////////////////////////////////////////
    418 
    419 
    420 // Fixed Lengths - FORWARD SCAN LOGIC - Scan each BLOCK LSB to MSB (low to high memory address)
    421 //template<class HASH_TABLE>
    422 //IDISA_ALWAYS_INLINE void do_block(HASH_TABLE & h_table, BitBlock ends, uint8_t buffer [], const uint32_t lgth,
    423 //                                uint8_t h0 [], uint8_t h1 [], const uint32_t h_lgth, const uint32_t h_block_size) {
    424 
    425 //    int32_t spos;
    426 //    int32_t epos;
    427 //    ForwardScanner<BitBlock, scanword_t> fscanner(&ends);
    428 
    429 //    epos = fscanner.scan_to_next();
    430 //    spos = (epos - lgth);
    431 
    432 //    if((epos != -1) && (spos < 0)) { // block boundary case
    433 
    434 //      assert(lgth<=LOOKBACK_SIZE); // segment boundary
    435 
    436 //      uint8_t * lb_buffer = buffer - ((lgth / BLOCK_SIZE) + 1)*BLOCK_SIZE;
    437 //      int32_t lb_spos = BLOCK_SIZE - ((-1*spos) & (BLOCK_SIZE-1));
    438 
    439 //      uint8_t * lb_h0 = h0 - ((lgth / BLOCK_SIZE) + 1)*h_block_size;
    440 //      uint8_t * lb_h1 = h1 - ((lgth / BLOCK_SIZE) + 1)*h_block_size;
    441 
    442 //      h_table.lookup_or_insert(lb_buffer, lb_spos, lgth,
    443 //                               lb_h0, lb_h1, h_lgth);
    444 
    445 //#ifdef ID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG
    446 //      cout << "Symbol(";
    447 //      cout << "End:" << epos;
    448 //      cout << ",Start:" << lb_spos;
    449 //      cout << ",Text:'" << string((char *)&(lb_buffer[lb_spos]), lgth);
    450 //      cout << "')" << endl;
    451 //#endif
    452 //      epos = fscanner.scan_to_next();
    453 //      spos = (epos - lgth);
    454 
    455 //    }
    456 
    457 //    while(epos != -1) {
    458 
    459 //      h_table.lookup_or_insert(buffer, spos, lgth,
    460 //                               h0, h1, h_lgth);
    461 
    462 //#ifdef ID_SYMBOL_TABLE_TEMPLATE_HPP_DEBUG
    463 //      cout << "Symbol(";
    464 //      cout << "End:" << epos;
    465 //      cout << ",Start:" << spos;
    466 //      cout << ",Text:'" << string((char *)&(buffer[spos]), lgth);
    467 //      cout << "')" << endl;
    468 //#endif
    469 //      epos = fscanner.scan_to_next();
    470 //      spos = (epos - lgth);
    471 
    472 //    }
    473 //}
    474 
    475403#endif // ID_SYMBOL_TABLE_TEMPLATE_HPP
    476404
Note: See TracChangeset for help on using the changeset viewer.