Ignore:
Timestamp:
Nov 15, 2011, 1:19:24 AM (8 years ago)
Author:
vla24
Message:

SymbolTable?: updated templates implementation to use BitBlockForwardIterator?. Fixed block boundary case handling for div2 grouping strategy.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • proto/SymbolTable/symtab_pbgs_div_template.cpp

    r1648 r1684  
    3434PBGSDivSymbolTable pbgs_symbol_table;
    3535
    36 /* StreamScan & Post Process Declarations */
    37 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    38 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    39 
    4036static inline int NameStrt_check(int pos);
    4137static inline int Name_check(int pos);
     
    5147static inline void s2p_do_block(BytePack U8[], Basis_bits & basis_bits);
    5248static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    53 static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, int chars_avail);
     49static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail);
     50
    5451void do_process(FILE *infile, FILE *outfile);
     52
     53static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     54static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
     55template <int L> static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
     56
    5557static inline int ScanForwardPos(BitBlock * block, int pos);
    5658static inline int compute_hash_value (int lgth, int start);
     
    128130    if (bitblock_has_bit(temp))
    129131    {
    130 #if DEBUG
    131         printf ("There is a 1 bit in the block\n");
    132 #endif
    133132        return count_forward_zeroes (temp);
    134133    }
    135     else
    136     {
    137 #if DEBUG
    138         printf ("There is no more 1 bit in the block. pos: %i\n", pos);
    139 #endif
    140         //handle boundary case
    141         block_boundary_case = true;
    142         last_elem_start = pos - BLOCK_SIZE;
    143 #if DEBUG
    144         printf ("last_elem_start: %i\n", last_elem_start);
    145 #endif
    146         return 0;
    147     }
    148 }
    149 
    150 /* StreamScan & Post Process Definitions */
    151 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    152 
    153         int blk;
    154         int block_pos = 0;
    155         int pos;
    156 
    157         for (blk = 0; blk < blk_count; blk++) {
    158                 ScanBlock s = stream[blk];
    159                 while(s) {
    160                         pos = (cfzl(s) + block_pos);
    161                         int code = (ProcessPos(pos));
    162                         if (code) {
    163                                 *error_pos_in_block = pos;
    164                                 return code; // error code
    165                         }
    166                         s = s & (s-1);  // clear rightmost bit.
    167                 }
    168                 block_pos += 8 * sizeof(ScanBlock);
    169         }
    170         return 0;
     134    return 0;
    171135}
    172136
     
    180144// length in [1,16]
    181145template <int L>
    182 static inline int ElemEnd_grouping(int end_pos) {
    183     int end = block_base + end_pos;
     146static inline int ElemEnd_grouping(int end) {
    184147    int start = end - L;
    185148    int hashvalue = compute_hash_value(L, start - block_base);
     
    197160
    198161// length > 16
    199 static inline int ElemStart_grouping(int start_pos, int lgth) {
    200     int start = start_pos + block_base;
     162static inline int ElemStart_grouping(int start, int lgth) {
    201163    int hashvalue = compute_hash_value(lgth, start - block_base);
    202164    int gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
     
    211173}
    212174
    213 // L = 2, pass in bitstream for symbols length [1,2]
    214 // L = 4, pass in bitstream for symbols length [3,4]
    215 // L = 6, pass in bitstream for symbols length [5,6]
    216 // L = 8, pass in bitstream for symbols length [7,8]
    217 // L = 10, pass in bitstream for symbols length [9,10]
    218 // L = 12, pass in bitstream for symbols length [11,12]
    219 // L = 14, pass in bitstream for symbols length [13,14]
    220 // L = 16, pass in bitstream for symbols length [15,16]
    221 // L = 17, pass in bitstream for symbols length longer than 16
    222 template <int L>
    223 static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count) {
    224     int blk;
    225     int block_pos = 0;
    226     for (blk = 0; blk < blk_count; blk++) {
    227         ScanBlock s = stream[blk];
    228         while(s) {
    229             int end_pos = cfzl(s) + block_pos;
    230             ElemEnd_grouping<L>(end_pos);
    231             s = s & (s-1);  // clear rightmost bit.
    232         }
    233         block_pos += 8 * sizeof(ScanBlock);
    234     }
    235     return 0;
    236 }
    237 
    238 template <>
    239 inline int StreamScanLengthGrouping<17>(ScanBlock * stream, int blk_count) {
    240     int blk;
    241     int block_pos = 0;
    242     for (blk = 0; blk < blk_count; blk++) {
    243         ScanBlock s = stream[blk];
    244         while(s) {
    245             int start_pos = cfzl(s) + block_pos;
    246             int end_pos = ScanForwardPos (&elem_ends, start_pos);
    247             if (end_pos)
    248             {
    249                 ElemStart_grouping(start_pos - 16, end_pos - start_pos + 16);
    250             }
    251             s = s & (s-1);  // clear rightmost bit.
    252         }
    253         block_pos += 8 * sizeof(ScanBlock);
    254     }
    255     return 0;
    256 }
    257 
    258175static inline int NameStrt_check(int pos) {
    259         int block_pos = block_base + pos;
    260         if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[block_pos]) == 0){
     176        if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[pos]) == 0){
    261177              return XMLTestSuiteError::NAME_START;
    262178        }
     
    265181
    266182static inline int Name_check(int pos) {
    267         int block_pos = block_base + pos;
    268         if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[block_pos]) == 0){
     183        if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[pos]) == 0){
    269184                  return XMLTestSuiteError::NAME;
    270185        }
     
    272187}
    273188
    274 static inline int PIName_check(int pos) {
    275         int block_pos = block_base + pos;
    276         int file_pos = block_pos+buffer_base;
    277         if (at_XxMmLll<ASCII>((unsigned char*)&source[block_pos]) && (source[block_pos+3]=='?' || source[block_pos+3]<= ' ')) {
     189static inline int PIName_check(int pos, int file_pos) {
     190        if (at_XxMmLll<ASCII>((unsigned char*)&source[pos]) && (source[pos+3]=='?' || source[pos+3]<= ' ')) {
    278191              // "<?xml" legal at start of file.
    279192              if (!((file_pos == 2) && at_XmlDecl_start<ASCII>((unsigned char*)&source[0]))) {
     
    285198
    286199static inline int CD_check(int pos) {
    287         int block_pos = block_base + pos;
    288         if (!at_CDATA1<ASCII>((unsigned char*)&source[block_pos])){
     200        if (!at_CDATA1<ASCII>((unsigned char*)&source[pos])){
    289201                  return XMLTestSuiteError::CDATA;
    290202        }
     
    293205
    294206static inline int GenRef_check(int pos) {
    295         int block_pos = block_base + pos;
    296         unsigned char* s = (unsigned char*)&source[block_pos];
     207        unsigned char* s = (unsigned char*)&source[pos];
    297208        if (!(at_Ref_gt<ASCII>(s)||at_Ref_lt<ASCII>(s)||at_Ref_amp<ASCII>(s)||at_Ref_quot<ASCII>(s)||at_Ref_apos<ASCII>(s))){
    298209              return XMLTestSuiteError::UNDEFREF;
     
    302213
    303214static inline int HexRef_check(int pos) {
    304         int block_pos = block_base + pos;
    305         unsigned char* s = (unsigned char*)&source[block_pos];
     215        unsigned char* s = (unsigned char*)&source[pos];
    306216        int ch_val = 0;
    307217        while(at_HexDigit<ASCII>(s)){
     
    322232
    323233static inline int DecRef_check(int pos) {
    324         int block_pos = block_base + pos;
    325         unsigned char* s = (unsigned char*)&source[block_pos];
     234        unsigned char* s = (unsigned char*)&source[pos];
    326235        int ch_val = 0;
    327236        while(at_HexDigit<ASCII>(s)){
     
    342251
    343252static inline int AttRef_check(int pos) {
    344         int block_pos = block_base + pos;
    345         unsigned char* s = (unsigned char*)&source[block_pos];
     253        unsigned char* s = (unsigned char*)&source[pos];
    346254        int ch_val = 0;
    347255        if(s[0]=='#'){
     
    370278}
    371279
     280static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     281
     282        BitBlockForwardIterator end;
     283        int pos, block_pos;
     284
     285        while(start != end) {
     286
     287                block_pos = block_base + *start;
     288                int rv = is_valid(block_pos);
     289
     290                if (rv) {
     291                        int error_line, error_column;
     292                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     293                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     294                        exit(-1);
     295                }
     296                start++;
     297        }
     298}
     299
     300static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     301
     302        BitBlockForwardIterator end;
     303        int pos, block_pos, file_pos;
     304
     305        while(start != end) {
     306
     307                block_pos = block_base + *start;
     308                file_pos = block_pos+buffer_base;
     309
     310
     311                int rv = is_valid(block_pos, file_pos);
     312
     313                if (rv) {
     314                        int error_line, error_column;
     315                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     316                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     317                        exit(-1);
     318                }
     319                start++;
     320        }
     321}
     322
     323// L = 2, pass in bitstream for symbols length [1,2]
     324// L = 4, pass in bitstream for symbols length [3,4]
     325// L = 6, pass in bitstream for symbols length [5,6]
     326// L = 8, pass in bitstream for symbols length [7,8]
     327// L = 10, pass in bitstream for symbols length [9,10]
     328// L = 12, pass in bitstream for symbols length [11,12]
     329// L = 14, pass in bitstream for symbols length [13,14]
     330// L = 16, pass in bitstream for symbols length [15,16]
     331// L = 17, pass in bitstream for symbols length longer than 16
     332template <int L>
     333static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
     334
     335    BitBlockForwardIterator end;
     336    int end_pos;
     337
     338    while(start != end) {
     339        end_pos = *start;
     340        ElemEnd_grouping<L>(end_pos + block_base);
     341        start++;
     342    }
     343}
     344
     345template <>
     346inline void validate_block_length_grouping<17>(BitBlockForwardIterator & start, int block_base) {
     347
     348    BitBlockForwardIterator end;
     349    int start_pos, end_pos;
     350
     351    while(start != end) {
     352        start_pos = *start;
     353        end_pos = ScanForwardPos (&elem_ends, start_pos);
     354        if (end_pos)
     355        {
     356            ElemStart_grouping(start_pos - 16 + block_base, end_pos - start_pos + 16);
     357        }
     358        else
     359        {
     360#if DEBUG
     361            printf ("There is no more 1 bit in the block. pos: %i | sym: %c%c[%c]\n", start_pos,
     362                    source[start_pos + block_base-2], source[start_pos + block_base-1], source[start_pos + block_base]);
     363#endif
     364            //handle boundary case
     365            block_boundary_case = true;
     366            last_elem_start = start_pos - 16 - BLOCK_SIZE;
     367#if DEBUG
     368            printf ("last_elem_start: %i\n", last_elem_start);
     369#endif
     370        }
     371        start++;
     372    }
     373}
     374
    372375static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail){
    373376
    374377    tracker.StoreNewlines(lex.LF);
    375     int rv, error_pos_in_block, error_line, error_column;
     378
    376379    elem_ends = tag_Callouts.ElemName_ends;
    377380    hashvalues[1] = hash_data.Hash_value;
     
    396399    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_1_to_2) )
    397400    {
    398         StreamScanLengthGrouping<2>((ScanBlock *) &tag_Callouts.ElemName_ends_1_to_2, sizeof(BitBlock)/sizeof(ScanBlock));
     401        BitBlockForwardIterator iter_length_grouping_2(&tag_Callouts.ElemName_ends_1_to_2);
     402        validate_block_length_grouping<2>(iter_length_grouping_2, block_base);
    399403    }
    400404
    401405    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_3_to_4) )
    402406    {
    403         StreamScanLengthGrouping<4>((ScanBlock *) &tag_Callouts.ElemName_ends_3_to_4, sizeof(BitBlock)/sizeof(ScanBlock));
     407        BitBlockForwardIterator iter_length_grouping_4(&tag_Callouts.ElemName_ends_3_to_4);
     408        validate_block_length_grouping<4>(iter_length_grouping_4, block_base);
    404409    }
    405410
    406411    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_5_to_6) )
    407412    {
    408         StreamScanLengthGrouping<6>((ScanBlock *) &tag_Callouts.ElemName_ends_5_to_6, sizeof(BitBlock)/sizeof(ScanBlock));
     413        BitBlockForwardIterator iter_length_grouping_6(&tag_Callouts.ElemName_ends_5_to_6);
     414        validate_block_length_grouping<6>(iter_length_grouping_6, block_base);
    409415    }
    410416
    411417    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_7_to_8) )
    412418    {
    413         StreamScanLengthGrouping<8>((ScanBlock *) &tag_Callouts.ElemName_ends_7_to_8, sizeof(BitBlock)/sizeof(ScanBlock));
     419        BitBlockForwardIterator iter_length_grouping_8(&tag_Callouts.ElemName_ends_7_to_8);
     420        validate_block_length_grouping<8>(iter_length_grouping_8, block_base);
    414421    }
    415422
    416423    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_9_to_10) )
    417424    {
    418         StreamScanLengthGrouping<10>((ScanBlock *) &tag_Callouts.ElemName_ends_9_to_10, sizeof(BitBlock)/sizeof(ScanBlock));
     425        BitBlockForwardIterator iter_length_grouping_10(&tag_Callouts.ElemName_ends_9_to_10);
     426        validate_block_length_grouping<10>(iter_length_grouping_10, block_base);
    419427    }
    420428
    421429    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_11_to_12) )
    422430    {
    423         StreamScanLengthGrouping<12>((ScanBlock *) &tag_Callouts.ElemName_ends_11_to_12, sizeof(BitBlock)/sizeof(ScanBlock));
     431        BitBlockForwardIterator iter_length_grouping_12(&tag_Callouts.ElemName_ends_11_to_12);
     432        validate_block_length_grouping<12>(iter_length_grouping_12, block_base);
    424433    }
    425434
    426435    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_13_to_14) )
    427436    {
    428         StreamScanLengthGrouping<14>((ScanBlock *) &tag_Callouts.ElemName_ends_13_to_14, sizeof(BitBlock)/sizeof(ScanBlock));
     437        BitBlockForwardIterator iter_length_grouping_14(&tag_Callouts.ElemName_ends_13_to_14);
     438        validate_block_length_grouping<14>(iter_length_grouping_14, block_base);
    429439    }
    430440
    431441    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_15_to_16) )
    432442    {
    433         StreamScanLengthGrouping<16>((ScanBlock *) &tag_Callouts.ElemName_ends_15_to_16, sizeof(BitBlock)/sizeof(ScanBlock));
     443        BitBlockForwardIterator iter_length_grouping_16(&tag_Callouts.ElemName_ends_15_to_16);
     444        validate_block_length_grouping<16>(iter_length_grouping_16, block_base);
    434445    }
    435446
    436447    if ( bitblock_has_bit(tag_Callouts.ElemName_remaining_ends) )
    437448    {
    438         StreamScanLengthGrouping<17>((ScanBlock *) &tag_Callouts.ElemName_remaining_ends, sizeof(BitBlock)/sizeof(ScanBlock));
     449        BitBlockForwardIterator iter_length_grouping_remaining(&tag_Callouts.ElemName_remaining_ends);
     450        validate_block_length_grouping<17>(iter_length_grouping_remaining, block_base);
    439451    }
    440452
     
    442454    memmove (&hashvalues[0], &hashvalues[1], 16);
    443455
    444 
    445456    if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
    446       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), NameStrt_check, &error_pos_in_block);
    447       if (rv) {
    448               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    449               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    450               exit(-1);
    451       }
    452 
    453       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_names, sizeof(BitBlock)/sizeof(ScanBlock), Name_check, &error_pos_in_block);
    454       if (rv) {
    455               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    456               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    457               exit(-1);
    458       }
    459     }
    460 
     457        BitBlockForwardIterator iter_NameStrt_check(&check_streams.non_ascii_name_starts);
     458        validate_block(iter_NameStrt_check, block_base, NameStrt_check);
     459        BitBlockForwardIterator iter_Name_check(&check_streams.non_ascii_names);
     460        validate_block(iter_Name_check, block_base, Name_check);
     461    }
    461462    if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
    462       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.PI_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), PIName_check, &error_pos_in_block);
    463       if (rv) {
    464               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    465               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    466               exit(-1);
    467       }
    468     }
    469 
     463        BitBlockForwardIterator iter_PI_name_starts(&(ctCDPI_Callouts.PI_name_starts));
     464        validate_block(iter_PI_name_starts, block_base, buffer_base, PIName_check);
     465    }
    470466    if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
    471       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.CD_starts, sizeof(BitBlock)/sizeof(ScanBlock), CD_check, &error_pos_in_block);
    472       if (rv) {
    473               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    474               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    475               exit(-1);
    476       }
    477     }
    478 
    479     if (bitblock_has_bit(ref_Callouts.GenRef_starts)){
    480       rv = StreamScan((ScanBlock *) &ref_Callouts.GenRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), GenRef_check, &error_pos_in_block);
    481       if (rv) {
    482               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    483               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    484               exit(-1);
    485       }
    486     }
    487 
    488     if (bitblock_has_bit(ref_Callouts.DecRef_starts)){
    489       rv = StreamScan((ScanBlock *) &ref_Callouts.DecRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), DecRef_check, &error_pos_in_block);
    490       if (rv) {
    491               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    492               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    493               exit(-1);
    494       }
    495     }
    496 
    497     if (bitblock_has_bit(ref_Callouts.HexRef_starts)){
    498       rv = StreamScan((ScanBlock *) &ref_Callouts.HexRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), HexRef_check, &error_pos_in_block);
    499       if (rv) {
    500               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    501               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    502               exit(-1);
    503       }
    504     }
    505 
    506     if (bitblock_has_bit(check_streams.att_refs)){
    507       rv = StreamScan((ScanBlock *) &check_streams.att_refs, sizeof(BitBlock)/sizeof(ScanBlock), AttRef_check, &error_pos_in_block);
    508       if (rv) {
    509               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    510               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    511               exit(-1);
    512       }
     467        BitBlockForwardIterator iter_CD_check(&ctCDPI_Callouts.CD_starts);
     468        validate_block(iter_CD_check, block_base, CD_check);
     469    }
     470    if(bitblock_has_bit(ref_Callouts.GenRef_starts)){
     471        BitBlockForwardIterator iter_GenRef_check(&ref_Callouts.GenRef_starts);
     472        validate_block(iter_GenRef_check, block_base, GenRef_check);
     473    }
     474    if(bitblock_has_bit(ref_Callouts.DecRef_starts)){
     475        BitBlockForwardIterator iter_DecRef_check(&ref_Callouts.DecRef_starts);
     476        validate_block(iter_DecRef_check, block_base, DecRef_check);
     477    }
     478    if(bitblock_has_bit(ref_Callouts.HexRef_starts)){
     479        BitBlockForwardIterator iter_HexRef_check(&ref_Callouts.HexRef_starts);
     480        validate_block(iter_HexRef_check, block_base, HexRef_check);
     481    }
     482    if(bitblock_has_bit(check_streams.att_refs)){
     483        BitBlockForwardIterator iter_AttRef_check(&check_streams.att_refs);
     484        validate_block(iter_AttRef_check, block_base, AttRef_check);
    513485    }
    514486
    515487    if(error_tracker.Has_Noted_Error()){
    516             tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
    517             ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
    518             exit(-1);
     488        int error_line, error_column;
     489        tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
     490        ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
     491        exit(-1);
    519492    }
    520493
Note: See TracChangeset for help on using the changeset viewer.