Changeset 1684 for proto


Ignore:
Timestamp:
Nov 15, 2011, 1:19:24 AM (8 years ago)
Author:
vla24
Message:

SymbolTable?: updated templates implementation to use BitBlockForwardIterator?. Fixed block boundary case handling for div2 grouping strategy.

Location:
proto/SymbolTable
Files:
16 edited

Legend:

Unmodified
Added
Removed
  • proto/SymbolTable/automate-build_idisa.sh

    r1683 r1684  
    1 make symtab_pbgs_id
     1make symtab_pbgs_div
    22cd src
    33make all
    4 ./xmlwf ../test/test_files/soap.xml
     4./xmlwf ../test/test_files/soap_div2.xml
    55cd ..
    66
  • proto/SymbolTable/dictionary_hash_template.cpp

    r1666 r1684  
    2020char * source;
    2121LineColTracker tracker;
    22 TagMatcher matcher;
    23 ErrorTracker error_tracker;
    2422BitBlock EOF_mask = simd<1>::constant<1>();
    2523
     
    3836static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    3937static inline void postprocess_do_block(Dictionary& dictionary);
     38static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     39static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
     40
    4041static inline void do_symbol_table_lookup();
    4142
     
    104105}
    105106
    106 
    107 /* StreamScan & Post Process Definitions */
    108 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    109 
    110         int blk;
    111         int block_pos = 0;
    112         int pos;
    113 
    114         for (blk = 0; blk < blk_count; blk++) {
    115                 ScanBlock s = stream[blk];
    116                 while(s) {
    117                         pos = (cfzl(s) + block_pos);
    118                         int code = (ProcessPos(pos));
    119                         if (code) {
    120                                 *error_pos_in_block = pos;
    121                                 return code; // error code
    122                         }
    123                         s = s & (s-1);  // clear rightmost bit.
    124                 }
    125                 block_pos += 8 * sizeof(ScanBlock);
    126         }
     107static inline int ElemStrt_check(int pos) {
     108        elem_starts_buf.push(buffer_base + pos);
    127109        return 0;
    128110}
    129111
    130 static inline int ElemStrt_check(int pos) {
    131         int block_pos = block_base + pos;
    132         elem_starts_buf.push(buffer_base + block_pos);
    133         return 0;
    134 }
    135 
    136112static inline int ElemEnd_check(int pos) {
    137         int block_pos = block_base + pos;
    138         elem_ends_buf.push(buffer_base + block_pos);
    139         return 0;
     113        elem_ends_buf.push(buffer_base + pos);
     114        return 0;
    140115}
    141116
     
    165140}
    166141
     142static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     143
     144        BitBlockForwardIterator end;
     145        int pos, block_pos;
     146
     147        while(start != end) {
     148
     149                block_pos = block_base + *start;
     150                int rv = is_valid(block_pos);
     151
     152                if (rv) {
     153                        int error_line, error_column;
     154                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     155                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     156                        exit(-1);
     157                }
     158                start++;
     159        }
     160}
     161
     162static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     163
     164        BitBlockForwardIterator end;
     165        int pos, block_pos, file_pos;
     166
     167        while(start != end) {
     168
     169                block_pos = block_base + *start;
     170                file_pos = block_pos+buffer_base;
     171
     172
     173                int rv = is_valid(block_pos, file_pos);
     174
     175                if (rv) {
     176                        int error_line, error_column;
     177                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     178                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     179                        exit(-1);
     180                }
     181                start++;
     182        }
     183}
     184
    167185static inline void postprocess_do_block(Dictionary& dictionary){
    168 
    169     int error_pos_in_block;
    170186
    171187    if ( bitblock_has_bit(dictionary.Word_starts))
    172188    {
    173         StreamScan((ScanBlock *) &dictionary.Word_starts, sizeof(BitBlock)/sizeof(ScanBlock), ElemStrt_check, &error_pos_in_block);
     189        BitBlockForwardIterator iter_length_grouping_starts(&dictionary.Word_starts);
     190        validate_block(iter_length_grouping_starts, block_base, ElemStrt_check);
    174191    }
    175192
    176193    if ( bitblock_has_bit(dictionary.Word_ends) )
    177194    {
    178         StreamScan((ScanBlock *) &dictionary.Word_ends, sizeof(BitBlock)/sizeof(ScanBlock), ElemEnd_check, &error_pos_in_block);
    179     }
    180 
     195        BitBlockForwardIterator iter_length_grouping_ends(&dictionary.Word_ends);
     196        validate_block(iter_length_grouping_ends, block_base, ElemEnd_check);
     197    }
    181198    do_symbol_table_lookup();
    182199}
  • proto/SymbolTable/dictionary_identity_template.cpp

    r1666 r1684  
     1//#define USE_ITER
     2
    13#include "../symtab_global.h"
    24#include <pbgs_identity_symbol_table.h>
     
    2022int buffer_last;
    2123char * source;
    22 LineColTracker tracker;
    23 TagMatcher matcher;
    24 ErrorTracker error_tracker;
    2524BitBlock EOF_mask = simd<1>::constant<1>();
    2625
     
    3332PBGSIdentitySymbolTable pbgs_symbol_table;
    3433
    35 /* StreamScan & Post Process Declarations */
    36 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    37 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    38 
    3934@global
    4035
     
    4237static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    4338static inline void postprocess_do_block(Dictionary& dictionary, Hash_data hash_data);
     39
     40void do_process(FILE *infile, FILE *outfile);
     41
     42static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
     43
    4444static inline int ElemStart_grouping(int start_pos, int L) ;
    4545static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count);
    4646static inline int ScanForwardPos(BitBlock * block, int pos);
    4747static inline int compute_hash_value (int lgth, int start);
    48 
    49 void do_process(FILE *infile, FILE *outfile);
    5048
    5149int main(int argc, char * argv[]) {
     
    129127}
    130128
    131 /* StreamScan & Post Process Definitions */
    132 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    133 
    134         int blk;
    135         int block_pos = 0;
    136         int pos;
    137 
    138         for (blk = 0; blk < blk_count; blk++) {
    139                 ScanBlock s = stream[blk];
    140                 while(s) {
    141                         pos = (cfzl(s) + block_pos);
    142                         int code = (ProcessPos(pos));
    143                         if (code) {
    144                                 *error_pos_in_block = pos;
    145                                 return code; // error code
    146                         }
    147                         s = s & (s-1);  // clear rightmost bit.
    148                 }
    149                 block_pos += 8 * sizeof(ScanBlock);
    150         }
    151         return 0;
    152 }
    153 
    154 
    155129static inline int compute_hash_value (int lgth, int start)
    156130{
     
    161135
    162136static inline int ElemStart_grouping(int start_pos, int L) {
    163     int start = block_base + start_pos;
    164     int end = start + L;
    165     int hashvalue = compute_hash_value(L, start - block_base);
     137    int hashvalue = compute_hash_value(L, start_pos - block_base);
    166138    int gid = 0;
    167139
     
    169141    {
    170142    case 1:
    171         gid = pbgs_symbol_table.Lookup_or_Insert_Name<1>(source + start, hashvalue);
    172         break;
     143        gid = pbgs_symbol_table.Lookup_or_Insert_Name<1>(source + start_pos, hashvalue);
     144        break;
    173145    case 2:
    174         gid = pbgs_symbol_table.Lookup_or_Insert_Name<2>(source + start, hashvalue);
    175         break;
     146        gid = pbgs_symbol_table.Lookup_or_Insert_Name<2>(source + start_pos, hashvalue);
     147        break;
    176148    case 3:
    177         gid = pbgs_symbol_table.Lookup_or_Insert_Name<3>(source + start, hashvalue);
    178         break;
     149        gid = pbgs_symbol_table.Lookup_or_Insert_Name<3>(source + start_pos, hashvalue);
     150        break;
    179151    case 4:
    180         gid = pbgs_symbol_table.Lookup_or_Insert_Name<4>(source + start, hashvalue);
    181         break;
     152        gid = pbgs_symbol_table.Lookup_or_Insert_Name<4>(source + start_pos, hashvalue);
     153        break;
    182154    case 5:
    183         gid = pbgs_symbol_table.Lookup_or_Insert_Name<5>(source + start, hashvalue);
    184         break;
     155        gid = pbgs_symbol_table.Lookup_or_Insert_Name<5>(source + start_pos, hashvalue);
     156        break;
    185157    case 6:
    186         gid = pbgs_symbol_table.Lookup_or_Insert_Name<6>(source + start, hashvalue);
    187         break;
     158        gid = pbgs_symbol_table.Lookup_or_Insert_Name<6>(source + start_pos, hashvalue);
     159        break;
    188160    case 7:
    189         gid = pbgs_symbol_table.Lookup_or_Insert_Name<7>(source + start, hashvalue);
    190         break;
     161        gid = pbgs_symbol_table.Lookup_or_Insert_Name<7>(source + start_pos, hashvalue);
     162        break;
    191163    case 8:
    192         gid = pbgs_symbol_table.Lookup_or_Insert_Name<8>(source + start, hashvalue);
    193         break;
     164        gid = pbgs_symbol_table.Lookup_or_Insert_Name<8>(source + start_pos, hashvalue);
     165        break;
    194166    case 9:
    195         gid = pbgs_symbol_table.Lookup_or_Insert_Name<9>(source + start, hashvalue);
    196         break;
     167        gid = pbgs_symbol_table.Lookup_or_Insert_Name<9>(source + start_pos, hashvalue);
     168        break;
    197169    case 10:
    198         gid = pbgs_symbol_table.Lookup_or_Insert_Name<10>(source + start, hashvalue);
    199         break;
     170        gid = pbgs_symbol_table.Lookup_or_Insert_Name<10>(source + start_pos, hashvalue);
     171        break;
    200172    case 11:
    201         gid = pbgs_symbol_table.Lookup_or_Insert_Name<11>(source + start, hashvalue);
    202         break;
     173        gid = pbgs_symbol_table.Lookup_or_Insert_Name<11>(source + start_pos, hashvalue);
     174        break;
    203175    case 12:
    204         gid = pbgs_symbol_table.Lookup_or_Insert_Name<12>(source + start, hashvalue);
    205         break;
     176        gid = pbgs_symbol_table.Lookup_or_Insert_Name<12>(source + start_pos, hashvalue);
     177        break;
    206178    case 13:
    207         gid = pbgs_symbol_table.Lookup_or_Insert_Name<13>(source + start, hashvalue);
    208         break;
     179        gid = pbgs_symbol_table.Lookup_or_Insert_Name<13>(source + start_pos, hashvalue);
     180        break;
    209181    case 14:
    210         gid = pbgs_symbol_table.Lookup_or_Insert_Name<14>(source + start, hashvalue);
    211         break;
     182        gid = pbgs_symbol_table.Lookup_or_Insert_Name<14>(source + start_pos, hashvalue);
     183        break;
    212184    case 15:
    213         gid = pbgs_symbol_table.Lookup_or_Insert_Name<15>(source + start, hashvalue);
    214         break;
     185        gid = pbgs_symbol_table.Lookup_or_Insert_Name<15>(source + start_pos, hashvalue);
     186        break;
    215187    case 16:
    216         gid = pbgs_symbol_table.Lookup_or_Insert_Name<16>(source + start, hashvalue);
    217         break;
     188        gid = pbgs_symbol_table.Lookup_or_Insert_Name<16>(source + start_pos, hashvalue);
     189        break;
    218190    default:
    219         gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, L);
    220         break;
     191        gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start_pos, hashvalue, L);
     192        break;
    221193    }
    222194
    223195    gids.push_back(gid);
    224196#if DEBUG
     197    int end = start_pos + L;
    225198    char* symbol = new char[L+1];
    226     strncpy ( symbol, source + start, L );
     199    strncpy ( symbol, source + start_pos, L );
    227200    symbol[L] ='\0';
    228     printf ("%s | L: %i | start: %i[%i] | end: %i[%i] | gid: %i | hashvalue: %i | symbol: %s\n", __FUNCTION__, L, start, start-buffer_base, end, end-buffer_base, gid, hashvalue, symbol );
     201    printf ("%s | L: %i | start: %i[%i] | end: %i[%i] | gid: %i | hashvalue: %i | symbol: %s\n", __FUNCTION__, L, start_pos, start_pos-buffer_base, end, end-buffer_base, gid, hashvalue, symbol );
    229202    delete symbol; symbol = 0;
    230203#endif
     
    232205}
    233206
    234 static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count) {
    235     int blk;
    236     int block_pos = 0;
    237     for (blk = 0; blk < blk_count; blk++) {
    238         ScanBlock s = stream[blk];
    239         while(s) {
    240             int start_pos = cfzl(s) + block_pos;
    241             int end_pos = ScanForwardPos (&elem_ends, start_pos);
    242             if (end_pos)
    243             {
    244                 ElemStart_grouping(start_pos, end_pos - start_pos);
    245             }
    246             s = s & (s-1);  // clear rightmost bit.
    247         }
    248         block_pos += 8 * sizeof(ScanBlock);
    249     }
    250     return 0;
     207inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
     208
     209    BitBlockForwardIterator end;
     210    int start_pos, end_pos, lgth;
     211
     212    while(start != end) {
     213        start_pos = *start;
     214        end_pos = ScanForwardPos (&elem_ends, start_pos);
     215        if (end_pos)
     216        {
     217            lgth = end_pos - start_pos;
     218            start_pos += block_base;
     219            ElemStart_grouping(start_pos, lgth);
     220        }
     221        start++;
     222    }
    251223}
    252224
     
    258230    if (block_boundary_case)
    259231    {
    260         int end_pos = count_forward_zeroes(elem_ends);
    261         ElemStart_grouping (last_elem_start, end_pos-last_elem_start);
    262         block_boundary_case = false;
     232        int end_pos = count_forward_zeroes(elem_ends);
     233        ElemStart_grouping (last_elem_start + block_base, end_pos-last_elem_start);
     234        block_boundary_case = false;
    263235    }
    264236
    265237    if ( bitblock_has_bit(dictionary.Word_starts) )
    266238    {
    267         StreamScanLengthGrouping((ScanBlock *) &dictionary.Word_starts, sizeof(BitBlock)/sizeof(ScanBlock));
     239        BitBlockForwardIterator iter_length_grouping(&dictionary.Word_starts);
     240        validate_block_length_grouping(iter_length_grouping, block_base);
    268241    }
    269242
  • proto/SymbolTable/dictionary_ls_template.cpp

    r1666 r1684  
    2424queue <size_t> elem_ends_buf;
    2525LSSymbolTable ls_symbol_table;
    26 
    27 /* StreamScan & Post Process Declarations */
    28 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    29 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
     26LineColTracker tracker;
    3027
    3128@global
     
    3431static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    3532static inline void postprocess_do_block(Dictionary& dictionary);
     33
     34void do_process(FILE *infile, FILE *outfile);
     35
     36static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     37static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
    3638static inline void do_symbol_table_lookup();
    37 
    38 void do_process(FILE *infile, FILE *outfile);
    3939
    4040int main(int argc, char * argv[]) {
     
    100100}
    101101
    102 
    103 /* StreamScan & Post Process Definitions */
    104 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    105 
    106         int blk;
    107         int block_pos = 0;
    108         int pos;
    109 
    110         for (blk = 0; blk < blk_count; blk++) {
    111                 ScanBlock s = stream[blk];
    112                 while(s) {
    113                         pos = (cfzl(s) + block_pos);
    114                         int code = (ProcessPos(pos));
    115                         if (code) {
    116                                 *error_pos_in_block = pos;
    117                                 return code; // error code
    118                         }
    119                         s = s & (s-1);  // clear rightmost bit.
    120                 }
    121                 block_pos += 8 * sizeof(ScanBlock);
    122         }
     102static inline int ElemStrt_check(int pos) {
     103        elem_starts_buf.push(buffer_base + pos);
    123104        return 0;
    124105}
    125106
    126 static inline int ElemStrt_check(int pos) {
    127         int block_pos = block_base + pos;
    128         elem_starts_buf.push(buffer_base + block_pos);
    129         return 0;
    130 }
    131 
    132107static inline int ElemEnd_check(int pos) {
    133         int block_pos = block_base + pos;
    134         elem_ends_buf.push(buffer_base + block_pos);
    135         return 0;
     108        elem_ends_buf.push(buffer_base + pos);
     109        return 0;
    136110}
    137111
     
    160134}
    161135
     136static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     137
     138        BitBlockForwardIterator end;
     139        int pos, block_pos;
     140
     141        while(start != end) {
     142
     143                block_pos = block_base + *start;
     144                int rv = is_valid(block_pos);
     145
     146                if (rv) {
     147                        int error_line, error_column;
     148                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     149                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     150                        exit(-1);
     151                }
     152                start++;
     153        }
     154}
     155
     156static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     157
     158        BitBlockForwardIterator end;
     159        int pos, block_pos, file_pos;
     160
     161        while(start != end) {
     162
     163                block_pos = block_base + *start;
     164                file_pos = block_pos+buffer_base;
     165
     166
     167                int rv = is_valid(block_pos, file_pos);
     168
     169                if (rv) {
     170                        int error_line, error_column;
     171                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     172                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     173                        exit(-1);
     174                }
     175                start++;
     176        }
     177}
     178
    162179static inline void postprocess_do_block(Dictionary& dictionary){
    163 
    164     int error_pos_in_block;
    165180
    166181    if ( bitblock_has_bit(dictionary.Word_starts))
    167182    {
    168         StreamScan((ScanBlock *) &dictionary.Word_starts, sizeof(BitBlock)/sizeof(ScanBlock), ElemStrt_check, &error_pos_in_block);
     183        BitBlockForwardIterator iter_length_grouping_starts(&dictionary.Word_starts);
     184        validate_block(iter_length_grouping_starts, block_base, ElemStrt_check);
    169185    }
    170186
    171187    if ( bitblock_has_bit(dictionary.Word_ends) )
    172188    {
    173         StreamScan((ScanBlock *) &dictionary.Word_ends, sizeof(BitBlock)/sizeof(ScanBlock), ElemEnd_check, &error_pos_in_block);
     189        BitBlockForwardIterator iter_length_grouping_ends(&dictionary.Word_ends);
     190        validate_block(iter_length_grouping_ends, block_base, ElemEnd_check);
    174191    }
    175192
  • proto/SymbolTable/dictionary_pbgs_div_template.cpp

    r1666 r1684  
    1 #define XML_PARSER
    21#include "../symtab_global.h"
    32#include <pbgs_div_symbol_table.h>
     
    1716#endif
    1817
    19 
    2018int block_base=0;
    2119int buffer_base=0;
    2220int buffer_last;
    2321char * source;
    24 LineColTracker tracker;
    25 TagMatcher matcher;
    2622BitBlock EOF_mask = simd<1>::constant<1>();
    27 ErrorTracker error_tracker;
    2823
    2924BitBlock elem_ends;
     
    3530PBGSDivSymbolTable pbgs_symbol_table;
    3631
    37 /* StreamScan & Post Process Declarations */
    38 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    39 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    40 
    4132@global
    4233
     
    4435static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    4536static inline void postprocess_do_block(Dictionary& dictionary, Hash_data hash_data);
     37
     38void do_process(FILE *infile, FILE *outfile);
     39
     40template <int L> static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
     41
    4642static inline int ScanForwardPos(BitBlock * block, int pos);
    4743static inline int compute_hash_value (int lgth, int start);
     
    4945template <int L> static inline int ElemEnd_grouping(int pos, int length);
    5046template <int L> static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count);
    51 
    52 void do_process(FILE *infile, FILE *outfile);
    5347
    5448int main(int argc, char * argv[]) {
     
    114108}
    115109
    116 
    117110static inline int ScanForwardPos(BitBlock * block, int pos)
    118111{
     
    122115    if (bitblock_has_bit(temp))
    123116    {
    124 #if DEBUG
    125         printf ("There is a 1 bit in the block\n");
    126 #endif
    127         return count_forward_zeroes (temp);
    128     }
    129     else
    130     {
    131 #if DEBUG
    132         printf ("There is no more 1 bit in the block. pos: %i\n", pos);
    133 #endif
    134         //handle boundary case
    135         block_boundary_case = true;
    136         last_elem_start = pos - BLOCK_SIZE;
    137 #if DEBUG
    138         printf ("last_elem_start: %i\n", last_elem_start);
    139 #endif
    140         return 0;
    141     }
    142 }
    143 
    144 /* StreamScan & Post Process Definitions */
    145 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    146 
    147         int blk;
    148         int block_pos = 0;
    149         int pos;
    150 
    151         for (blk = 0; blk < blk_count; blk++) {
    152                 ScanBlock s = stream[blk];
    153                 while(s) {
    154                         pos = (cfzl(s) + block_pos);
    155                         int code = (ProcessPos(pos));
    156                         if (code) {
    157                                 *error_pos_in_block = pos;
    158                                 return code; // error code
    159                         }
    160                         s = s & (s-1);  // clear rightmost bit.
    161                 }
    162                 block_pos += 8 * sizeof(ScanBlock);
    163         }
    164         return 0;
     117        return count_forward_zeroes (temp);
     118    }
     119    return 0;
    165120}
    166121
     
    174129// length in [1,16]
    175130template <int L>
    176 static inline int ElemEnd_grouping(int end_pos) {
    177     int end = block_base + end_pos;
     131static inline int ElemEnd_grouping(int end) {
    178132    int start = end - L;
    179133    int hashvalue = compute_hash_value(L, start - block_base);
     
    191145
    192146// length > 16
    193 static inline int ElemStart_grouping(int start_pos, int lgth) {
    194     int start = start_pos + block_base;
     147static inline int ElemStart_grouping(int start, int lgth) {
    195148    int hashvalue = compute_hash_value(lgth, start - block_base);
    196149    int gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
     
    215168// L = 17, pass in bitstream for symbols length longer than 16
    216169template <int L>
    217 static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count) {
    218     int blk;
    219     int block_pos = 0;
    220     for (blk = 0; blk < blk_count; blk++) {
    221         ScanBlock s = stream[blk];
    222         while(s) {
    223             int end_pos = cfzl(s) + block_pos;
    224             ElemEnd_grouping<L>(end_pos);
    225             s = s & (s-1);  // clear rightmost bit.
     170static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
     171
     172    BitBlockForwardIterator end;
     173    int end_pos;
     174
     175    while(start != end) {
     176        end_pos = *start;
     177        ElemEnd_grouping<L>(end_pos + block_base);
     178        start++;
     179    }
     180}
     181
     182template <>
     183inline void validate_block_length_grouping<17>(BitBlockForwardIterator & start, int block_base) {
     184
     185    BitBlockForwardIterator end;
     186    int start_pos, end_pos;
     187
     188    while(start != end) {
     189        start_pos = *start;
     190        end_pos = ScanForwardPos (&elem_ends, start_pos);
     191        if (end_pos)
     192        {
     193            ElemStart_grouping(start_pos - 16 + block_base, end_pos - start_pos + 16);
    226194        }
    227         block_pos += 8 * sizeof(ScanBlock);
    228     }
    229     return 0;
    230 }
    231 
    232 template <>
    233 inline int StreamScanLengthGrouping<17>(ScanBlock * stream, int blk_count) {
    234     int blk;
    235     int block_pos = 0;
    236     for (blk = 0; blk < blk_count; blk++) {
    237         ScanBlock s = stream[blk];
    238         while(s) {
    239             int start_pos = cfzl(s) + block_pos;
    240             int end_pos = ScanForwardPos (&elem_ends, start_pos);
    241             if (end_pos)
    242             {
    243                 ElemStart_grouping(start_pos - 16, end_pos - start_pos + 16);
    244             }
    245             s = s & (s-1);  // clear rightmost bit.
    246         }
    247         block_pos += 8 * sizeof(ScanBlock);
    248     }
    249     return 0;
    250 }
    251 
     195        else
     196        {
     197#if DEBUG
     198            printf ("There is no more 1 bit in the block. pos: %i | sym: %c%c[%c]\n", start_pos,
     199                    source[start_pos + block_base-2], source[start_pos + block_base-1], source[start_pos + block_base]);
     200#endif
     201            //handle boundary case
     202            block_boundary_case = true;
     203            last_elem_start = start_pos - 16 - BLOCK_SIZE;
     204#if DEBUG
     205            printf ("last_elem_start: %i\n", last_elem_start);
     206#endif
     207        }
     208        start++;
     209    }
     210}
    252211
    253212static inline void postprocess_do_block(Dictionary& dictionary, Hash_data hash_data){
     
    260219    {
    261220#if DEBUG
    262         printf ("block boundary case! Special handle!\n");
     221        printf ("block boundary case! Special handle!\n");
    263222#endif
    264223        int lgth = count_forward_zeroes(elem_ends)-last_elem_start;
     
    270229        printf ("%s | start: %i[%i] | lgth: %i | hashvalue: %i | gid: %i \n", __FUNCTION__, start, start - block_base, lgth, hashvalue, gid);
    271230#endif
    272         block_boundary_case = false;
     231        block_boundary_case = false;
    273232    }
    274233
    275234    if ( bitblock_has_bit(dictionary.Word_ends_1_to_2) )
    276235    {
    277         StreamScanLengthGrouping<2>((ScanBlock *) &dictionary.Word_ends_1_to_2, sizeof(BitBlock)/sizeof(ScanBlock));
     236        BitBlockForwardIterator iter_length_grouping_2(&dictionary.Word_ends_1_to_2);
     237        validate_block_length_grouping<2>(iter_length_grouping_2, block_base);
    278238    }
    279239
    280240    if ( bitblock_has_bit(dictionary.Word_ends_3_to_4) )
    281241    {
    282         StreamScanLengthGrouping<4>((ScanBlock *) &dictionary.Word_ends_3_to_4, sizeof(BitBlock)/sizeof(ScanBlock));
     242        BitBlockForwardIterator iter_length_grouping_4(&dictionary.Word_ends_3_to_4);
     243        validate_block_length_grouping<4>(iter_length_grouping_4, block_base);
    283244    }
    284245
    285246    if ( bitblock_has_bit(dictionary.Word_ends_5_to_6) )
    286247    {
    287         StreamScanLengthGrouping<6>((ScanBlock *) &dictionary.Word_ends_5_to_6, sizeof(BitBlock)/sizeof(ScanBlock));
     248        BitBlockForwardIterator iter_length_grouping_6(&dictionary.Word_ends_5_to_6);
     249        validate_block_length_grouping<6>(iter_length_grouping_6, block_base);
    288250    }
    289251
    290252    if ( bitblock_has_bit(dictionary.Word_ends_7_to_8) )
    291253    {
    292         StreamScanLengthGrouping<8>((ScanBlock *) &dictionary.Word_ends_7_to_8, sizeof(BitBlock)/sizeof(ScanBlock));
     254        BitBlockForwardIterator iter_length_grouping_8(&dictionary.Word_ends_7_to_8);
     255        validate_block_length_grouping<8>(iter_length_grouping_8, block_base);
    293256    }
    294257
    295258    if ( bitblock_has_bit(dictionary.Word_ends_9_to_10) )
    296259    {
    297         StreamScanLengthGrouping<10>((ScanBlock *) &dictionary.Word_ends_9_to_10, sizeof(BitBlock)/sizeof(ScanBlock));
     260        BitBlockForwardIterator iter_length_grouping_10(&dictionary.Word_ends_9_to_10);
     261        validate_block_length_grouping<10>(iter_length_grouping_10, block_base);
    298262    }
    299263
    300264    if ( bitblock_has_bit(dictionary.Word_ends_11_to_12) )
    301265    {
    302         StreamScanLengthGrouping<12>((ScanBlock *) &dictionary.Word_ends_11_to_12, sizeof(BitBlock)/sizeof(ScanBlock));
     266        BitBlockForwardIterator iter_length_grouping_12(&dictionary.Word_ends_11_to_12);
     267        validate_block_length_grouping<12>(iter_length_grouping_12, block_base);
    303268    }
    304269
    305270    if ( bitblock_has_bit(dictionary.Word_ends_13_to_14) )
    306271    {
    307         StreamScanLengthGrouping<14>((ScanBlock *) &dictionary.Word_ends_13_to_14, sizeof(BitBlock)/sizeof(ScanBlock));
     272        BitBlockForwardIterator iter_length_grouping_14(&dictionary.Word_ends_13_to_14);
     273        validate_block_length_grouping<14>(iter_length_grouping_14, block_base);
    308274    }
    309275
    310276    if ( bitblock_has_bit(dictionary.Word_ends_15_to_16) )
    311277    {
    312         StreamScanLengthGrouping<16>((ScanBlock *) &dictionary.Word_ends_15_to_16, sizeof(BitBlock)/sizeof(ScanBlock));
     278        BitBlockForwardIterator iter_length_grouping_16(&dictionary.Word_ends_15_to_16);
     279        validate_block_length_grouping<16>(iter_length_grouping_16, block_base);
    313280    }
    314281
    315282    if ( bitblock_has_bit(dictionary.Word_remaining_ends) )
    316283    {
    317         StreamScanLengthGrouping<17>((ScanBlock *) &dictionary.Word_remaining_ends, sizeof(BitBlock)/sizeof(ScanBlock));
     284        BitBlockForwardIterator iter_length_grouping_remaining(&dictionary.Word_remaining_ends);
     285        validate_block_length_grouping<17>(iter_length_grouping_remaining, block_base);
    318286    }
    319287
  • proto/SymbolTable/dictionary_pbgs_identity_template.cpp

    r1666 r1684  
    3232PBGSIdentitySymbolTable pbgs_symbol_table;
    3333
    34 /* StreamScan & Post Process Declarations */
    35 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    36 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    37 
    3834@global
    3935
     
    4137static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    4238static inline void postprocess_do_block(Dictionary& dictionary, Hash_data hash_data);
     39
     40void do_process(FILE *infile, FILE *outfile);
     41
     42template <int L> static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
     43
    4344static inline int ScanBackwardPos(BitBlock * block, int pos);
    4445static inline int compute_hash_value (int lgth, int start);
    4546template <int L> static inline int ElemEnd_grouping(int pos);
    46 template <int L> static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count);
    47 
    48 void do_process(FILE *infile, FILE *outfile);
    4947
    5048int main(int argc, char * argv[]) {
     
    127125}
    128126
    129 /* StreamScan & Post Process Definitions */
    130 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    131 
    132         int blk;
    133         int block_pos = 0;
    134         int pos;
    135 
    136         for (blk = 0; blk < blk_count; blk++) {
    137                 ScanBlock s = stream[blk];
    138                 while(s) {
    139                         pos = (cfzl(s) + block_pos);
    140                         int code = (ProcessPos(pos));
    141                         if (code) {
    142                                 *error_pos_in_block = pos;
    143                                 return code; // error code
    144                         }
    145                         s = s & (s-1);  // clear rightmost bit.
    146                 }
    147                 block_pos += 8 * sizeof(ScanBlock);
    148         }
    149         return 0;
    150 }
    151 
    152127static inline int compute_hash_value (int lgth, int start)
    153128{
     
    158133
    159134template <int L>
    160 static inline int ElemEnd_grouping(int pos) {
    161     int end = block_base + pos;
     135static inline int ElemEnd_grouping(int end) {
    162136    int start = end - L;
    163137    int hashvalue = compute_hash_value(L, start - block_base);
     
    175149
    176150template<>
    177 inline int ElemEnd_grouping<17>(int pos) {
    178     int end = block_base + pos;
    179     int start = ScanBackwardPos (&elem_starts, pos) + block_base;
     151inline int ElemEnd_grouping<17>(int end) {
     152    int start = ScanBackwardPos (&elem_starts, end - block_base) + block_base;
    180153    int lgth = end - start;
    181154    int hashvalue = compute_hash_value(lgth, start - block_base);
     
    188161//    else
    189162    {
    190         gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
     163        gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
    191164    }
    192165    gids.push_back(gid);
     
    201174
    202175template <int L>
    203 static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count) {
    204     int blk;
    205     int block_pos = 0;
    206     for (blk = 0; blk < blk_count; blk++) {
    207         ScanBlock s = stream[blk];
    208         while(s) {
    209             int end_pos = cfzl(s) + block_pos;
    210             ElemEnd_grouping<L>(end_pos);
    211             s = s & (s-1);  // clear rightmost bit.
    212         }
    213         block_pos += 8 * sizeof(ScanBlock);
    214     }
    215     return 0;
     176static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
     177
     178    BitBlockForwardIterator end;
     179    int block_pos;
     180
     181    while(start != end) {
     182        block_pos = block_base + *start;
     183        ElemEnd_grouping<L>(block_pos);
     184        start++;
     185    }
    216186}
    217187
     
    223193    if ( bitblock_has_bit(dictionary.Word_ends_1) )
    224194    {
    225         StreamScanLengthGrouping<1>((ScanBlock *) &dictionary.Word_ends_1, sizeof(BitBlock)/sizeof(ScanBlock));
     195        BitBlockForwardIterator iter_length_grouping_1(&dictionary.Word_ends_1);
     196        validate_block_length_grouping<1>(iter_length_grouping_1, block_base);
    226197    }
    227198
    228199    if ( bitblock_has_bit(dictionary.Word_ends_2) )
    229200    {
    230         StreamScanLengthGrouping<2>((ScanBlock *) &dictionary.Word_ends_2, sizeof(BitBlock)/sizeof(ScanBlock));
     201        BitBlockForwardIterator iter_length_grouping_2(&dictionary.Word_ends_2);
     202        validate_block_length_grouping<2>(iter_length_grouping_2, block_base);
    231203    }
    232204
    233205    if ( bitblock_has_bit(dictionary.Word_ends_3) )
    234206    {
    235         StreamScanLengthGrouping<3>((ScanBlock *) &dictionary.Word_ends_3, sizeof(BitBlock)/sizeof(ScanBlock));
     207        BitBlockForwardIterator iter_length_grouping_3(&dictionary.Word_ends_3);
     208        validate_block_length_grouping<3>(iter_length_grouping_3, block_base);
    236209    }
    237210
    238211    if ( bitblock_has_bit(dictionary.Word_ends_4) )
    239212    {
    240         StreamScanLengthGrouping<4>((ScanBlock *) &dictionary.Word_ends_4, sizeof(BitBlock)/sizeof(ScanBlock));
     213        BitBlockForwardIterator iter_length_grouping_4(&dictionary.Word_ends_4);
     214        validate_block_length_grouping<4>(iter_length_grouping_4, block_base);
    241215    }
    242216
    243217    if ( bitblock_has_bit(dictionary.Word_ends_5) )
    244218    {
    245         StreamScanLengthGrouping<5>((ScanBlock *) &dictionary.Word_ends_5, sizeof(BitBlock)/sizeof(ScanBlock));
     219        BitBlockForwardIterator iter_length_grouping_5(&dictionary.Word_ends_5);
     220        validate_block_length_grouping<5>(iter_length_grouping_5, block_base);
    246221    }
    247222
    248223    if ( bitblock_has_bit(dictionary.Word_ends_6) )
    249224    {
    250         StreamScanLengthGrouping<6>((ScanBlock *) &dictionary.Word_ends_6, sizeof(BitBlock)/sizeof(ScanBlock));
     225        BitBlockForwardIterator iter_length_grouping_6(&dictionary.Word_ends_6);
     226        validate_block_length_grouping<6>(iter_length_grouping_6, block_base);
    251227    }
    252228
    253229    if ( bitblock_has_bit(dictionary.Word_ends_7) )
    254230    {
    255         StreamScanLengthGrouping<7>((ScanBlock *) &dictionary.Word_ends_7, sizeof(BitBlock)/sizeof(ScanBlock));
     231        BitBlockForwardIterator iter_length_grouping_7(&dictionary.Word_ends_7);
     232        validate_block_length_grouping<7>(iter_length_grouping_7, block_base);
    256233    }
    257234
    258235    if ( bitblock_has_bit(dictionary.Word_ends_8) )
    259236    {
    260         StreamScanLengthGrouping<8>((ScanBlock *) &dictionary.Word_ends_8, sizeof(BitBlock)/sizeof(ScanBlock));
     237        BitBlockForwardIterator iter_length_grouping_8(&dictionary.Word_ends_8);
     238        validate_block_length_grouping<8>(iter_length_grouping_8, block_base);
    261239    }
    262240
    263241    if ( bitblock_has_bit(dictionary.Word_ends_9) )
    264242    {
    265         StreamScanLengthGrouping<9>((ScanBlock *) &dictionary.Word_ends_9, sizeof(BitBlock)/sizeof(ScanBlock));
     243        BitBlockForwardIterator iter_length_grouping_9(&dictionary.Word_ends_9);
     244        validate_block_length_grouping<9>(iter_length_grouping_9, block_base);
    266245    }
    267246
    268247    if ( bitblock_has_bit(dictionary.Word_ends_10) )
    269248    {
    270         StreamScanLengthGrouping<10>((ScanBlock *) &dictionary.Word_ends_10, sizeof(BitBlock)/sizeof(ScanBlock));
     249        BitBlockForwardIterator iter_length_grouping_10(&dictionary.Word_ends_10);
     250        validate_block_length_grouping<10>(iter_length_grouping_10, block_base);
    271251    }
    272252
    273253    if ( bitblock_has_bit(dictionary.Word_ends_11) )
    274254    {
    275         StreamScanLengthGrouping<11>((ScanBlock *) &dictionary.Word_ends_11, sizeof(BitBlock)/sizeof(ScanBlock));
     255        BitBlockForwardIterator iter_length_grouping_11(&dictionary.Word_ends_11);
     256        validate_block_length_grouping<11>(iter_length_grouping_11, block_base);
    276257    }
    277258
    278259    if ( bitblock_has_bit(dictionary.Word_ends_12) )
    279260    {
    280         StreamScanLengthGrouping<12>((ScanBlock *) &dictionary.Word_ends_12, sizeof(BitBlock)/sizeof(ScanBlock));
     261        BitBlockForwardIterator iter_length_grouping_12(&dictionary.Word_ends_12);
     262        validate_block_length_grouping<12>(iter_length_grouping_12, block_base);
    281263    }
    282264
    283265    if ( bitblock_has_bit(dictionary.Word_ends_13) )
    284266    {
    285         StreamScanLengthGrouping<13>((ScanBlock *) &dictionary.Word_ends_13, sizeof(BitBlock)/sizeof(ScanBlock));
     267        BitBlockForwardIterator iter_length_grouping_13(&dictionary.Word_ends_13);
     268        validate_block_length_grouping<13>(iter_length_grouping_13, block_base);
    286269    }
    287270
    288271    if ( bitblock_has_bit(dictionary.Word_ends_14) )
    289272    {
    290         StreamScanLengthGrouping<14>((ScanBlock *) &dictionary.Word_ends_14, sizeof(BitBlock)/sizeof(ScanBlock));
     273        BitBlockForwardIterator iter_length_grouping_14(&dictionary.Word_ends_14);
     274        validate_block_length_grouping<14>(iter_length_grouping_14, block_base);
    291275    }
    292276
    293277    if ( bitblock_has_bit(dictionary.Word_ends_15) )
    294278    {
    295         StreamScanLengthGrouping<15>((ScanBlock *) &dictionary.Word_ends_15, sizeof(BitBlock)/sizeof(ScanBlock));
     279        BitBlockForwardIterator iter_length_grouping_15(&dictionary.Word_ends_15);
     280        validate_block_length_grouping<15>(iter_length_grouping_15, block_base);
    296281    }
    297282
    298283    if ( bitblock_has_bit(dictionary.Word_ends_16) )
    299284    {
    300         StreamScanLengthGrouping<16>((ScanBlock *) &dictionary.Word_ends_16, sizeof(BitBlock)/sizeof(ScanBlock));
     285        BitBlockForwardIterator iter_length_grouping_16(&dictionary.Word_ends_16);
     286        validate_block_length_grouping<16>(iter_length_grouping_16, block_base);
    301287    }
    302288
    303289    if ( bitblock_has_bit(dictionary.Word_ends_17_and_longer) )
    304290    {
    305         StreamScanLengthGrouping<17>((ScanBlock *) &dictionary.Word_ends_17_and_longer, sizeof(BitBlock)/sizeof(ScanBlock));
     291        BitBlockForwardIterator iter_length_grouping_remaining(&dictionary.Word_ends_17_and_longer);
     292        validate_block_length_grouping<17>(iter_length_grouping_remaining, block_base);
    306293    }
    307294
  • proto/SymbolTable/dictionary_pbgs_log_template.cpp

    r1666 r1684  
    2222int buffer_last;
    2323char * source;
    24 LineColTracker tracker;
    25 TagMatcher matcher;
    2624BitBlock EOF_mask = simd<1>::constant<1>();
    27 ErrorTracker error_tracker;
    28 
    2925BitBlock elem_starts;
    3026int previous_block_last_elem_start;
     
    3430PBGSLogSymbolTable pbgs_symbol_table;
    3531
    36 /* StreamScan & Post Process Declarations */
    37 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    38 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    39 
    4032@global
    4133
     
    4335static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    4436static inline void postprocess_do_block(Dictionary& dictionary, Hash_data hash_data);
     37
     38void do_process(FILE *infile, FILE *outfile);
     39
     40template <int L> static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
     41
    4542static inline int ScanBackwardPos(BitBlock * block, int pos);
    4643static inline int compute_hash_value (int lgth, int start);
    4744template <int L> static inline int ElemEnd_grouping(int pos, int length);
    4845template <int L> static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count);
    49 
    50 void do_process(FILE *infile, FILE *outfile);
    5146
    5247int main(int argc, char * argv[]) {
     
    251246//    else
    252247    {
    253         gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
     248        gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
    254249    }
    255250    gids.push_back(gid);
     
    268263// L = 17, pass in bitstream for symbols length longer than 16
    269264template <int L>
    270 static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count) {
    271     int blk;
    272     int block_pos = 0;
    273     for (blk = 0; blk < blk_count; blk++) {
    274         ScanBlock s = stream[blk];
    275         while(s) {
    276             int end_pos = cfzl(s) + block_pos;
    277             int start_pos = ScanBackwardPos (&elem_starts, end_pos);
    278             int length = end_pos - start_pos;
    279             ElemEnd_grouping<L>(start_pos, length);
    280             s = s & (s-1);  // clear rightmost bit.
    281         }
    282         block_pos += 8 * sizeof(ScanBlock);
    283     }
    284     return 0;
     265static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
     266
     267    BitBlockForwardIterator end;
     268    int end_pos, start_pos, length;
     269
     270    while(start != end) {
     271        end_pos = /*block_base + */*start;
     272        start_pos = ScanBackwardPos (&elem_starts, end_pos);
     273        length = end_pos - start_pos;
     274        ElemEnd_grouping<L>(start_pos, length);
     275        start++;
     276    }
    285277}
    286278
    287279// pass in bitstream for symbols length 1
    288280template <>
    289 inline int StreamScanLengthGrouping<1>(ScanBlock * stream, int blk_count) {
    290     int blk;
    291     int block_pos = 0;
    292     for (blk = 0; blk < blk_count; blk++) {
    293         ScanBlock s = stream[blk];
    294         while(s) {
    295             int end_pos = cfzl(s) + block_pos;
    296             ElemEnd_grouping<1>(end_pos, 1);
    297             s = s & (s-1);  // clear rightmost bit.
    298         }
    299         block_pos += 8 * sizeof(ScanBlock);
    300     }
    301     return 0;
     281inline void validate_block_length_grouping<1>(BitBlockForwardIterator & start, int block_base) {
     282
     283    BitBlockForwardIterator end;
     284    int end_pos, start_pos, length;
     285
     286    while(start != end) {
     287        end_pos = /*block_base + */*start;
     288        start_pos = ScanBackwardPos (&elem_starts, end_pos);
     289        length = end_pos - start_pos;
     290        ElemEnd_grouping<1>(end_pos, 1);
     291        start++;
     292    }
    302293}
    303294
    304295// pass in bitstream for symbols length 2
    305296template <>
    306 inline int StreamScanLengthGrouping<2>(ScanBlock * stream, int blk_count) {
    307     int blk;
    308     int block_pos = 0;
    309     for (blk = 0; blk < blk_count; blk++) {
    310         ScanBlock s = stream[blk];
    311         while(s) {
    312             int end_pos = cfzl(s) + block_pos;
    313             ElemEnd_grouping<2>(end_pos, 2);
    314             s = s & (s-1);  // clear rightmost bit.
    315         }
    316         block_pos += 8 * sizeof(ScanBlock);
    317     }
    318     return 0;
     297inline void validate_block_length_grouping<2>(BitBlockForwardIterator & start, int block_base) {
     298
     299    BitBlockForwardIterator end;
     300    int end_pos, start_pos, length;
     301
     302    while(start != end) {
     303        end_pos = /*block_base + */*start;
     304        start_pos = ScanBackwardPos (&elem_starts, end_pos);
     305        length = end_pos - start_pos;
     306        ElemEnd_grouping<2>(end_pos, 2);
     307        start++;
     308    }
    319309}
    320310
     
    326316    if ( bitblock_has_bit(dictionary.Word_ends_1) )
    327317    {
    328         StreamScanLengthGrouping<1>((ScanBlock *) &dictionary.Word_ends_1, sizeof(BitBlock)/sizeof(ScanBlock));
     318        BitBlockForwardIterator iter_length_grouping_1(&dictionary.Word_ends_1);
     319        validate_block_length_grouping<1>(iter_length_grouping_1, block_base);
    329320    }
    330321
    331322    if ( bitblock_has_bit(dictionary.Word_ends_2) )
    332323    {
    333         StreamScanLengthGrouping<2>((ScanBlock *) &dictionary.Word_ends_2, sizeof(BitBlock)/sizeof(ScanBlock));
     324        BitBlockForwardIterator iter_length_grouping_2(&dictionary.Word_ends_2);
     325        validate_block_length_grouping<2>(iter_length_grouping_2, block_base);
    334326    }
    335327
    336328    if ( bitblock_has_bit(dictionary.Word_ends_3_to_4) )
    337329    {
    338         StreamScanLengthGrouping<4>((ScanBlock *) &dictionary.Word_ends_3_to_4, sizeof(BitBlock)/sizeof(ScanBlock));
     330        BitBlockForwardIterator iter_length_grouping_4(&dictionary.Word_ends_3_to_4);
     331        validate_block_length_grouping<4>(iter_length_grouping_4, block_base);
    339332    }
    340333
    341334    if ( bitblock_has_bit(dictionary.Word_ends_5_to_8) )
    342335    {
    343         StreamScanLengthGrouping<8>((ScanBlock *) &dictionary.Word_ends_5_to_8, sizeof(BitBlock)/sizeof(ScanBlock));
     336        BitBlockForwardIterator iter_length_grouping_8(&dictionary.Word_ends_5_to_8);
     337        validate_block_length_grouping<8>(iter_length_grouping_8, block_base);
    344338    }
    345339
    346340    if ( bitblock_has_bit(dictionary.Word_ends_9_to_16) )
    347341    {
    348         StreamScanLengthGrouping<16>((ScanBlock *) &dictionary.Word_ends_9_to_16, sizeof(BitBlock)/sizeof(ScanBlock));
     342        BitBlockForwardIterator iter_length_grouping_16(&dictionary.Word_ends_9_to_16);
     343        validate_block_length_grouping<16>(iter_length_grouping_16, block_base);
    349344    }
    350345
    351346    if ( bitblock_has_bit(dictionary.Word_ends_17_and_longer) )
    352347    {
    353         StreamScanLengthGrouping<17>((ScanBlock *) &dictionary.Word_ends_17_and_longer, sizeof(BitBlock)/sizeof(ScanBlock));
     348        BitBlockForwardIterator iter_length_grouping_remaining(&dictionary.Word_ends_17_and_longer);
     349        validate_block_length_grouping<17>(iter_length_grouping_remaining, block_base);
    354350    }
    355351
  • proto/SymbolTable/dictionary_stl_template.cpp

    r1666 r1684  
    2020char * source;
    2121BitBlock EOF_mask = simd<1>::constant<1>();
     22LineColTracker tracker;
    2223
    2324queue <size_t> elem_starts_buf;
     
    2526vector <int> gids;
    2627SymbolTable symbol_table;
    27 
    28 /* StreamScan & Post Process Declarations */
    29 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    30 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    3128
    3229@global
     
    10198}
    10299
    103 
    104 /* StreamScan & Post Process Definitions */
    105 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    106 
    107         int blk;
    108         int block_pos = 0;
    109         int pos;
    110 
    111         for (blk = 0; blk < blk_count; blk++) {
    112                 ScanBlock s = stream[blk];
    113                 while(s) {
    114                         pos = (cfzl(s) + block_pos);
    115                         int code = (ProcessPos(pos));
    116                         if (code) {
    117                                 *error_pos_in_block = pos;
    118                                 return code; // error code
    119                         }
    120                         s = s & (s-1);  // clear rightmost bit.
    121                 }
    122                 block_pos += 8 * sizeof(ScanBlock);
    123         }
    124         return 0;
    125 }
    126 
    127100static inline int ElemStrt_check(int pos) {
    128         int block_pos = block_base + pos;
    129         elem_starts_buf.push(buffer_base + block_pos);
     101        elem_starts_buf.push(buffer_base + pos);
    130102        return 0;
    131103}
    132104
    133105static inline int ElemEnd_check(int pos) {
    134         int block_pos = block_base + pos;
    135         elem_ends_buf.push(buffer_base + block_pos);
     106        elem_ends_buf.push(buffer_base + pos);
    136107        return 0;
    137108}
     
    162133}
    163134
     135static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     136
     137        BitBlockForwardIterator end;
     138        int pos, block_pos;
     139
     140        while(start != end) {
     141
     142                block_pos = block_base + *start;
     143                int rv = is_valid(block_pos);
     144
     145                if (rv) {
     146                        int error_line, error_column;
     147                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     148                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     149                        exit(-1);
     150                }
     151                start++;
     152        }
     153}
     154
     155static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     156
     157        BitBlockForwardIterator end;
     158        int pos, block_pos, file_pos;
     159
     160        while(start != end) {
     161
     162                block_pos = block_base + *start;
     163                file_pos = block_pos+buffer_base;
     164
     165
     166                int rv = is_valid(block_pos, file_pos);
     167
     168                if (rv) {
     169                        int error_line, error_column;
     170                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     171                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     172                        exit(-1);
     173                }
     174                start++;
     175        }
     176}
     177
    164178static inline void postprocess_do_block(Dictionary& dictionary){
    165 
    166     int error_pos_in_block;
    167179
    168180    if ( bitblock_has_bit(dictionary.Word_starts))
    169181    {
    170         StreamScan((ScanBlock *) &dictionary.Word_starts, sizeof(BitBlock)/sizeof(ScanBlock), ElemStrt_check, &error_pos_in_block);
     182        BitBlockForwardIterator iter_length_grouping_starts(&dictionary.Word_starts);
     183        validate_block(iter_length_grouping_starts, block_base, ElemStrt_check);
    171184    }
    172185
    173186    if ( bitblock_has_bit(dictionary.Word_ends) )
    174187    {
    175         StreamScan((ScanBlock *) &dictionary.Word_ends, sizeof(BitBlock)/sizeof(ScanBlock), ElemEnd_check, &error_pos_in_block);
     188        BitBlockForwardIterator iter_length_grouping_ends(&dictionary.Word_ends);
     189        validate_block(iter_length_grouping_ends, block_base, ElemEnd_check);
    176190    }
    177191
  • proto/SymbolTable/symtab_global.h

    r1671 r1684  
    66//typedef long ScanBlock;
    77#define SIMD_type BitBlock
    8 #define DEBUG 0 
     8#define DEBUG 0
    99#define SEGMENT_BLOCKS 12
    1010#define BUFFER_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
  • proto/SymbolTable/symtab_hash_template.cpp

    r1666 r1684  
    2929HashSymbolTable symbol_table;
    3030
    31 /* StreamScan & Post Process Declarations */
    32 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    33 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    34 
    3531static inline int NameStrt_check(int pos);
    3632static inline int Name_check(int pos);
     
    4743static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    4844static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, int chars_avail);
     45void do_process(FILE *infile, FILE *outfile);
     46
     47static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     48static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
     49
    4950static inline void do_symbol_table_lookup();
    5051
     
    113114}
    114115
    115 /* StreamScan & Post Process Definitions */
    116 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    117 
    118         int blk;
    119         int block_pos = 0;
    120         int pos;
    121 
    122         for (blk = 0; blk < blk_count; blk++) {
    123                 ScanBlock s = stream[blk];
    124                 while(s) {
    125                         pos = (cfzl(s) + block_pos);
    126                         int code = (ProcessPos(pos));
    127                         if (code) {
    128                                 *error_pos_in_block = pos;
    129                                 return code; // error code
    130                         }
    131                         s = s & (s-1);  // clear rightmost bit.
    132                 }
    133                 block_pos += 8 * sizeof(ScanBlock);
    134         }
    135         return 0;
    136 }
    137 
    138116static inline int ElemStrt_check(int pos) {
    139         int block_pos = block_base + pos;
    140         elem_starts_buf.push(buffer_base + block_pos);
     117        elem_starts_buf.push(buffer_base + pos);
    141118        return 0;
    142119}
    143120
    144121static inline int ElemEnd_check(int pos) {
    145         int block_pos = block_base + pos;
    146         elem_ends_buf.push(buffer_base + block_pos);
     122        elem_ends_buf.push(buffer_base + pos);
    147123        return 0;
    148124}
    149125
    150126static inline int NameStrt_check(int pos) {
    151         int block_pos = block_base + pos;
    152         if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[block_pos]) == 0){
     127        if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[pos]) == 0){
    153128              return XMLTestSuiteError::NAME_START;
    154129        }
     
    157132
    158133static inline int Name_check(int pos) {
    159         int block_pos = block_base + pos;
    160         if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[block_pos]) == 0){
     134        if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[pos]) == 0){
    161135                  return XMLTestSuiteError::NAME;
    162136        }
     
    164138}
    165139
    166 static inline int PIName_check(int pos) {
    167         int block_pos = block_base + pos;
    168         int file_pos = block_pos+buffer_base;
    169         if (at_XxMmLll<ASCII>((unsigned char*)&source[block_pos]) && (source[block_pos+3]=='?' || source[block_pos+3]<= ' ')) {
     140static inline int PIName_check(int pos, int file_pos) {
     141        if (at_XxMmLll<ASCII>((unsigned char*)&source[pos]) && (source[pos+3]=='?' || source[pos+3]<= ' ')) {
    170142              // "<?xml" legal at start of file.
    171143              if (!((file_pos == 2) && at_XmlDecl_start<ASCII>((unsigned char*)&source[0]))) {
     
    177149
    178150static inline int CD_check(int pos) {
    179         int block_pos = block_base + pos;
    180         if (!at_CDATA1<ASCII>((unsigned char*)&source[block_pos])){
     151        if (!at_CDATA1<ASCII>((unsigned char*)&source[pos])){
    181152                  return XMLTestSuiteError::CDATA;
    182153        }
     
    185156
    186157static inline int GenRef_check(int pos) {
    187         int block_pos = block_base + pos;
    188         unsigned char* s = (unsigned char*)&source[block_pos];
     158        unsigned char* s = (unsigned char*)&source[pos];
    189159        if (!(at_Ref_gt<ASCII>(s)||at_Ref_lt<ASCII>(s)||at_Ref_amp<ASCII>(s)||at_Ref_quot<ASCII>(s)||at_Ref_apos<ASCII>(s))){
    190160              return XMLTestSuiteError::UNDEFREF;
     
    194164
    195165static inline int HexRef_check(int pos) {
    196         int block_pos = block_base + pos;
    197         unsigned char* s = (unsigned char*)&source[block_pos];
     166        unsigned char* s = (unsigned char*)&source[pos];
    198167        int ch_val = 0;
    199168        while(at_HexDigit<ASCII>(s)){
     
    214183
    215184static inline int DecRef_check(int pos) {
    216         int block_pos = block_base + pos;
    217         unsigned char* s = (unsigned char*)&source[block_pos];
     185        unsigned char* s = (unsigned char*)&source[pos];
    218186        int ch_val = 0;
    219187        while(at_HexDigit<ASCII>(s)){
     
    234202
    235203static inline int AttRef_check(int pos) {
    236         int block_pos = block_base + pos;
    237         unsigned char* s = (unsigned char*)&source[block_pos];
     204        unsigned char* s = (unsigned char*)&source[pos];
    238205        int ch_val = 0;
    239206        if(s[0]=='#'){
     
    262229}
    263230
     231static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     232
     233        BitBlockForwardIterator end;
     234        int pos, block_pos;
     235
     236        while(start != end) {
     237
     238                block_pos = block_base + *start;
     239                int rv = is_valid(block_pos);
     240
     241                if (rv) {
     242                        int error_line, error_column;
     243                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     244                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     245                        exit(-1);
     246                }
     247                start++;
     248        }
     249}
     250
     251static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     252
     253        BitBlockForwardIterator end;
     254        int pos, block_pos, file_pos;
     255
     256        while(start != end) {
     257
     258                block_pos = block_base + *start;
     259                file_pos = block_pos+buffer_base;
     260
     261
     262                int rv = is_valid(block_pos, file_pos);
     263
     264                if (rv) {
     265                        int error_line, error_column;
     266                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     267                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     268                        exit(-1);
     269                }
     270                start++;
     271        }
     272}
     273
    264274static inline void do_symbol_table_lookup()
    265275{
     
    290300
    291301    tracker.StoreNewlines(lex.LF);
    292     int rv, error_pos_in_block, error_line, error_column;
    293302
    294303    if ( bitblock_has_bit(tag_Callouts.ElemName_starts))
    295304    {
    296         StreamScan((ScanBlock *) &tag_Callouts.ElemName_starts, sizeof(BitBlock)/sizeof(ScanBlock), ElemStrt_check, &error_pos_in_block);
     305        BitBlockForwardIterator iter_length_grouping_starts(&tag_Callouts.ElemName_starts);
     306        validate_block(iter_length_grouping_starts, block_base, ElemStrt_check);
    297307    }
    298308
    299309    if ( bitblock_has_bit(tag_Callouts.ElemName_ends) )
    300310    {
    301         StreamScan((ScanBlock *) &tag_Callouts.ElemName_ends, sizeof(BitBlock)/sizeof(ScanBlock), ElemEnd_check, &error_pos_in_block);
     311        BitBlockForwardIterator iter_length_grouping_ends(&tag_Callouts.ElemName_ends);
     312        validate_block(iter_length_grouping_ends, block_base, ElemEnd_check);
    302313    }
    303314
     
    306317
    307318    if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
    308       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), NameStrt_check, &error_pos_in_block);
    309       if (rv) {
    310               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    311               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    312               exit(-1);
    313       }
    314 
    315       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_names, sizeof(BitBlock)/sizeof(ScanBlock), Name_check, &error_pos_in_block);
    316       if (rv) {
    317               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    318               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    319               exit(-1);
    320       }
    321     }
    322 
     319        BitBlockForwardIterator iter_NameStrt_check(&check_streams.non_ascii_name_starts);
     320        validate_block(iter_NameStrt_check, block_base, NameStrt_check);
     321        BitBlockForwardIterator iter_Name_check(&check_streams.non_ascii_names);
     322        validate_block(iter_Name_check, block_base, Name_check);
     323    }
    323324    if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
    324       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.PI_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), PIName_check, &error_pos_in_block);
    325       if (rv) {
    326               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    327               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    328               exit(-1);
    329       }
    330     }
    331 
     325        BitBlockForwardIterator iter_PI_name_starts(&(ctCDPI_Callouts.PI_name_starts));
     326        validate_block(iter_PI_name_starts, block_base, buffer_base, PIName_check);
     327    }
    332328    if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
    333       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.CD_starts, sizeof(BitBlock)/sizeof(ScanBlock), CD_check, &error_pos_in_block);
    334       if (rv) {
    335               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    336               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    337               exit(-1);
    338       }
    339     }
    340 
    341     if (bitblock_has_bit(ref_Callouts.GenRef_starts)){
    342       rv = StreamScan((ScanBlock *) &ref_Callouts.GenRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), GenRef_check, &error_pos_in_block);
    343       if (rv) {
    344               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    345               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    346               exit(-1);
    347       }
    348     }
    349 
    350     if (bitblock_has_bit(ref_Callouts.DecRef_starts)){
    351       rv = StreamScan((ScanBlock *) &ref_Callouts.DecRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), DecRef_check, &error_pos_in_block);
    352       if (rv) {
    353               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    354               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    355               exit(-1);
    356       }
    357     }
    358 
    359     if (bitblock_has_bit(ref_Callouts.HexRef_starts)){
    360       rv = StreamScan((ScanBlock *) &ref_Callouts.HexRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), HexRef_check, &error_pos_in_block);
    361       if (rv) {
    362               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    363               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    364               exit(-1);
    365       }
    366     }
    367 
    368     if (bitblock_has_bit(check_streams.att_refs)){
    369       rv = StreamScan((ScanBlock *) &check_streams.att_refs, sizeof(BitBlock)/sizeof(ScanBlock), AttRef_check, &error_pos_in_block);
    370       if (rv) {
    371               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    372               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    373               exit(-1);
    374       }
     329        BitBlockForwardIterator iter_CD_check(&ctCDPI_Callouts.CD_starts);
     330        validate_block(iter_CD_check, block_base, CD_check);
     331    }
     332    if(bitblock_has_bit(ref_Callouts.GenRef_starts)){
     333        BitBlockForwardIterator iter_GenRef_check(&ref_Callouts.GenRef_starts);
     334        validate_block(iter_GenRef_check, block_base, GenRef_check);
     335    }
     336    if(bitblock_has_bit(ref_Callouts.DecRef_starts)){
     337        BitBlockForwardIterator iter_DecRef_check(&ref_Callouts.DecRef_starts);
     338        validate_block(iter_DecRef_check, block_base, DecRef_check);
     339    }
     340    if(bitblock_has_bit(ref_Callouts.HexRef_starts)){
     341        BitBlockForwardIterator iter_HexRef_check(&ref_Callouts.HexRef_starts);
     342        validate_block(iter_HexRef_check, block_base, HexRef_check);
     343    }
     344    if(bitblock_has_bit(check_streams.att_refs)){
     345        BitBlockForwardIterator iter_AttRef_check(&check_streams.att_refs);
     346        validate_block(iter_AttRef_check, block_base, AttRef_check);
    375347    }
    376348
    377349    if(error_tracker.Has_Noted_Error()){
    378             tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
    379             ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
    380             exit(-1);
     350        int error_line, error_column;
     351        tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
     352        ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
     353        exit(-1);
    381354    }
    382355
  • proto/SymbolTable/symtab_identity_template.cpp

    r1648 r1684  
    3333PBGSIdentitySymbolTable pbgs_symbol_table;
    3434
    35 /* StreamScan & Post Process Declarations */
    36 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    37 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    38 
    3935static inline int NameStrt_check(int pos);
    4036static inline int Name_check(int pos);
     
    5046static inline void s2p_do_block(BytePack U8[], Basis_bits & basis_bits);
    5147static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    52 static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, int chars_avail);
     48static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail);
     49
    5350void do_process(FILE *infile, FILE *outfile);
     51
     52static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     53static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
     54static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
     55
    5456static inline int ElemStart_grouping(int start_pos, int L) ;
    5557static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count);
     
    137139}
    138140
    139 /* StreamScan & Post Process Definitions */
    140 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    141 
    142         int blk;
    143         int block_pos = 0;
    144         int pos;
    145 
    146         for (blk = 0; blk < blk_count; blk++) {
    147                 ScanBlock s = stream[blk];
    148                 while(s) {
    149                         pos = (cfzl(s) + block_pos);
    150                         int code = (ProcessPos(pos));
    151                         if (code) {
    152                                 *error_pos_in_block = pos;
    153                                 return code; // error code
    154                         }
    155                         s = s & (s-1);  // clear rightmost bit.
    156                 }
    157                 block_pos += 8 * sizeof(ScanBlock);
    158         }
    159         return 0;
    160 }
    161 
    162141static inline int compute_hash_value (int lgth, int start)
    163142{
     
    168147
    169148static inline int ElemStart_grouping(int start_pos, int L) {
    170     int start = block_base + start_pos;
    171     int end = start + L;
    172     int hashvalue = compute_hash_value(L, start - block_base);
     149    int hashvalue = compute_hash_value(L, start_pos - block_base);
    173150    int gid = 0;
    174151
     
    176153    {
    177154    case 1:
    178         gid = pbgs_symbol_table.Lookup_or_Insert_Name<1>(source + start, hashvalue);
     155        gid = pbgs_symbol_table.Lookup_or_Insert_Name<1>(source + start_pos, hashvalue);
    179156        break;
    180157    case 2:
    181         gid = pbgs_symbol_table.Lookup_or_Insert_Name<2>(source + start, hashvalue);
     158        gid = pbgs_symbol_table.Lookup_or_Insert_Name<2>(source + start_pos, hashvalue);
    182159        break;
    183160    case 3:
    184         gid = pbgs_symbol_table.Lookup_or_Insert_Name<3>(source + start, hashvalue);
     161        gid = pbgs_symbol_table.Lookup_or_Insert_Name<3>(source + start_pos, hashvalue);
    185162        break;
    186163    case 4:
    187         gid = pbgs_symbol_table.Lookup_or_Insert_Name<4>(source + start, hashvalue);
     164        gid = pbgs_symbol_table.Lookup_or_Insert_Name<4>(source + start_pos, hashvalue);
    188165        break;
    189166    case 5:
    190         gid = pbgs_symbol_table.Lookup_or_Insert_Name<5>(source + start, hashvalue);
     167        gid = pbgs_symbol_table.Lookup_or_Insert_Name<5>(source + start_pos, hashvalue);
    191168        break;
    192169    case 6:
    193         gid = pbgs_symbol_table.Lookup_or_Insert_Name<6>(source + start, hashvalue);
     170        gid = pbgs_symbol_table.Lookup_or_Insert_Name<6>(source + start_pos, hashvalue);
    194171        break;
    195172    case 7:
    196         gid = pbgs_symbol_table.Lookup_or_Insert_Name<7>(source + start, hashvalue);
     173        gid = pbgs_symbol_table.Lookup_or_Insert_Name<7>(source + start_pos, hashvalue);
    197174        break;
    198175    case 8:
    199         gid = pbgs_symbol_table.Lookup_or_Insert_Name<8>(source + start, hashvalue);
     176        gid = pbgs_symbol_table.Lookup_or_Insert_Name<8>(source + start_pos, hashvalue);
    200177        break;
    201178    case 9:
    202         gid = pbgs_symbol_table.Lookup_or_Insert_Name<9>(source + start, hashvalue);
     179        gid = pbgs_symbol_table.Lookup_or_Insert_Name<9>(source + start_pos, hashvalue);
    203180        break;
    204181    case 10:
    205         gid = pbgs_symbol_table.Lookup_or_Insert_Name<10>(source + start, hashvalue);
     182        gid = pbgs_symbol_table.Lookup_or_Insert_Name<10>(source + start_pos, hashvalue);
    206183        break;
    207184    case 11:
    208         gid = pbgs_symbol_table.Lookup_or_Insert_Name<11>(source + start, hashvalue);
     185        gid = pbgs_symbol_table.Lookup_or_Insert_Name<11>(source + start_pos, hashvalue);
    209186        break;
    210187    case 12:
    211         gid = pbgs_symbol_table.Lookup_or_Insert_Name<12>(source + start, hashvalue);
     188        gid = pbgs_symbol_table.Lookup_or_Insert_Name<12>(source + start_pos, hashvalue);
    212189        break;
    213190    case 13:
    214         gid = pbgs_symbol_table.Lookup_or_Insert_Name<13>(source + start, hashvalue);
     191        gid = pbgs_symbol_table.Lookup_or_Insert_Name<13>(source + start_pos, hashvalue);
    215192        break;
    216193    case 14:
    217         gid = pbgs_symbol_table.Lookup_or_Insert_Name<14>(source + start, hashvalue);
     194        gid = pbgs_symbol_table.Lookup_or_Insert_Name<14>(source + start_pos, hashvalue);
    218195        break;
    219196    case 15:
    220         gid = pbgs_symbol_table.Lookup_or_Insert_Name<15>(source + start, hashvalue);
     197        gid = pbgs_symbol_table.Lookup_or_Insert_Name<15>(source + start_pos, hashvalue);
    221198        break;
    222199    case 16:
    223         gid = pbgs_symbol_table.Lookup_or_Insert_Name<16>(source + start, hashvalue);
     200        gid = pbgs_symbol_table.Lookup_or_Insert_Name<16>(source + start_pos, hashvalue);
    224201        break;
    225202    default:
    226         gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, L);
     203        gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start_pos, hashvalue, L);
    227204        break;
    228205    }
     
    230207    gids.push_back(gid);
    231208#if DEBUG
     209    int end = start_pos + L;
    232210    char* symbol = new char[L+1];
    233     strncpy ( symbol, source + start, L );
     211    strncpy ( symbol, source + start_pos, L );
    234212    symbol[L] ='\0';
    235     printf ("%s | L: %i | start: %i[%i] | end: %i[%i] | gid: %i | hashvalue: %i | symbol: %s\n", __FUNCTION__, L, start, start-buffer_base, end, end-buffer_base, gid, hashvalue, symbol );
     213    printf ("%s | L: %i | start: %i[%i] | end: %i[%i] | gid: %i | hashvalue: %i | symbol: %s\n", __FUNCTION__, L, start_pos, start_pos-buffer_base, end, end-buffer_base, gid, hashvalue, symbol );
    236214    delete symbol; symbol = 0;
    237215#endif
     
    239217}
    240218
    241 static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count) {
    242     int blk;
    243     int block_pos = 0;
    244     for (blk = 0; blk < blk_count; blk++) {
    245         ScanBlock s = stream[blk];
    246         while(s) {
    247             int start_pos = cfzl(s) + block_pos;
    248             int end_pos = ScanForwardPos (&elem_ends, start_pos);
    249             if (end_pos)
    250             {
    251                 ElemStart_grouping(start_pos, end_pos - start_pos);
    252             }
    253             s = s & (s-1);  // clear rightmost bit.
    254         }
    255         block_pos += 8 * sizeof(ScanBlock);
    256     }
    257     return 0;
    258 }
    259 
    260219static inline int NameStrt_check(int pos) {
    261         int block_pos = block_base + pos;
    262         if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[block_pos]) == 0){
     220        if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[pos]) == 0){
    263221              return XMLTestSuiteError::NAME_START;
    264222        }
     
    267225
    268226static inline int Name_check(int pos) {
    269         int block_pos = block_base + pos;
    270         if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[block_pos]) == 0){
     227        if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[pos]) == 0){
    271228                  return XMLTestSuiteError::NAME;
    272229        }
     
    274231}
    275232
    276 static inline int PIName_check(int pos) {
    277         int block_pos = block_base + pos;
    278         int file_pos = block_pos+buffer_base;
    279         if (at_XxMmLll<ASCII>((unsigned char*)&source[block_pos]) && (source[block_pos+3]=='?' || source[block_pos+3]<= ' ')) {
     233static inline int PIName_check(int pos, int file_pos) {
     234        if (at_XxMmLll<ASCII>((unsigned char*)&source[pos]) && (source[pos+3]=='?' || source[pos+3]<= ' ')) {
    280235              // "<?xml" legal at start of file.
    281236              if (!((file_pos == 2) && at_XmlDecl_start<ASCII>((unsigned char*)&source[0]))) {
     
    287242
    288243static inline int CD_check(int pos) {
    289         int block_pos = block_base + pos;
    290         if (!at_CDATA1<ASCII>((unsigned char*)&source[block_pos])){
     244        if (!at_CDATA1<ASCII>((unsigned char*)&source[pos])){
    291245                  return XMLTestSuiteError::CDATA;
    292246        }
     
    295249
    296250static inline int GenRef_check(int pos) {
    297         int block_pos = block_base + pos;
    298         unsigned char* s = (unsigned char*)&source[block_pos];
     251        unsigned char* s = (unsigned char*)&source[pos];
    299252        if (!(at_Ref_gt<ASCII>(s)||at_Ref_lt<ASCII>(s)||at_Ref_amp<ASCII>(s)||at_Ref_quot<ASCII>(s)||at_Ref_apos<ASCII>(s))){
    300253              return XMLTestSuiteError::UNDEFREF;
     
    304257
    305258static inline int HexRef_check(int pos) {
    306         int block_pos = block_base + pos;
    307         unsigned char* s = (unsigned char*)&source[block_pos];
     259        unsigned char* s = (unsigned char*)&source[pos];
    308260        int ch_val = 0;
    309261        while(at_HexDigit<ASCII>(s)){
     
    324276
    325277static inline int DecRef_check(int pos) {
    326         int block_pos = block_base + pos;
    327         unsigned char* s = (unsigned char*)&source[block_pos];
     278        unsigned char* s = (unsigned char*)&source[pos];
    328279        int ch_val = 0;
    329280        while(at_HexDigit<ASCII>(s)){
     
    344295
    345296static inline int AttRef_check(int pos) {
    346         int block_pos = block_base + pos;
    347         unsigned char* s = (unsigned char*)&source[block_pos];
     297        unsigned char* s = (unsigned char*)&source[pos];
    348298        int ch_val = 0;
    349299        if(s[0]=='#'){
     
    372322}
    373323
     324static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     325
     326        BitBlockForwardIterator end;
     327        int pos, block_pos;
     328
     329        while(start != end) {
     330
     331                block_pos = block_base + *start;
     332                int rv = is_valid(block_pos);
     333
     334                if (rv) {
     335                        int error_line, error_column;
     336                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     337                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     338                        exit(-1);
     339                }
     340                start++;
     341        }
     342}
     343
     344static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     345
     346        BitBlockForwardIterator end;
     347        int pos, block_pos, file_pos;
     348
     349        while(start != end) {
     350
     351                block_pos = block_base + *start;
     352                file_pos = block_pos+buffer_base;
     353
     354
     355                int rv = is_valid(block_pos, file_pos);
     356
     357                if (rv) {
     358                        int error_line, error_column;
     359                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     360                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     361                        exit(-1);
     362                }
     363                start++;
     364        }
     365}
     366
     367inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
     368
     369    BitBlockForwardIterator end;
     370    int start_pos, end_pos, lgth;
     371
     372    while(start != end) {
     373        start_pos = *start;
     374        end_pos = ScanForwardPos (&elem_ends, start_pos);
     375        if (end_pos)
     376        {
     377            lgth = end_pos - start_pos;
     378            start_pos += block_base;
     379            ElemStart_grouping(start_pos, lgth);
     380        }
     381        start++;
     382    }
     383}
     384
    374385static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail){
    375386
    376387    tracker.StoreNewlines(lex.LF);
    377     int rv, error_pos_in_block, error_line, error_column;
    378388    elem_ends = tag_Callouts.ElemName_ends;
    379389    hashvalues[1] = hash_data.Hash_value;
     
    382392    {
    383393        int end_pos = count_forward_zeroes(elem_ends);
    384         ElemStart_grouping (last_elem_start, end_pos-last_elem_start);
     394        ElemStart_grouping (last_elem_start + block_base, end_pos-last_elem_start);
    385395        block_boundary_case = false;
    386396    }
     
    388398    if ( bitblock_has_bit(tag_Callouts.ElemName_starts) )
    389399    {
    390         StreamScanLengthGrouping((ScanBlock *) &tag_Callouts.ElemName_starts, sizeof(BitBlock)/sizeof(ScanBlock));
     400        BitBlockForwardIterator iter_length_grouping(&tag_Callouts.ElemName_starts);
     401        validate_block_length_grouping(iter_length_grouping, block_base);
    391402    }
    392403
     
    395406
    396407    if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
    397       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), NameStrt_check, &error_pos_in_block);
    398       if (rv) {
    399               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    400               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    401               exit(-1);
    402       }
    403 
    404       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_names, sizeof(BitBlock)/sizeof(ScanBlock), Name_check, &error_pos_in_block);
    405       if (rv) {
    406               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    407               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    408               exit(-1);
    409       }
    410     }
    411 
     408        BitBlockForwardIterator iter_NameStrt_check(&check_streams.non_ascii_name_starts);
     409        validate_block(iter_NameStrt_check, block_base, NameStrt_check);
     410        BitBlockForwardIterator iter_Name_check(&check_streams.non_ascii_names);
     411        validate_block(iter_Name_check, block_base, Name_check);
     412    }
    412413    if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
    413       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.PI_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), PIName_check, &error_pos_in_block);
    414       if (rv) {
    415               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    416               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    417               exit(-1);
    418       }
    419     }
    420 
     414        BitBlockForwardIterator iter_PI_name_starts(&(ctCDPI_Callouts.PI_name_starts));
     415        validate_block(iter_PI_name_starts, block_base, buffer_base, PIName_check);
     416    }
    421417    if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
    422       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.CD_starts, sizeof(BitBlock)/sizeof(ScanBlock), CD_check, &error_pos_in_block);
    423       if (rv) {
    424               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    425               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    426               exit(-1);
    427       }
    428     }
    429 
    430     if (bitblock_has_bit(ref_Callouts.GenRef_starts)){
    431       rv = StreamScan((ScanBlock *) &ref_Callouts.GenRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), GenRef_check, &error_pos_in_block);
    432       if (rv) {
    433               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    434               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    435               exit(-1);
    436       }
    437     }
    438 
    439     if (bitblock_has_bit(ref_Callouts.DecRef_starts)){
    440       rv = StreamScan((ScanBlock *) &ref_Callouts.DecRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), DecRef_check, &error_pos_in_block);
    441       if (rv) {
    442               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    443               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    444               exit(-1);
    445       }
    446     }
    447 
    448     if (bitblock_has_bit(ref_Callouts.HexRef_starts)){
    449       rv = StreamScan((ScanBlock *) &ref_Callouts.HexRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), HexRef_check, &error_pos_in_block);
    450       if (rv) {
    451               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    452               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    453               exit(-1);
    454       }
    455     }
    456 
    457     if (bitblock_has_bit(check_streams.att_refs)){
    458       rv = StreamScan((ScanBlock *) &check_streams.att_refs, sizeof(BitBlock)/sizeof(ScanBlock), AttRef_check, &error_pos_in_block);
    459       if (rv) {
    460               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    461               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    462               exit(-1);
    463       }
     418        BitBlockForwardIterator iter_CD_check(&ctCDPI_Callouts.CD_starts);
     419        validate_block(iter_CD_check, block_base, CD_check);
     420    }
     421    if(bitblock_has_bit(ref_Callouts.GenRef_starts)){
     422        BitBlockForwardIterator iter_GenRef_check(&ref_Callouts.GenRef_starts);
     423        validate_block(iter_GenRef_check, block_base, GenRef_check);
     424    }
     425    if(bitblock_has_bit(ref_Callouts.DecRef_starts)){
     426        BitBlockForwardIterator iter_DecRef_check(&ref_Callouts.DecRef_starts);
     427        validate_block(iter_DecRef_check, block_base, DecRef_check);
     428    }
     429    if(bitblock_has_bit(ref_Callouts.HexRef_starts)){
     430        BitBlockForwardIterator iter_HexRef_check(&ref_Callouts.HexRef_starts);
     431        validate_block(iter_HexRef_check, block_base, HexRef_check);
     432    }
     433    if(bitblock_has_bit(check_streams.att_refs)){
     434        BitBlockForwardIterator iter_AttRef_check(&check_streams.att_refs);
     435        validate_block(iter_AttRef_check, block_base, AttRef_check);
    464436    }
    465437
    466438    if(error_tracker.Has_Noted_Error()){
    467             tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
    468             ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
    469             exit(-1);
     439        int error_line, error_column;
     440        tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
     441        ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
     442        exit(-1);
    470443    }
    471444
  • proto/SymbolTable/symtab_ls_template.cpp

    r1648 r1684  
    2929LSSymbolTable ls_symbol_table;
    3030
    31 /* StreamScan & Post Process Declarations */
    32 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    33 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    34 
    3531static inline int NameStrt_check(int pos);
    3632static inline int Name_check(int pos);
     
    4743static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    4844static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, int chars_avail);
     45void do_process(FILE *infile, FILE *outfile);
     46
     47static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     48static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
    4949static inline void do_symbol_table_lookup();
    50 
    51 void do_process(FILE *infile, FILE *outfile);
    5250
    5351int main(int argc, char * argv[]) {
     
    113111}
    114112
    115 /* StreamScan & Post Process Definitions */
    116 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    117 
    118         int blk;
    119         int block_pos = 0;
    120         int pos;
    121 
    122         for (blk = 0; blk < blk_count; blk++) {
    123                 ScanBlock s = stream[blk];
    124                 while(s) {
    125                         pos = (cfzl(s) + block_pos);
    126                         int code = (ProcessPos(pos));
    127                         if (code) {
    128                                 *error_pos_in_block = pos;
    129                                 return code; // error code
    130                         }
    131                         s = s & (s-1);  // clear rightmost bit.
    132                 }
    133                 block_pos += 8 * sizeof(ScanBlock);
    134         }
    135         return 0;
    136 }
    137 
    138113static inline int ElemStrt_check(int pos) {
    139         int block_pos = block_base + pos;
    140         elem_starts_buf.push(buffer_base + block_pos);
     114        elem_starts_buf.push(buffer_base + pos);
    141115        return 0;
    142116}
    143117
    144118static inline int ElemEnd_check(int pos) {
    145         int block_pos = block_base + pos;
    146         elem_ends_buf.push(buffer_base + block_pos);
     119        elem_ends_buf.push(buffer_base + pos);
    147120        return 0;
    148121}
    149122
    150123static inline int NameStrt_check(int pos) {
    151         int block_pos = block_base + pos;
    152         if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[block_pos]) == 0){
     124        if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[pos]) == 0){
    153125              return XMLTestSuiteError::NAME_START;
    154126        }
     
    157129
    158130static inline int Name_check(int pos) {
    159         int block_pos = block_base + pos;
    160         if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[block_pos]) == 0){
     131        if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[pos]) == 0){
    161132                  return XMLTestSuiteError::NAME;
    162133        }
     
    164135}
    165136
    166 static inline int PIName_check(int pos) {
    167         int block_pos = block_base + pos;
    168         int file_pos = block_pos+buffer_base;
    169         if (at_XxMmLll<ASCII>((unsigned char*)&source[block_pos]) && (source[block_pos+3]=='?' || source[block_pos+3]<= ' ')) {
     137static inline int PIName_check(int pos, int file_pos) {
     138        if (at_XxMmLll<ASCII>((unsigned char*)&source[pos]) && (source[pos+3]=='?' || source[pos+3]<= ' ')) {
    170139              // "<?xml" legal at start of file.
    171140              if (!((file_pos == 2) && at_XmlDecl_start<ASCII>((unsigned char*)&source[0]))) {
     
    177146
    178147static inline int CD_check(int pos) {
    179         int block_pos = block_base + pos;
    180         if (!at_CDATA1<ASCII>((unsigned char*)&source[block_pos])){
     148        if (!at_CDATA1<ASCII>((unsigned char*)&source[pos])){
    181149                  return XMLTestSuiteError::CDATA;
    182150        }
     
    185153
    186154static inline int GenRef_check(int pos) {
    187         int block_pos = block_base + pos;
    188         unsigned char* s = (unsigned char*)&source[block_pos];
     155        unsigned char* s = (unsigned char*)&source[pos];
    189156        if (!(at_Ref_gt<ASCII>(s)||at_Ref_lt<ASCII>(s)||at_Ref_amp<ASCII>(s)||at_Ref_quot<ASCII>(s)||at_Ref_apos<ASCII>(s))){
    190157              return XMLTestSuiteError::UNDEFREF;
     
    194161
    195162static inline int HexRef_check(int pos) {
    196         int block_pos = block_base + pos;
    197         unsigned char* s = (unsigned char*)&source[block_pos];
     163        unsigned char* s = (unsigned char*)&source[pos];
    198164        int ch_val = 0;
    199165        while(at_HexDigit<ASCII>(s)){
     
    214180
    215181static inline int DecRef_check(int pos) {
    216         int block_pos = block_base + pos;
    217         unsigned char* s = (unsigned char*)&source[block_pos];
     182        unsigned char* s = (unsigned char*)&source[pos];
    218183        int ch_val = 0;
    219184        while(at_HexDigit<ASCII>(s)){
     
    234199
    235200static inline int AttRef_check(int pos) {
    236         int block_pos = block_base + pos;
    237         unsigned char* s = (unsigned char*)&source[block_pos];
     201        unsigned char* s = (unsigned char*)&source[pos];
    238202        int ch_val = 0;
    239203        if(s[0]=='#'){
     
    262226}
    263227
     228static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     229
     230        BitBlockForwardIterator end;
     231        int pos, block_pos;
     232
     233        while(start != end) {
     234
     235                block_pos = block_base + *start;
     236                int rv = is_valid(block_pos);
     237
     238                if (rv) {
     239                        int error_line, error_column;
     240                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     241                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     242                        exit(-1);
     243                }
     244                start++;
     245        }
     246}
     247
     248static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     249
     250        BitBlockForwardIterator end;
     251        int pos, block_pos, file_pos;
     252
     253        while(start != end) {
     254
     255                block_pos = block_base + *start;
     256                file_pos = block_pos+buffer_base;
     257
     258
     259                int rv = is_valid(block_pos, file_pos);
     260
     261                if (rv) {
     262                        int error_line, error_column;
     263                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     264                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     265                        exit(-1);
     266                }
     267                start++;
     268        }
     269}
     270
     271
    264272static inline void do_symbol_table_lookup()
    265273{
     
    290298
    291299    tracker.StoreNewlines(lex.LF);
    292     int rv, error_pos_in_block, error_line, error_column;
    293300
    294301    if ( bitblock_has_bit(tag_Callouts.ElemName_starts))
    295302    {
    296         StreamScan((ScanBlock *) &tag_Callouts.ElemName_starts, sizeof(BitBlock)/sizeof(ScanBlock), ElemStrt_check, &error_pos_in_block);
     303        BitBlockForwardIterator iter_length_grouping_starts(&tag_Callouts.ElemName_starts);
     304        validate_block(iter_length_grouping_starts, block_base, ElemStrt_check);
    297305    }
    298306
    299307    if ( bitblock_has_bit(tag_Callouts.ElemName_ends) )
    300308    {
    301         StreamScan((ScanBlock *) &tag_Callouts.ElemName_ends, sizeof(BitBlock)/sizeof(ScanBlock), ElemEnd_check, &error_pos_in_block);
     309        BitBlockForwardIterator iter_length_grouping_ends(&tag_Callouts.ElemName_ends);
     310        validate_block(iter_length_grouping_ends, block_base, ElemEnd_check);
    302311    }
    303312
    304313    do_symbol_table_lookup();
    305314
     315
    306316    if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
    307       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), NameStrt_check, &error_pos_in_block);
    308       if (rv) {
    309               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    310               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    311               exit(-1);
    312       }
    313 
    314       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_names, sizeof(BitBlock)/sizeof(ScanBlock), Name_check, &error_pos_in_block);
    315       if (rv) {
    316               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    317               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    318               exit(-1);
    319       }
    320     }
    321 
     317        BitBlockForwardIterator iter_NameStrt_check(&check_streams.non_ascii_name_starts);
     318        validate_block(iter_NameStrt_check, block_base, NameStrt_check);
     319        BitBlockForwardIterator iter_Name_check(&check_streams.non_ascii_names);
     320        validate_block(iter_Name_check, block_base, Name_check);
     321    }
    322322    if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
    323       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.PI_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), PIName_check, &error_pos_in_block);
    324       if (rv) {
    325               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    326               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    327               exit(-1);
    328       }
    329     }
    330 
     323        BitBlockForwardIterator iter_PI_name_starts(&(ctCDPI_Callouts.PI_name_starts));
     324        validate_block(iter_PI_name_starts, block_base, buffer_base, PIName_check);
     325    }
    331326    if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
    332       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.CD_starts, sizeof(BitBlock)/sizeof(ScanBlock), CD_check, &error_pos_in_block);
    333       if (rv) {
    334               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    335               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    336               exit(-1);
    337       }
    338     }
    339 
    340     if (bitblock_has_bit(ref_Callouts.GenRef_starts)){
    341       rv = StreamScan((ScanBlock *) &ref_Callouts.GenRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), GenRef_check, &error_pos_in_block);
    342       if (rv) {
    343               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    344               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    345               exit(-1);
    346       }
    347     }
    348 
    349     if (bitblock_has_bit(ref_Callouts.DecRef_starts)){
    350       rv = StreamScan((ScanBlock *) &ref_Callouts.DecRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), DecRef_check, &error_pos_in_block);
    351       if (rv) {
    352               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    353               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    354               exit(-1);
    355       }
    356     }
    357 
    358     if (bitblock_has_bit(ref_Callouts.HexRef_starts)){
    359       rv = StreamScan((ScanBlock *) &ref_Callouts.HexRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), HexRef_check, &error_pos_in_block);
    360       if (rv) {
    361               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    362               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    363               exit(-1);
    364       }
    365     }
    366 
    367     if (bitblock_has_bit(check_streams.att_refs)){
    368       rv = StreamScan((ScanBlock *) &check_streams.att_refs, sizeof(BitBlock)/sizeof(ScanBlock), AttRef_check, &error_pos_in_block);
    369       if (rv) {
    370               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    371               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    372               exit(-1);
    373       }
     327        BitBlockForwardIterator iter_CD_check(&ctCDPI_Callouts.CD_starts);
     328        validate_block(iter_CD_check, block_base, CD_check);
     329    }
     330    if(bitblock_has_bit(ref_Callouts.GenRef_starts)){
     331        BitBlockForwardIterator iter_GenRef_check(&ref_Callouts.GenRef_starts);
     332        validate_block(iter_GenRef_check, block_base, GenRef_check);
     333    }
     334    if(bitblock_has_bit(ref_Callouts.DecRef_starts)){
     335        BitBlockForwardIterator iter_DecRef_check(&ref_Callouts.DecRef_starts);
     336        validate_block(iter_DecRef_check, block_base, DecRef_check);
     337    }
     338    if(bitblock_has_bit(ref_Callouts.HexRef_starts)){
     339        BitBlockForwardIterator iter_HexRef_check(&ref_Callouts.HexRef_starts);
     340        validate_block(iter_HexRef_check, block_base, HexRef_check);
     341    }
     342    if(bitblock_has_bit(check_streams.att_refs)){
     343        BitBlockForwardIterator iter_AttRef_check(&check_streams.att_refs);
     344        validate_block(iter_AttRef_check, block_base, AttRef_check);
    374345    }
    375346
    376347    if(error_tracker.Has_Noted_Error()){
    377             tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
    378             ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
    379             exit(-1);
     348        int error_line, error_column;
     349        tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
     350        ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
     351        exit(-1);
    380352    }
    381353
  • proto/SymbolTable/symtab_pbgs_div_template.cpp

    r1648 r1684  
    3434PBGSDivSymbolTable pbgs_symbol_table;
    3535
    36 /* StreamScan & Post Process Declarations */
    37 //      static inline int StreamScanToFirst(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block)
    38 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block);
    39 
    4036static inline int NameStrt_check(int pos);
    4137static inline int Name_check(int pos);
     
    5147static inline void s2p_do_block(BytePack U8[], Basis_bits & basis_bits);
    5248static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    53 static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, int chars_avail);
     49static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail);
     50
    5451void do_process(FILE *infile, FILE *outfile);
     52
     53static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     54static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
     55template <int L> static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
     56
    5557static inline int ScanForwardPos(BitBlock * block, int pos);
    5658static inline int compute_hash_value (int lgth, int start);
     
    128130    if (bitblock_has_bit(temp))
    129131    {
    130 #if DEBUG
    131         printf ("There is a 1 bit in the block\n");
    132 #endif
    133132        return count_forward_zeroes (temp);
    134133    }
    135     else
    136     {
    137 #if DEBUG
    138         printf ("There is no more 1 bit in the block. pos: %i\n", pos);
    139 #endif
    140         //handle boundary case
    141         block_boundary_case = true;
    142         last_elem_start = pos - BLOCK_SIZE;
    143 #if DEBUG
    144         printf ("last_elem_start: %i\n", last_elem_start);
    145 #endif
    146         return 0;
    147     }
    148 }
    149 
    150 /* StreamScan & Post Process Definitions */
    151 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    152 
    153         int blk;
    154         int block_pos = 0;
    155         int pos;
    156 
    157         for (blk = 0; blk < blk_count; blk++) {
    158                 ScanBlock s = stream[blk];
    159                 while(s) {
    160                         pos = (cfzl(s) + block_pos);
    161                         int code = (ProcessPos(pos));
    162                         if (code) {
    163                                 *error_pos_in_block = pos;
    164                                 return code; // error code
    165                         }
    166                         s = s & (s-1);  // clear rightmost bit.
    167                 }
    168                 block_pos += 8 * sizeof(ScanBlock);
    169         }
    170         return 0;
     134    return 0;
    171135}
    172136
     
    180144// length in [1,16]
    181145template <int L>
    182 static inline int ElemEnd_grouping(int end_pos) {
    183     int end = block_base + end_pos;
     146static inline int ElemEnd_grouping(int end) {
    184147    int start = end - L;
    185148    int hashvalue = compute_hash_value(L, start - block_base);
     
    197160
    198161// length > 16
    199 static inline int ElemStart_grouping(int start_pos, int lgth) {
    200     int start = start_pos + block_base;
     162static inline int ElemStart_grouping(int start, int lgth) {
    201163    int hashvalue = compute_hash_value(lgth, start - block_base);
    202164    int gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
     
    211173}
    212174
    213 // L = 2, pass in bitstream for symbols length [1,2]
    214 // L = 4, pass in bitstream for symbols length [3,4]
    215 // L = 6, pass in bitstream for symbols length [5,6]
    216 // L = 8, pass in bitstream for symbols length [7,8]
    217 // L = 10, pass in bitstream for symbols length [9,10]
    218 // L = 12, pass in bitstream for symbols length [11,12]
    219 // L = 14, pass in bitstream for symbols length [13,14]
    220 // L = 16, pass in bitstream for symbols length [15,16]
    221 // L = 17, pass in bitstream for symbols length longer than 16
    222 template <int L>
    223 static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count) {
    224     int blk;
    225     int block_pos = 0;
    226     for (blk = 0; blk < blk_count; blk++) {
    227         ScanBlock s = stream[blk];
    228         while(s) {
    229             int end_pos = cfzl(s) + block_pos;
    230             ElemEnd_grouping<L>(end_pos);
    231             s = s & (s-1);  // clear rightmost bit.
    232         }
    233         block_pos += 8 * sizeof(ScanBlock);
    234     }
    235     return 0;
    236 }
    237 
    238 template <>
    239 inline int StreamScanLengthGrouping<17>(ScanBlock * stream, int blk_count) {
    240     int blk;
    241     int block_pos = 0;
    242     for (blk = 0; blk < blk_count; blk++) {
    243         ScanBlock s = stream[blk];
    244         while(s) {
    245             int start_pos = cfzl(s) + block_pos;
    246             int end_pos = ScanForwardPos (&elem_ends, start_pos);
    247             if (end_pos)
    248             {
    249                 ElemStart_grouping(start_pos - 16, end_pos - start_pos + 16);
    250             }
    251             s = s & (s-1);  // clear rightmost bit.
    252         }
    253         block_pos += 8 * sizeof(ScanBlock);
    254     }
    255     return 0;
    256 }
    257 
    258175static inline int NameStrt_check(int pos) {
    259         int block_pos = block_base + pos;
    260         if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[block_pos]) == 0){
     176        if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[pos]) == 0){
    261177              return XMLTestSuiteError::NAME_START;
    262178        }
     
    265181
    266182static inline int Name_check(int pos) {
    267         int block_pos = block_base + pos;
    268         if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[block_pos]) == 0){
     183        if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[pos]) == 0){
    269184                  return XMLTestSuiteError::NAME;
    270185        }
     
    272187}
    273188
    274 static inline int PIName_check(int pos) {
    275         int block_pos = block_base + pos;
    276         int file_pos = block_pos+buffer_base;
    277         if (at_XxMmLll<ASCII>((unsigned char*)&source[block_pos]) && (source[block_pos+3]=='?' || source[block_pos+3]<= ' ')) {
     189static inline int PIName_check(int pos, int file_pos) {
     190        if (at_XxMmLll<ASCII>((unsigned char*)&source[pos]) && (source[pos+3]=='?' || source[pos+3]<= ' ')) {
    278191              // "<?xml" legal at start of file.
    279192              if (!((file_pos == 2) && at_XmlDecl_start<ASCII>((unsigned char*)&source[0]))) {
     
    285198
    286199static inline int CD_check(int pos) {
    287         int block_pos = block_base + pos;
    288         if (!at_CDATA1<ASCII>((unsigned char*)&source[block_pos])){
     200        if (!at_CDATA1<ASCII>((unsigned char*)&source[pos])){
    289201                  return XMLTestSuiteError::CDATA;
    290202        }
     
    293205
    294206static inline int GenRef_check(int pos) {
    295         int block_pos = block_base + pos;
    296         unsigned char* s = (unsigned char*)&source[block_pos];
     207        unsigned char* s = (unsigned char*)&source[pos];
    297208        if (!(at_Ref_gt<ASCII>(s)||at_Ref_lt<ASCII>(s)||at_Ref_amp<ASCII>(s)||at_Ref_quot<ASCII>(s)||at_Ref_apos<ASCII>(s))){
    298209              return XMLTestSuiteError::UNDEFREF;
     
    302213
    303214static inline int HexRef_check(int pos) {
    304         int block_pos = block_base + pos;
    305         unsigned char* s = (unsigned char*)&source[block_pos];
     215        unsigned char* s = (unsigned char*)&source[pos];
    306216        int ch_val = 0;
    307217        while(at_HexDigit<ASCII>(s)){
     
    322232
    323233static inline int DecRef_check(int pos) {
    324         int block_pos = block_base + pos;
    325         unsigned char* s = (unsigned char*)&source[block_pos];
     234        unsigned char* s = (unsigned char*)&source[pos];
    326235        int ch_val = 0;
    327236        while(at_HexDigit<ASCII>(s)){
     
    342251
    343252static inline int AttRef_check(int pos) {
    344         int block_pos = block_base + pos;
    345         unsigned char* s = (unsigned char*)&source[block_pos];
     253        unsigned char* s = (unsigned char*)&source[pos];
    346254        int ch_val = 0;
    347255        if(s[0]=='#'){
     
    370278}
    371279
     280static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     281
     282        BitBlockForwardIterator end;
     283        int pos, block_pos;
     284
     285        while(start != end) {
     286
     287                block_pos = block_base + *start;
     288                int rv = is_valid(block_pos);
     289
     290                if (rv) {
     291                        int error_line, error_column;
     292                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     293                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     294                        exit(-1);
     295                }
     296                start++;
     297        }
     298}
     299
     300static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     301
     302        BitBlockForwardIterator end;
     303        int pos, block_pos, file_pos;
     304
     305        while(start != end) {
     306
     307                block_pos = block_base + *start;
     308                file_pos = block_pos+buffer_base;
     309
     310
     311                int rv = is_valid(block_pos, file_pos);
     312
     313                if (rv) {
     314                        int error_line, error_column;
     315                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     316                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     317                        exit(-1);
     318                }
     319                start++;
     320        }
     321}
     322
     323// L = 2, pass in bitstream for symbols length [1,2]
     324// L = 4, pass in bitstream for symbols length [3,4]
     325// L = 6, pass in bitstream for symbols length [5,6]
     326// L = 8, pass in bitstream for symbols length [7,8]
     327// L = 10, pass in bitstream for symbols length [9,10]
     328// L = 12, pass in bitstream for symbols length [11,12]
     329// L = 14, pass in bitstream for symbols length [13,14]
     330// L = 16, pass in bitstream for symbols length [15,16]
     331// L = 17, pass in bitstream for symbols length longer than 16
     332template <int L>
     333static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
     334
     335    BitBlockForwardIterator end;
     336    int end_pos;
     337
     338    while(start != end) {
     339        end_pos = *start;
     340        ElemEnd_grouping<L>(end_pos + block_base);
     341        start++;
     342    }
     343}
     344
     345template <>
     346inline void validate_block_length_grouping<17>(BitBlockForwardIterator & start, int block_base) {
     347
     348    BitBlockForwardIterator end;
     349    int start_pos, end_pos;
     350
     351    while(start != end) {
     352        start_pos = *start;
     353        end_pos = ScanForwardPos (&elem_ends, start_pos);
     354        if (end_pos)
     355        {
     356            ElemStart_grouping(start_pos - 16 + block_base, end_pos - start_pos + 16);
     357        }
     358        else
     359        {
     360#if DEBUG
     361            printf ("There is no more 1 bit in the block. pos: %i | sym: %c%c[%c]\n", start_pos,
     362                    source[start_pos + block_base-2], source[start_pos + block_base-1], source[start_pos + block_base]);
     363#endif
     364            //handle boundary case
     365            block_boundary_case = true;
     366            last_elem_start = start_pos - 16 - BLOCK_SIZE;
     367#if DEBUG
     368            printf ("last_elem_start: %i\n", last_elem_start);
     369#endif
     370        }
     371        start++;
     372    }
     373}
     374
    372375static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail){
    373376
    374377    tracker.StoreNewlines(lex.LF);
    375     int rv, error_pos_in_block, error_line, error_column;
     378
    376379    elem_ends = tag_Callouts.ElemName_ends;
    377380    hashvalues[1] = hash_data.Hash_value;
     
    396399    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_1_to_2) )
    397400    {
    398         StreamScanLengthGrouping<2>((ScanBlock *) &tag_Callouts.ElemName_ends_1_to_2, sizeof(BitBlock)/sizeof(ScanBlock));
     401        BitBlockForwardIterator iter_length_grouping_2(&tag_Callouts.ElemName_ends_1_to_2);
     402        validate_block_length_grouping<2>(iter_length_grouping_2, block_base);
    399403    }
    400404
    401405    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_3_to_4) )
    402406    {
    403         StreamScanLengthGrouping<4>((ScanBlock *) &tag_Callouts.ElemName_ends_3_to_4, sizeof(BitBlock)/sizeof(ScanBlock));
     407        BitBlockForwardIterator iter_length_grouping_4(&tag_Callouts.ElemName_ends_3_to_4);
     408        validate_block_length_grouping<4>(iter_length_grouping_4, block_base);
    404409    }
    405410
    406411    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_5_to_6) )
    407412    {
    408         StreamScanLengthGrouping<6>((ScanBlock *) &tag_Callouts.ElemName_ends_5_to_6, sizeof(BitBlock)/sizeof(ScanBlock));
     413        BitBlockForwardIterator iter_length_grouping_6(&tag_Callouts.ElemName_ends_5_to_6);
     414        validate_block_length_grouping<6>(iter_length_grouping_6, block_base);
    409415    }
    410416
    411417    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_7_to_8) )
    412418    {
    413         StreamScanLengthGrouping<8>((ScanBlock *) &tag_Callouts.ElemName_ends_7_to_8, sizeof(BitBlock)/sizeof(ScanBlock));
     419        BitBlockForwardIterator iter_length_grouping_8(&tag_Callouts.ElemName_ends_7_to_8);
     420        validate_block_length_grouping<8>(iter_length_grouping_8, block_base);
    414421    }
    415422
    416423    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_9_to_10) )
    417424    {
    418         StreamScanLengthGrouping<10>((ScanBlock *) &tag_Callouts.ElemName_ends_9_to_10, sizeof(BitBlock)/sizeof(ScanBlock));
     425        BitBlockForwardIterator iter_length_grouping_10(&tag_Callouts.ElemName_ends_9_to_10);
     426        validate_block_length_grouping<10>(iter_length_grouping_10, block_base);
    419427    }
    420428
    421429    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_11_to_12) )
    422430    {
    423         StreamScanLengthGrouping<12>((ScanBlock *) &tag_Callouts.ElemName_ends_11_to_12, sizeof(BitBlock)/sizeof(ScanBlock));
     431        BitBlockForwardIterator iter_length_grouping_12(&tag_Callouts.ElemName_ends_11_to_12);
     432        validate_block_length_grouping<12>(iter_length_grouping_12, block_base);
    424433    }
    425434
    426435    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_13_to_14) )
    427436    {
    428         StreamScanLengthGrouping<14>((ScanBlock *) &tag_Callouts.ElemName_ends_13_to_14, sizeof(BitBlock)/sizeof(ScanBlock));
     437        BitBlockForwardIterator iter_length_grouping_14(&tag_Callouts.ElemName_ends_13_to_14);
     438        validate_block_length_grouping<14>(iter_length_grouping_14, block_base);
    429439    }
    430440
    431441    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_15_to_16) )
    432442    {
    433         StreamScanLengthGrouping<16>((ScanBlock *) &tag_Callouts.ElemName_ends_15_to_16, sizeof(BitBlock)/sizeof(ScanBlock));
     443        BitBlockForwardIterator iter_length_grouping_16(&tag_Callouts.ElemName_ends_15_to_16);
     444        validate_block_length_grouping<16>(iter_length_grouping_16, block_base);
    434445    }
    435446
    436447    if ( bitblock_has_bit(tag_Callouts.ElemName_remaining_ends) )
    437448    {
    438         StreamScanLengthGrouping<17>((ScanBlock *) &tag_Callouts.ElemName_remaining_ends, sizeof(BitBlock)/sizeof(ScanBlock));
     449        BitBlockForwardIterator iter_length_grouping_remaining(&tag_Callouts.ElemName_remaining_ends);
     450        validate_block_length_grouping<17>(iter_length_grouping_remaining, block_base);
    439451    }
    440452
     
    442454    memmove (&hashvalues[0], &hashvalues[1], 16);
    443455
    444 
    445456    if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
    446       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), NameStrt_check, &error_pos_in_block);
    447       if (rv) {
    448               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    449               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    450               exit(-1);
    451       }
    452 
    453       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_names, sizeof(BitBlock)/sizeof(ScanBlock), Name_check, &error_pos_in_block);
    454       if (rv) {
    455               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    456               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    457               exit(-1);
    458       }
    459     }
    460 
     457        BitBlockForwardIterator iter_NameStrt_check(&check_streams.non_ascii_name_starts);
     458        validate_block(iter_NameStrt_check, block_base, NameStrt_check);
     459        BitBlockForwardIterator iter_Name_check(&check_streams.non_ascii_names);
     460        validate_block(iter_Name_check, block_base, Name_check);
     461    }
    461462    if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
    462       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.PI_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), PIName_check, &error_pos_in_block);
    463       if (rv) {
    464               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    465               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    466               exit(-1);
    467       }
    468     }
    469 
     463        BitBlockForwardIterator iter_PI_name_starts(&(ctCDPI_Callouts.PI_name_starts));
     464        validate_block(iter_PI_name_starts, block_base, buffer_base, PIName_check);
     465    }
    470466    if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
    471       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.CD_starts, sizeof(BitBlock)/sizeof(ScanBlock), CD_check, &error_pos_in_block);
    472       if (rv) {
    473               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    474               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    475               exit(-1);
    476       }
    477     }
    478 
    479     if (bitblock_has_bit(ref_Callouts.GenRef_starts)){
    480       rv = StreamScan((ScanBlock *) &ref_Callouts.GenRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), GenRef_check, &error_pos_in_block);
    481       if (rv) {
    482               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    483               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    484               exit(-1);
    485       }
    486     }
    487 
    488     if (bitblock_has_bit(ref_Callouts.DecRef_starts)){
    489       rv = StreamScan((ScanBlock *) &ref_Callouts.DecRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), DecRef_check, &error_pos_in_block);
    490       if (rv) {
    491               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    492               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    493               exit(-1);
    494       }
    495     }
    496 
    497     if (bitblock_has_bit(ref_Callouts.HexRef_starts)){
    498       rv = StreamScan((ScanBlock *) &ref_Callouts.HexRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), HexRef_check, &error_pos_in_block);
    499       if (rv) {
    500               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    501               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    502               exit(-1);
    503       }
    504     }
    505 
    506     if (bitblock_has_bit(check_streams.att_refs)){
    507       rv = StreamScan((ScanBlock *) &check_streams.att_refs, sizeof(BitBlock)/sizeof(ScanBlock), AttRef_check, &error_pos_in_block);
    508       if (rv) {
    509               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    510               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    511               exit(-1);
    512       }
     467        BitBlockForwardIterator iter_CD_check(&ctCDPI_Callouts.CD_starts);
     468        validate_block(iter_CD_check, block_base, CD_check);
     469    }
     470    if(bitblock_has_bit(ref_Callouts.GenRef_starts)){
     471        BitBlockForwardIterator iter_GenRef_check(&ref_Callouts.GenRef_starts);
     472        validate_block(iter_GenRef_check, block_base, GenRef_check);
     473    }
     474    if(bitblock_has_bit(ref_Callouts.DecRef_starts)){
     475        BitBlockForwardIterator iter_DecRef_check(&ref_Callouts.DecRef_starts);
     476        validate_block(iter_DecRef_check, block_base, DecRef_check);
     477    }
     478    if(bitblock_has_bit(ref_Callouts.HexRef_starts)){
     479        BitBlockForwardIterator iter_HexRef_check(&ref_Callouts.HexRef_starts);
     480        validate_block(iter_HexRef_check, block_base, HexRef_check);
     481    }
     482    if(bitblock_has_bit(check_streams.att_refs)){
     483        BitBlockForwardIterator iter_AttRef_check(&check_streams.att_refs);
     484        validate_block(iter_AttRef_check, block_base, AttRef_check);
    513485    }
    514486
    515487    if(error_tracker.Has_Noted_Error()){
    516             tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
    517             ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
    518             exit(-1);
     488        int error_line, error_column;
     489        tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
     490        ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
     491        exit(-1);
    519492    }
    520493
  • proto/SymbolTable/symtab_pbgs_identity_template.cpp

    r1683 r1684  
    1818int block_base=0;
    1919int buffer_base=0;
     20char * source;
    2021int buffer_last;
    21 char * source;
     22
    2223LineColTracker tracker;
    2324TagMatcher matcher;
     25ErrorTracker error_tracker;
    2426BitBlock EOF_mask = simd<1>::constant<1>();
    25 ErrorTracker error_tracker;
    2627
    2728BitBlock elem_starts;
     
    3132vector <int> gids;
    3233PBGSIdentitySymbolTable pbgs_symbol_table;
     34
    3335
    3436static inline int NameStrt_check(int pos);
     
    4143static inline int AttRef_check(int pos);
    4244
    43 
    44 
    4545@global
    4646
    4747static inline void s2p_do_block(BytePack U8[], Basis_bits & basis_bits);
    4848static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    49 static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, int chars_avail);
     49static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, int chars_avail);
     50
    5051void do_process(FILE *infile, FILE *outfile);
     52
     53static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     54static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
     55template <int L> static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
     56
    5157static inline int ScanBackwardPos(BitBlock * block, int pos);
    5258static inline int compute_hash_value (int lgth, int start);
    5359template <int L> static inline int ElemEnd_grouping(int pos);
    54 template <int L> static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
    5560
    5661int main(int argc, char * argv[]) {
     62
    5763        char * infilename, * outfilename;
    5864        FILE *infile, *outfile;
     
    141147
    142148template <int L>
    143 static inline int ElemEnd_grouping(int pos) {
    144     int end = pos;
     149static inline int ElemEnd_grouping(int end) {
    145150    int start = end - L;
    146151    int hashvalue = compute_hash_value(L, start - block_base);
     
    158163
    159164template<>
    160 inline int ElemEnd_grouping<17>(int pos) {
    161     int end = pos;
    162     int start = ScanBackwardPos (&elem_starts, pos - block_base) + block_base;
     165inline int ElemEnd_grouping<17>(int end) {
     166    int start = ScanBackwardPos (&elem_starts, end - block_base) + block_base;
    163167    int lgth = end - start;
    164168    int hashvalue = compute_hash_value(lgth, start - block_base);
     
    298302}
    299303
     304/* StreamScan & Post Process Definitions */
     305static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
     306
     307        int blk;
     308        int block_pos = 0;
     309        int pos;
     310
     311        for (blk = 0; blk < blk_count; blk++) {
     312                ScanBlock s = stream[blk];
     313                while(s) {
     314                        pos = (cfzl(s) + block_pos);
     315                        int code = (ProcessPos(pos));
     316                        if (code) {
     317                                *error_pos_in_block = pos;
     318                                return code; // error code
     319                        }
     320                        s = s & (s-1);  // clear rightmost bit.
     321                }
     322                block_pos += 8 * sizeof(ScanBlock);
     323        }
     324        return 0;
     325}
    300326#else
    301 
    302 
    303 
    304327static inline int NameStrt_check(int pos) {
    305     if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[pos]) == 0){
    306         return XMLTestSuiteError::NAME_START;
    307     }
    308     return 0;
     328        if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[pos]) == 0){
     329              return XMLTestSuiteError::NAME_START;
     330        }
     331        return 0;
    309332}
    310333
    311334static inline int Name_check(int pos) {
    312     if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[pos]) == 0){
    313         return XMLTestSuiteError::NAME;
    314     }
    315     return 0;
     335        if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[pos]) == 0){
     336                  return XMLTestSuiteError::NAME;
     337        }
     338        return 0;
    316339}
    317340
     
    406429        return 0;
    407430}
    408 #endif
    409 
    410 /* StreamScan & Post Process Definitions */
    411 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    412 
    413         int blk;
    414         int block_pos = 0;
    415         int pos;
    416 
    417         for (blk = 0; blk < blk_count; blk++) {
    418                 ScanBlock s = stream[blk];
    419                 while(s) {
    420                         pos = (cfzl(s) + block_pos);
    421                         int code = (ProcessPos(pos));
    422                         if (code) {
    423                                 *error_pos_in_block = pos;
    424                                 return code; // error code
    425                         }
    426                         s = s & (s-1);  // clear rightmost bit.
    427                 }
    428                 block_pos += 8 * sizeof(ScanBlock);
    429         }
    430         return 0;
    431 }
    432 
    433 template <int L>
    434 static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
    435 
    436     BitBlockForwardIterator end;
    437     int block_pos;
    438 
    439     while(start != end) {
    440         block_pos = block_base + *start;
    441         ElemEnd_grouping<L>(block_pos);
    442         start++;
    443     }
    444 }
    445431
    446432static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
    447433
    448     BitBlockForwardIterator end;
    449     int pos, block_pos;
    450 
    451     while(start != end) {
    452 
    453         block_pos = block_base + *start;
    454         int rv = is_valid(block_pos);
    455 
    456         if (rv) {
    457                 int error_line, error_column;
    458                 tracker.get_Line_and_Column(block_pos, error_line, error_column);
    459                 ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    460                 exit(-1);
    461         }
    462         start++;
    463     }
    464 }
    465 
    466 static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
    467 
    468434        BitBlockForwardIterator end;
    469         int pos, block_pos, file_pos;
     435        int pos, block_pos;
    470436
    471437        while(start != end) {
    472438
    473439                block_pos = block_base + *start;
    474                 file_pos = block_pos+buffer_base;
    475 
    476 
    477                 int rv = is_valid(block_pos, file_pos);
     440                int rv = is_valid(block_pos);
    478441
    479442                if (rv) {
     
    487450}
    488451
     452static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     453
     454        BitBlockForwardIterator end;
     455        int pos, block_pos, file_pos;
     456
     457        while(start != end) {
     458
     459                block_pos = block_base + *start;
     460                file_pos = block_pos+buffer_base;
     461
     462
     463                int rv = is_valid(block_pos, file_pos);
     464
     465                if (rv) {
     466                        int error_line, error_column;
     467                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     468                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     469                        exit(-1);
     470                }
     471                start++;
     472        }
     473}
     474#endif
     475
     476
     477template <int L>
     478static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
     479
     480    BitBlockForwardIterator end;
     481    int block_pos;
     482
     483    while(start != end) {
     484        block_pos = block_base + *start;
     485        ElemEnd_grouping<L>(block_pos);
     486        start++;
     487    }
     488}
     489
    489490static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail){
    490491    tracker.StoreNewlines(lex.LF);
     
    493494    hashvalues[1] = hash_data.Hash_value;
    494495
     496#ifdef USE_ITER
    495497    BitBlockForwardIterator iter;
    496498
     
    595597        iter.init(&tag_Callouts.ElemName_ends_17_and_longer);
    596598        validate_block_length_grouping<17>(iter, block_base);
     599    }
     600
     601    // Store the last starting position in case we hit boundary case
     602    previous_block_last_elem_start = - count_reverse_zeroes (elem_starts);
     603
     604    //copy current hash value data as previous one.
     605    memmove (&hashvalues[0], &hashvalues[1], 16);
     606
     607    if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
     608            iter.init(&check_streams.non_ascii_name_starts);
     609            validate_block(iter, block_base, NameStrt_check);
     610            iter.init(&check_streams.non_ascii_names);
     611            validate_block(iter, block_base, Name_check);
     612    }
     613    if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
     614            iter.init(&(ctCDPI_Callouts.PI_name_starts));
     615            validate_block(iter, block_base, buffer_base, PIName_check);
     616    }
     617    if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
     618            iter.init(&ctCDPI_Callouts.CD_starts);
     619            validate_block(iter, block_base, CD_check);
     620    }
     621    if(bitblock_has_bit(ref_Callouts.GenRef_starts)){
     622            iter.init(&ref_Callouts.GenRef_starts);
     623            validate_block(iter, block_base, GenRef_check);
     624    }
     625    if(bitblock_has_bit(ref_Callouts.DecRef_starts)){
     626            iter.init(&ref_Callouts.DecRef_starts);
     627            validate_block(iter, block_base, DecRef_check);
     628    }
     629    if(bitblock_has_bit(ref_Callouts.HexRef_starts)){
     630            iter.init(&ref_Callouts.HexRef_starts);
     631            validate_block(iter, block_base, HexRef_check);
     632    }
     633    if(bitblock_has_bit(check_streams.att_refs)){
     634            iter.init(&check_streams.att_refs);
     635            validate_block(iter, block_base, AttRef_check);
     636    }
     637
     638    if(error_tracker.Has_Noted_Error()){
     639        int error_line, error_column;
     640        tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
     641        ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
     642        exit(-1);
     643    }
     644
     645    matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
     646    tracker.AdvanceBlock();
     647#else
     648    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_1) )
     649    {
     650        BitBlockForwardIterator iter_length_grouping_1(&tag_Callouts.ElemName_ends_1);
     651        validate_block_length_grouping<1>(iter_length_grouping_1, block_base);
     652    }
     653
     654    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_2) )
     655    {
     656        BitBlockForwardIterator iter_length_grouping_2(&tag_Callouts.ElemName_ends_2);
     657        validate_block_length_grouping<2>(iter_length_grouping_2, block_base);
     658    }
     659
     660    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_3) )
     661    {
     662        BitBlockForwardIterator iter_length_grouping_3(&tag_Callouts.ElemName_ends_3);
     663        validate_block_length_grouping<3>(iter_length_grouping_3, block_base);
     664    }
     665
     666    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_4) )
     667    {
     668        BitBlockForwardIterator iter_length_grouping_4(&tag_Callouts.ElemName_ends_4);
     669        validate_block_length_grouping<4>(iter_length_grouping_4, block_base);
     670    }
     671
     672    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_5) )
     673    {
     674        BitBlockForwardIterator iter_length_grouping_5(&tag_Callouts.ElemName_ends_5);
     675        validate_block_length_grouping<5>(iter_length_grouping_5, block_base);
     676    }
     677
     678    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_6) )
     679    {
     680        BitBlockForwardIterator iter_length_grouping_6(&tag_Callouts.ElemName_ends_6);
     681        validate_block_length_grouping<6>(iter_length_grouping_6, block_base);
     682    }
     683
     684    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_7) )
     685    {
     686        BitBlockForwardIterator iter_length_grouping_7(&tag_Callouts.ElemName_ends_7);
     687        validate_block_length_grouping<7>(iter_length_grouping_7, block_base);
     688    }
     689
     690    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_8) )
     691    {
     692        BitBlockForwardIterator iter_length_grouping_8(&tag_Callouts.ElemName_ends_8);
     693        validate_block_length_grouping<8>(iter_length_grouping_8, block_base);
     694    }
     695
     696    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_9) )
     697    {
     698        BitBlockForwardIterator iter_length_grouping_9(&tag_Callouts.ElemName_ends_9);
     699        validate_block_length_grouping<9>(iter_length_grouping_9, block_base);
     700    }
     701
     702    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_10) )
     703    {
     704        BitBlockForwardIterator iter_length_grouping_10(&tag_Callouts.ElemName_ends_10);
     705        validate_block_length_grouping<10>(iter_length_grouping_10, block_base);
     706    }
     707
     708    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_11) )
     709    {
     710        BitBlockForwardIterator iter_length_grouping_11(&tag_Callouts.ElemName_ends_11);
     711        validate_block_length_grouping<11>(iter_length_grouping_11, block_base);
     712    }
     713
     714    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_12) )
     715    {
     716        BitBlockForwardIterator iter_length_grouping_12(&tag_Callouts.ElemName_ends_12);
     717        validate_block_length_grouping<12>(iter_length_grouping_12, block_base);
     718    }
     719
     720    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_13) )
     721    {
     722        BitBlockForwardIterator iter_length_grouping_13(&tag_Callouts.ElemName_ends_13);
     723        validate_block_length_grouping<13>(iter_length_grouping_13, block_base);
     724    }
     725
     726    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_14) )
     727    {
     728        BitBlockForwardIterator iter_length_grouping_14(&tag_Callouts.ElemName_ends_14);
     729        validate_block_length_grouping<14>(iter_length_grouping_14, block_base);
     730    }
     731
     732    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_15) )
     733    {
     734        BitBlockForwardIterator iter_length_grouping_15(&tag_Callouts.ElemName_ends_15);
     735        validate_block_length_grouping<15>(iter_length_grouping_15, block_base);
     736    }
     737
     738    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_16) )
     739    {
     740        BitBlockForwardIterator iter_length_grouping_16(&tag_Callouts.ElemName_ends_16);
     741        validate_block_length_grouping<16>(iter_length_grouping_16, block_base);
     742    }
     743
     744    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_17_and_longer) )
     745    {
     746        BitBlockForwardIterator iter_length_grouping_remaining(&tag_Callouts.ElemName_ends_17_and_longer);
     747        validate_block_length_grouping<17>(iter_length_grouping_remaining, block_base);
    597748    }
    598749
     
    688839#else
    689840    if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
    690             iter.init(&check_streams.non_ascii_name_starts);
    691             validate_block(iter, block_base, NameStrt_check);
    692             iter.init(&check_streams.non_ascii_names);
    693             validate_block(iter, block_base, Name_check);
     841        BitBlockForwardIterator iter_NameStrt_check(&check_streams.non_ascii_name_starts);
     842        validate_block(iter_NameStrt_check, block_base, NameStrt_check);
     843        BitBlockForwardIterator iter_Name_check(&check_streams.non_ascii_names);
     844        validate_block(iter_Name_check, block_base, Name_check);
    694845    }
    695846    if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
    696             iter.init(&(ctCDPI_Callouts.PI_name_starts));
    697             validate_block(iter, block_base, buffer_base, PIName_check);
     847        BitBlockForwardIterator iter_PI_name_starts(&(ctCDPI_Callouts.PI_name_starts));
     848        validate_block(iter_PI_name_starts, block_base, buffer_base, PIName_check);
     849
    698850    }
    699851    if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
    700             iter.init(&ctCDPI_Callouts.CD_starts);
    701             validate_block(iter, block_base, CD_check);
     852        BitBlockForwardIterator iter_CD_check(&ctCDPI_Callouts.CD_starts);
     853        validate_block(iter_CD_check, block_base, CD_check);
    702854    }
    703855    if(bitblock_has_bit(ref_Callouts.GenRef_starts)){
    704             iter.init(&ref_Callouts.GenRef_starts);
    705             validate_block(iter, block_base, GenRef_check);
     856        BitBlockForwardIterator iter_GenRef_check(&ref_Callouts.GenRef_starts);
     857        validate_block(iter_GenRef_check, block_base, GenRef_check);
    706858    }
    707859    if(bitblock_has_bit(ref_Callouts.DecRef_starts)){
    708             iter.init(&ref_Callouts.DecRef_starts);
    709             validate_block(iter, block_base, DecRef_check);
     860        BitBlockForwardIterator iter_DecRef_check(&ref_Callouts.DecRef_starts);
     861        validate_block(iter_DecRef_check, block_base, DecRef_check);
    710862    }
    711863    if(bitblock_has_bit(ref_Callouts.HexRef_starts)){
    712             iter.init(&ref_Callouts.HexRef_starts);
    713             validate_block(iter, block_base, HexRef_check);
     864        BitBlockForwardIterator iter_HexRef_check(&ref_Callouts.HexRef_starts);
     865        validate_block(iter_HexRef_check, block_base, HexRef_check);
    714866    }
    715867    if(bitblock_has_bit(check_streams.att_refs)){
    716             iter.init(&check_streams.att_refs);
    717             validate_block(iter, block_base, AttRef_check);
     868        BitBlockForwardIterator iter_AttRef_check(&check_streams.att_refs);
     869        validate_block(iter_AttRef_check, block_base, AttRef_check);
    718870    }
    719871
    720872    if(error_tracker.Has_Noted_Error()){
    721             int error_line, error_column;
    722             tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
    723             ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
    724             exit(-1);
     873        int error_line, error_column;
     874        tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
     875        ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
     876        exit(-1);
    725877    }
    726878
     
    730882
    731883#endif
    732 
     884#endif
    733885}
    734886
  • proto/SymbolTable/symtab_pbgs_log_template.cpp

    r1648 r1684  
    5151static inline void s2p_do_block(BytePack U8[], Basis_bits & basis_bits);
    5252static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    53 static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, int chars_avail);
     53static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail);
     54
    5455void do_process(FILE *infile, FILE *outfile);
     56
     57static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     58static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
     59template <int L> static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base);
     60
    5561static inline int ScanBackwardPos(BitBlock * block, int pos);
    5662static inline int compute_hash_value (int lgth, int start);
     
    118124  basis_bits.bit_6 = simd_and(basis_bits.bit_6, EOF_mask);
    119125  basis_bits.bit_7 = simd_and(basis_bits.bit_7, EOF_mask);
    120 }
    121 
    122 /* StreamScan & Post Process Definitions */
    123 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    124 
    125         int blk;
    126         int block_pos = 0;
    127         int pos;
    128 
    129         for (blk = 0; blk < blk_count; blk++) {
    130                 ScanBlock s = stream[blk];
    131                 while(s) {
    132                         pos = (cfzl(s) + block_pos);
    133                         int code = (ProcessPos(pos));
    134                         if (code) {
    135                                 *error_pos_in_block = pos;
    136                                 return code; // error code
    137                         }
    138                         s = s & (s-1);  // clear rightmost bit.
    139                 }
    140                 block_pos += 8 * sizeof(ScanBlock);
    141         }
    142         return 0;
    143126}
    144127
     
    293276}
    294277
    295 // L = 4, pass in bitstream for symbols length [3,4]
    296 // L = 8, pass in bitstream for symbols length [5,8]
    297 // L = 16, pass in bitstream for symbols length [9,16]
    298 // L = 17, pass in bitstream for symbols length longer than 16
    299 template <int L>
    300 static inline int StreamScanLengthGrouping(ScanBlock * stream, int blk_count) {
    301     int blk;
    302     int block_pos = 0;
    303     for (blk = 0; blk < blk_count; blk++) {
    304         ScanBlock s = stream[blk];
    305         while(s) {
    306             int end_pos = cfzl(s) + block_pos;
    307             int start_pos = ScanBackwardPos (&elem_starts, end_pos);
    308             int length = end_pos - start_pos;
    309             ElemEnd_grouping<L>(start_pos, length);
    310             s = s & (s-1);  // clear rightmost bit.
    311         }
    312         block_pos += 8 * sizeof(ScanBlock);
    313     }
    314     return 0;
    315 }
    316 
    317 // pass in bitstream for symbols length 1
    318 template <>
    319 inline int StreamScanLengthGrouping<1>(ScanBlock * stream, int blk_count) {
    320     int blk;
    321     int block_pos = 0;
    322     for (blk = 0; blk < blk_count; blk++) {
    323         ScanBlock s = stream[blk];
    324         while(s) {
    325             int end_pos = cfzl(s) + block_pos;
    326             ElemEnd_grouping<1>(end_pos, 1);
    327             s = s & (s-1);  // clear rightmost bit.
    328         }
    329         block_pos += 8 * sizeof(ScanBlock);
    330     }
    331     return 0;
    332 }
    333 
    334 // pass in bitstream for symbols length 2
    335 template <>
    336 inline int StreamScanLengthGrouping<2>(ScanBlock * stream, int blk_count) {
    337     int blk;
    338     int block_pos = 0;
    339     for (blk = 0; blk < blk_count; blk++) {
    340         ScanBlock s = stream[blk];
    341         while(s) {
    342             int end_pos = cfzl(s) + block_pos;
    343             ElemEnd_grouping<2>(end_pos, 2);
    344             s = s & (s-1);  // clear rightmost bit.
    345         }
    346         block_pos += 8 * sizeof(ScanBlock);
    347     }
    348     return 0;
    349 }
    350 
    351278static inline int NameStrt_check(int pos) {
    352         int block_pos = block_base + pos;
    353         if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[block_pos]) == 0){
     279        if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[pos]) == 0){
    354280              return XMLTestSuiteError::NAME_START;
    355281        }
     
    358284
    359285static inline int Name_check(int pos) {
    360         int block_pos = block_base + pos;
    361         if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[block_pos]) == 0){
     286        if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[pos]) == 0){
    362287                  return XMLTestSuiteError::NAME;
    363288        }
     
    365290}
    366291
    367 static inline int PIName_check(int pos) {
    368         int block_pos = block_base + pos;
    369         int file_pos = block_pos+buffer_base;
    370         if (at_XxMmLll<ASCII>((unsigned char*)&source[block_pos]) && (source[block_pos+3]=='?' || source[block_pos+3]<= ' ')) {
     292static inline int PIName_check(int pos, int file_pos) {
     293        if (at_XxMmLll<ASCII>((unsigned char*)&source[pos]) && (source[pos+3]=='?' || source[pos+3]<= ' ')) {
    371294              // "<?xml" legal at start of file.
    372295              if (!((file_pos == 2) && at_XmlDecl_start<ASCII>((unsigned char*)&source[0]))) {
     
    378301
    379302static inline int CD_check(int pos) {
    380         int block_pos = block_base + pos;
    381         if (!at_CDATA1<ASCII>((unsigned char*)&source[block_pos])){
     303        if (!at_CDATA1<ASCII>((unsigned char*)&source[pos])){
    382304                  return XMLTestSuiteError::CDATA;
    383305        }
     
    386308
    387309static inline int GenRef_check(int pos) {
    388         int block_pos = block_base + pos;
    389         unsigned char* s = (unsigned char*)&source[block_pos];
     310        unsigned char* s = (unsigned char*)&source[pos];
    390311        if (!(at_Ref_gt<ASCII>(s)||at_Ref_lt<ASCII>(s)||at_Ref_amp<ASCII>(s)||at_Ref_quot<ASCII>(s)||at_Ref_apos<ASCII>(s))){
    391312              return XMLTestSuiteError::UNDEFREF;
     
    395316
    396317static inline int HexRef_check(int pos) {
    397         int block_pos = block_base + pos;
    398         unsigned char* s = (unsigned char*)&source[block_pos];
     318        unsigned char* s = (unsigned char*)&source[pos];
    399319        int ch_val = 0;
    400320        while(at_HexDigit<ASCII>(s)){
     
    415335
    416336static inline int DecRef_check(int pos) {
    417         int block_pos = block_base + pos;
    418         unsigned char* s = (unsigned char*)&source[block_pos];
     337        unsigned char* s = (unsigned char*)&source[pos];
    419338        int ch_val = 0;
    420339        while(at_HexDigit<ASCII>(s)){
     
    435354
    436355static inline int AttRef_check(int pos) {
    437         int block_pos = block_base + pos;
    438         unsigned char* s = (unsigned char*)&source[block_pos];
     356        unsigned char* s = (unsigned char*)&source[pos];
    439357        int ch_val = 0;
    440358        if(s[0]=='#'){
     
    463381}
    464382
     383static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     384
     385        BitBlockForwardIterator end;
     386        int pos, block_pos;
     387
     388        while(start != end) {
     389
     390                block_pos = block_base + *start;
     391                int rv = is_valid(block_pos);
     392
     393                if (rv) {
     394                        int error_line, error_column;
     395                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     396                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     397                        exit(-1);
     398                }
     399                start++;
     400        }
     401}
     402
     403static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     404
     405        BitBlockForwardIterator end;
     406        int pos, block_pos, file_pos;
     407
     408        while(start != end) {
     409
     410                block_pos = block_base + *start;
     411                file_pos = block_pos+buffer_base;
     412
     413
     414                int rv = is_valid(block_pos, file_pos);
     415
     416                if (rv) {
     417                        int error_line, error_column;
     418                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     419                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     420                        exit(-1);
     421                }
     422                start++;
     423        }
     424}
     425
     426// L = 4, pass in bitstream for symbols length [3,4]
     427// L = 8, pass in bitstream for symbols length [5,8]
     428// L = 16, pass in bitstream for symbols length [9,16]
     429// L = 17, pass in bitstream for symbols length longer than 16
     430template <int L>
     431static inline void validate_block_length_grouping(BitBlockForwardIterator & start, int block_base) {
     432
     433    BitBlockForwardIterator end;
     434    int end_pos, start_pos, length;
     435
     436    while(start != end) {
     437        end_pos = /*block_base + */*start;
     438        start_pos = ScanBackwardPos (&elem_starts, end_pos);
     439        length = end_pos - start_pos;
     440        ElemEnd_grouping<L>(start_pos, length);
     441        start++;
     442    }
     443}
     444
     445// pass in bitstream for symbols length 1
     446template <>
     447inline void validate_block_length_grouping<1>(BitBlockForwardIterator & start, int block_base) {
     448
     449    BitBlockForwardIterator end;
     450    int end_pos, start_pos, length;
     451
     452    while(start != end) {
     453        end_pos = /*block_base + */*start;
     454        start_pos = ScanBackwardPos (&elem_starts, end_pos);
     455        length = end_pos - start_pos;
     456        ElemEnd_grouping<1>(end_pos, 1);
     457        start++;
     458    }
     459}
     460
     461// pass in bitstream for symbols length 2
     462template <>
     463inline void validate_block_length_grouping<2>(BitBlockForwardIterator & start, int block_base) {
     464
     465    BitBlockForwardIterator end;
     466    int end_pos, start_pos, length;
     467
     468    while(start != end) {
     469        end_pos = /*block_base + */*start;
     470        start_pos = ScanBackwardPos (&elem_starts, end_pos);
     471        length = end_pos - start_pos;
     472        ElemEnd_grouping<2>(end_pos, 2);
     473        start++;
     474    }
     475}
     476
    465477static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail){
    466478
    467479    tracker.StoreNewlines(lex.LF);
    468     int rv, error_pos_in_block, error_line, error_column;
     480
    469481    elem_starts = tag_Callouts.ElemName_starts;
    470482    hashvalues[1] = hash_data.Hash_value;
     
    472484    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_1) )
    473485    {
    474         StreamScanLengthGrouping<1>((ScanBlock *) &tag_Callouts.ElemName_ends_1, sizeof(BitBlock)/sizeof(ScanBlock));
     486        BitBlockForwardIterator iter_length_grouping_1(&tag_Callouts.ElemName_ends_1);
     487        validate_block_length_grouping<1>(iter_length_grouping_1, block_base);
    475488    }
    476489
    477490    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_2) )
    478491    {
    479         StreamScanLengthGrouping<2>((ScanBlock *) &tag_Callouts.ElemName_ends_2, sizeof(BitBlock)/sizeof(ScanBlock));
     492        BitBlockForwardIterator iter_length_grouping_2(&tag_Callouts.ElemName_ends_2);
     493        validate_block_length_grouping<2>(iter_length_grouping_2, block_base);
    480494    }
    481495
    482496    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_3_to_4) )
    483497    {
    484         StreamScanLengthGrouping<4>((ScanBlock *) &tag_Callouts.ElemName_ends_3_to_4, sizeof(BitBlock)/sizeof(ScanBlock));
     498        BitBlockForwardIterator iter_length_grouping_4(&tag_Callouts.ElemName_ends_3_to_4);
     499        validate_block_length_grouping<4>(iter_length_grouping_4, block_base);
    485500    }
    486501
    487502    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_5_to_8) )
    488503    {
    489         StreamScanLengthGrouping<8>((ScanBlock *) &tag_Callouts.ElemName_ends_5_to_8, sizeof(BitBlock)/sizeof(ScanBlock));
     504        BitBlockForwardIterator iter_length_grouping_8(&tag_Callouts.ElemName_ends_5_to_8);
     505        validate_block_length_grouping<8>(iter_length_grouping_8, block_base);
    490506    }
    491507
    492508    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_9_to_16) )
    493509    {
    494         StreamScanLengthGrouping<16>((ScanBlock *) &tag_Callouts.ElemName_ends_9_to_16, sizeof(BitBlock)/sizeof(ScanBlock));
     510        BitBlockForwardIterator iter_length_grouping_16(&tag_Callouts.ElemName_ends_9_to_16);
     511        validate_block_length_grouping<16>(iter_length_grouping_16, block_base);
    495512    }
    496513
    497514    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_17_and_longer) )
    498515    {
    499         StreamScanLengthGrouping<17>((ScanBlock *) &tag_Callouts.ElemName_ends_17_and_longer, sizeof(BitBlock)/sizeof(ScanBlock));
     516        BitBlockForwardIterator iter_length_grouping_remaining(&tag_Callouts.ElemName_ends_17_and_longer);
     517        validate_block_length_grouping<17>(iter_length_grouping_remaining, block_base);
    500518    }
    501519
     
    507525
    508526    if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
    509       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), NameStrt_check, &error_pos_in_block);
    510       if (rv) {
    511               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    512               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    513               exit(-1);
    514       }
    515 
    516       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_names, sizeof(BitBlock)/sizeof(ScanBlock), Name_check, &error_pos_in_block);
    517       if (rv) {
    518               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    519               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    520               exit(-1);
    521       }
    522     }
    523 
     527        BitBlockForwardIterator iter_NameStrt_check(&check_streams.non_ascii_name_starts);
     528        validate_block(iter_NameStrt_check, block_base, NameStrt_check);
     529        BitBlockForwardIterator iter_Name_check(&check_streams.non_ascii_names);
     530        validate_block(iter_Name_check, block_base, Name_check);
     531    }
    524532    if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
    525       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.PI_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), PIName_check, &error_pos_in_block);
    526       if (rv) {
    527               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    528               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    529               exit(-1);
    530       }
    531     }
    532 
     533        BitBlockForwardIterator iter_PI_name_starts(&(ctCDPI_Callouts.PI_name_starts));
     534        validate_block(iter_PI_name_starts, block_base, buffer_base, PIName_check);
     535    }
    533536    if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
    534       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.CD_starts, sizeof(BitBlock)/sizeof(ScanBlock), CD_check, &error_pos_in_block);
    535       if (rv) {
    536               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    537               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    538               exit(-1);
    539       }
    540     }
    541 
    542     if (bitblock_has_bit(ref_Callouts.GenRef_starts)){
    543       rv = StreamScan((ScanBlock *) &ref_Callouts.GenRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), GenRef_check, &error_pos_in_block);
    544       if (rv) {
    545               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    546               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    547               exit(-1);
    548       }
    549     }
    550 
    551     if (bitblock_has_bit(ref_Callouts.DecRef_starts)){
    552       rv = StreamScan((ScanBlock *) &ref_Callouts.DecRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), DecRef_check, &error_pos_in_block);
    553       if (rv) {
    554               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    555               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    556               exit(-1);
    557       }
    558     }
    559 
    560     if (bitblock_has_bit(ref_Callouts.HexRef_starts)){
    561       rv = StreamScan((ScanBlock *) &ref_Callouts.HexRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), HexRef_check, &error_pos_in_block);
    562       if (rv) {
    563               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    564               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    565               exit(-1);
    566       }
    567     }
    568 
    569     if (bitblock_has_bit(check_streams.att_refs)){
    570       rv = StreamScan((ScanBlock *) &check_streams.att_refs, sizeof(BitBlock)/sizeof(ScanBlock), AttRef_check, &error_pos_in_block);
    571       if (rv) {
    572               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    573               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    574               exit(-1);
    575       }
     537        BitBlockForwardIterator iter_CD_check(&ctCDPI_Callouts.CD_starts);
     538        validate_block(iter_CD_check, block_base, CD_check);
     539    }
     540    if(bitblock_has_bit(ref_Callouts.GenRef_starts)){
     541        BitBlockForwardIterator iter_GenRef_check(&ref_Callouts.GenRef_starts);
     542        validate_block(iter_GenRef_check, block_base, GenRef_check);
     543    }
     544    if(bitblock_has_bit(ref_Callouts.DecRef_starts)){
     545        BitBlockForwardIterator iter_DecRef_check(&ref_Callouts.DecRef_starts);
     546        validate_block(iter_DecRef_check, block_base, DecRef_check);
     547    }
     548    if(bitblock_has_bit(ref_Callouts.HexRef_starts)){
     549        BitBlockForwardIterator iter_HexRef_check(&ref_Callouts.HexRef_starts);
     550        validate_block(iter_HexRef_check, block_base, HexRef_check);
     551    }
     552    if(bitblock_has_bit(check_streams.att_refs)){
     553        BitBlockForwardIterator iter_AttRef_check(&check_streams.att_refs);
     554        validate_block(iter_AttRef_check, block_base, AttRef_check);
    576555    }
    577556
    578557    if(error_tracker.Has_Noted_Error()){
    579             tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
    580             ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
    581             exit(-1);
     558        int error_line, error_column;
     559        tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
     560        ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
     561        exit(-1);
    582562    }
    583563
    584564    matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
    585565    tracker.AdvanceBlock();
     566
    586567}
    587568
  • proto/SymbolTable/symtab_stl_template.cpp

    r1648 r1684  
    4747static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask);
    4848static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, int chars_avail);
     49void do_process(FILE *infile, FILE *outfile);
     50
     51static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int));
     52static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int,int));
     53
    4954static inline void do_symbol_table_lookup();
    5055
    51 void do_process(FILE *infile, FILE *outfile);
    5256
    5357int main(int argc, char * argv[]) {
     
    113117}
    114118
    115 
    116 /* StreamScan & Post Process Definitions */
    117 static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int), int * error_pos_in_block) {
    118 
    119         int blk;
    120         int block_pos = 0;
    121         int pos;
    122 
    123         for (blk = 0; blk < blk_count; blk++) {
    124                 ScanBlock s = stream[blk];
    125                 while(s) {
    126                         pos = (cfzl(s) + block_pos);
    127                         int code = (ProcessPos(pos));
    128                         if (code) {
    129                                 *error_pos_in_block = pos;
    130                                 return code; // error code
    131                         }
    132                         s = s & (s-1);  // clear rightmost bit.
    133                 }
    134                 block_pos += 8 * sizeof(ScanBlock);
    135         }
    136         return 0;
    137 }
    138 
    139119static inline int ElemStrt_check(int pos) {
    140         int block_pos = block_base + pos;
    141         elem_starts_buf.push(buffer_base + block_pos);
    142         return 0;
     120        elem_starts_buf.push(buffer_base + pos);
     121        return 0;
    143122}
    144123
    145124static inline int ElemEnd_check(int pos) {
    146         int block_pos = block_base + pos;
    147         elem_ends_buf.push(buffer_base + block_pos);
    148         return 0;
     125        elem_ends_buf.push(buffer_base + pos);
     126        return 0;
    149127}
    150128
    151129static inline int NameStrt_check(int pos) {
    152         int block_pos = block_base + pos;
    153         if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[block_pos]) == 0){
     130        if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[pos]) == 0){
    154131              return XMLTestSuiteError::NAME_START;
    155132        }
     
    158135
    159136static inline int Name_check(int pos) {
    160         int block_pos = block_base + pos;
    161         if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[block_pos]) == 0){
     137        if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[pos]) == 0){
    162138                  return XMLTestSuiteError::NAME;
    163139        }
     
    165141}
    166142
    167 static inline int PIName_check(int pos) {
    168         int block_pos = block_base + pos;
    169         int file_pos = block_pos+buffer_base;
    170         if (at_XxMmLll<ASCII>((unsigned char*)&source[block_pos]) && (source[block_pos+3]=='?' || source[block_pos+3]<= ' ')) {
     143static inline int PIName_check(int pos, int file_pos) {
     144        if (at_XxMmLll<ASCII>((unsigned char*)&source[pos]) && (source[pos+3]=='?' || source[pos+3]<= ' ')) {
    171145              // "<?xml" legal at start of file.
    172146              if (!((file_pos == 2) && at_XmlDecl_start<ASCII>((unsigned char*)&source[0]))) {
     
    178152
    179153static inline int CD_check(int pos) {
    180         int block_pos = block_base + pos;
    181         if (!at_CDATA1<ASCII>((unsigned char*)&source[block_pos])){
     154        if (!at_CDATA1<ASCII>((unsigned char*)&source[pos])){
    182155                  return XMLTestSuiteError::CDATA;
    183156        }
     
    186159
    187160static inline int GenRef_check(int pos) {
    188         int block_pos = block_base + pos;
    189         unsigned char* s = (unsigned char*)&source[block_pos];
     161        unsigned char* s = (unsigned char*)&source[pos];
    190162        if (!(at_Ref_gt<ASCII>(s)||at_Ref_lt<ASCII>(s)||at_Ref_amp<ASCII>(s)||at_Ref_quot<ASCII>(s)||at_Ref_apos<ASCII>(s))){
    191163              return XMLTestSuiteError::UNDEFREF;
     
    195167
    196168static inline int HexRef_check(int pos) {
    197         int block_pos = block_base + pos;
    198         unsigned char* s = (unsigned char*)&source[block_pos];
     169        unsigned char* s = (unsigned char*)&source[pos];
    199170        int ch_val = 0;
    200171        while(at_HexDigit<ASCII>(s)){
     
    215186
    216187static inline int DecRef_check(int pos) {
    217         int block_pos = block_base + pos;
    218         unsigned char* s = (unsigned char*)&source[block_pos];
     188        unsigned char* s = (unsigned char*)&source[pos];
    219189        int ch_val = 0;
    220190        while(at_HexDigit<ASCII>(s)){
     
    235205
    236206static inline int AttRef_check(int pos) {
    237         int block_pos = block_base + pos;
    238         unsigned char* s = (unsigned char*)&source[block_pos];
     207        unsigned char* s = (unsigned char*)&source[pos];
    239208        int ch_val = 0;
    240209        if(s[0]=='#'){
     
    263232}
    264233
     234static inline void validate_block(BitBlockForwardIterator & start, int block_base, int is_valid(int)) {
     235
     236        BitBlockForwardIterator end;
     237        int pos, block_pos;
     238
     239        while(start != end) {
     240
     241                block_pos = block_base + *start;
     242                int rv = is_valid(block_pos);
     243
     244                if (rv) {
     245                        int error_line, error_column;
     246                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     247                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     248                        exit(-1);
     249                }
     250                start++;
     251        }
     252}
     253
     254static inline void validate_block(BitBlockForwardIterator & start, int block_base, int buffer_base, int is_valid(int,int)) {
     255
     256        BitBlockForwardIterator end;
     257        int pos, block_pos, file_pos;
     258
     259        while(start != end) {
     260
     261                block_pos = block_base + *start;
     262                file_pos = block_pos+buffer_base;
     263
     264
     265                int rv = is_valid(block_pos, file_pos);
     266
     267                if (rv) {
     268                        int error_line, error_column;
     269                        tracker.get_Line_and_Column(block_pos, error_line, error_column);
     270                        ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
     271                        exit(-1);
     272                }
     273                start++;
     274        }
     275}
     276
    265277static inline void do_symbol_table_lookup()
    266278{
     
    291303
    292304    tracker.StoreNewlines(lex.LF);
    293     int rv, error_pos_in_block, error_line, error_column;
    294305
    295306    if ( bitblock_has_bit(tag_Callouts.ElemName_starts))
    296307    {
    297         StreamScan((ScanBlock *) &tag_Callouts.ElemName_starts, sizeof(BitBlock)/sizeof(ScanBlock), ElemStrt_check, &error_pos_in_block);
     308        BitBlockForwardIterator iter_length_grouping_starts(&tag_Callouts.ElemName_starts);
     309        validate_block(iter_length_grouping_starts, block_base, ElemStrt_check);
    298310    }
    299311
    300312    if ( bitblock_has_bit(tag_Callouts.ElemName_ends) )
    301313    {
    302         StreamScan((ScanBlock *) &tag_Callouts.ElemName_ends, sizeof(BitBlock)/sizeof(ScanBlock), ElemEnd_check, &error_pos_in_block);
     314        BitBlockForwardIterator iter_length_grouping_ends(&tag_Callouts.ElemName_ends);
     315        validate_block(iter_length_grouping_ends, block_base, ElemEnd_check);
    303316    }
    304317
     
    306319
    307320    if (bitblock_has_bit(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
    308       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), NameStrt_check, &error_pos_in_block);
    309       if (rv) {
    310               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    311               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    312               exit(-1);
    313       }
    314 
    315       rv = StreamScan((ScanBlock *) &check_streams.non_ascii_names, sizeof(BitBlock)/sizeof(ScanBlock), Name_check, &error_pos_in_block);
    316       if (rv) {
    317               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    318               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    319               exit(-1);
    320       }
    321     }
    322 
     321        BitBlockForwardIterator iter_NameStrt_check(&check_streams.non_ascii_name_starts);
     322        validate_block(iter_NameStrt_check, block_base, NameStrt_check);
     323        BitBlockForwardIterator iter_Name_check(&check_streams.non_ascii_names);
     324        validate_block(iter_Name_check, block_base, Name_check);
     325    }
    323326    if (bitblock_has_bit(ctCDPI_Callouts.PI_name_starts)){
    324       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.PI_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), PIName_check, &error_pos_in_block);
    325       if (rv) {
    326               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    327               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    328               exit(-1);
    329       }
    330     }
    331 
     327        BitBlockForwardIterator iter_PI_name_starts(&(ctCDPI_Callouts.PI_name_starts));
     328        validate_block(iter_PI_name_starts, block_base, buffer_base, PIName_check);
     329    }
    332330    if (bitblock_has_bit(ctCDPI_Callouts.CD_starts)){
    333       rv = StreamScan((ScanBlock *) &ctCDPI_Callouts.CD_starts, sizeof(BitBlock)/sizeof(ScanBlock), CD_check, &error_pos_in_block);
    334       if (rv) {
    335               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    336               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    337               exit(-1);
    338       }
    339     }
    340 
    341     if (bitblock_has_bit(ref_Callouts.GenRef_starts)){
    342       rv = StreamScan((ScanBlock *) &ref_Callouts.GenRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), GenRef_check, &error_pos_in_block);
    343       if (rv) {
    344               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    345               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    346               exit(-1);
    347       }
    348     }
    349 
    350     if (bitblock_has_bit(ref_Callouts.DecRef_starts)){
    351       rv = StreamScan((ScanBlock *) &ref_Callouts.DecRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), DecRef_check, &error_pos_in_block);
    352       if (rv) {
    353               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    354               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    355               exit(-1);
    356       }
    357     }
    358 
    359     if (bitblock_has_bit(ref_Callouts.HexRef_starts)){
    360       rv = StreamScan((ScanBlock *) &ref_Callouts.HexRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), HexRef_check, &error_pos_in_block);
    361       if (rv) {
    362               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    363               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    364               exit(-1);
    365       }
    366     }
    367 
    368     if (bitblock_has_bit(check_streams.att_refs)){
    369       rv = StreamScan((ScanBlock *) &check_streams.att_refs, sizeof(BitBlock)/sizeof(ScanBlock), AttRef_check, &error_pos_in_block);
    370       if (rv) {
    371               tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
    372               ReportError(XMLTestSuiteError::get_msg(rv), error_line, error_column);
    373               exit(-1);
    374       }
     331        BitBlockForwardIterator iter_CD_check(&ctCDPI_Callouts.CD_starts);
     332        validate_block(iter_CD_check, block_base, CD_check);
     333    }
     334    if(bitblock_has_bit(ref_Callouts.GenRef_starts)){
     335        BitBlockForwardIterator iter_GenRef_check(&ref_Callouts.GenRef_starts);
     336        validate_block(iter_GenRef_check, block_base, GenRef_check);
     337    }
     338    if(bitblock_has_bit(ref_Callouts.DecRef_starts)){
     339        BitBlockForwardIterator iter_DecRef_check(&ref_Callouts.DecRef_starts);
     340        validate_block(iter_DecRef_check, block_base, DecRef_check);
     341    }
     342    if(bitblock_has_bit(ref_Callouts.HexRef_starts)){
     343        BitBlockForwardIterator iter_HexRef_check(&ref_Callouts.HexRef_starts);
     344        validate_block(iter_HexRef_check, block_base, HexRef_check);
     345    }
     346    if(bitblock_has_bit(check_streams.att_refs)){
     347        BitBlockForwardIterator iter_AttRef_check(&check_streams.att_refs);
     348        validate_block(iter_AttRef_check, block_base, AttRef_check);
    375349    }
    376350
    377351    if(error_tracker.Has_Noted_Error()){
    378             tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
    379             ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
    380             exit(-1);
     352        int error_line, error_column;
     353        tracker.get_Line_and_Column(error_tracker.Noted_Pos_In_Block(), error_line, error_column);
     354        ReportError(error_tracker.Noted_Error_Msg(), error_line, error_column);
     355        exit(-1);
    381356    }
    382357
Note: See TracChangeset for help on using the changeset viewer.