Changeset 3219 for proto/xmlschema


Ignore:
Timestamp:
May 29, 2013, 12:37:08 PM (6 years ago)
Author:
shiyangy
Message:
 
Location:
proto/xmlschema/src
Files:
18 deleted
3 edited

Legend:

Unmodified
Added
Removed
  • proto/xmlschema/src/Makefile

    r2234 r3219  
    77CFLAGS= $(SSE) -O3
    88SSE=-msse2 #-msse4.1 -DUSE_PTEST #-mssse3 -msse4.1 -msse4.2
     9INCLUDES=-I../util -I../lib -I../lib/cc-lib
    910
    1011PAPI_DIR=/usr/local/papi
     
    1314ifeq ($(ARCH),32)
    1415    AFLAGS=-march=pentium4
    15     CODE_CLOCKER=-DCODE_CLOCKER -I$(PAPI_DIR)/include -L$(PAPI_DIR)/lib -lpapi
     16    PAPI=-I$(PAPI_DIR)/include -L$(PAPI_DIR)/lib -lpapi -DPAPI -Ilib/cclib
    1617else
    1718    AFLAGS=-march=nocona -m64
    18     CODE_CLOCKER=-DCODE_CLOCKER -I$(PAPI_DIR)/include -L$(PAPI_DIR)/lib -lpapi
     19    PAPI=-I$(PAPI_DIR)/include -L$(PAPI_DIR)/lib -lpapi -DPAPI -Ilib/cclib
    1920endif
    2021
     22avx:    $(SRCFILE)
     23        $(CC) -o $(OUTFILE) $(SRCFILE) $(INCLUDES) -march=nocona -m64 -mavx
     24
    2125all:    $(SRCFILE)
    22         $(CC) -o $(OUTFILE) $(SRCFILE) $(AFLAGS)
     26        $(CC) -o $(OUTFILE) $(SRCFILE) $(INCLUDES) $(AFLAGS)
    2327
    24 avx:    $(SRCFILE)
    25         $(CC) -o $(OUTFILE) $(SRCFILE) -march=nocona -m64 -mavx
     28xmlwfj: $(SRCFILEJ)
     29        $(CC) -o $(OUTFILEJ) $(SRCFILEJ) $(INCLUDES) $(AFLAGS)
    2630
    2731threads:        $(SRCFILE)
    28         g++ -O3 -msse2 -o $(OUTFILE) $(SRCFILE) $(AFLAGS) -lpthread -DBUFFER_PROFILING
     32        g++ -O3 -msse2 -o $(OUTFILE) $(SRCFILE) $(INCLUDES) $(AFLAGS) -lpthread -DBUFFER_PROFILING
     33
     34papi_profiling: $(SRCFILE)
     35        $(CC) -o $(OUTFILE) $(SRCFILE) $(INCLUDES) $(AFLAGS) $(PAPI)
    2936
    3037buffer_profiling: $(SRCFILE)
    31         $(CC) -o $(OUTFILE) $(SRCFILE) $(AFLAGS) -DBUFFER_PROFILING
     38        $(CC) -o $(OUTFILE) $(SRCFILE) $(INCLUDES) $(AFLAGS) -DBUFFER_PROFILING
     39       
     40buffer_profilingj: $(SRCFILE)
     41        $(CC) -o $(OUTFILEJ) $(SRCFILEJ) $(INCLUDES) $(AFLAGS) -DBUFFER_PROFILING
    3242
    33 buffer_profiling_perf: $(SRCFILE)
    34         $(CC) -o $(OUTFILE) $(SRCFILE) $(AFLAGS) -DBUFFER_PROFILING  -DPERF_SCRIPT
    35 
    36 asm:  $(SRCFILE)
    37         $(CC) -o $(OUTFILE).s $(SRCFILE) $(AFLAGS) -S -fverbose-asm
    38        
    3943as: $(OUTFILE).s
    4044        as $(OUTFILE).s -o $(OUTFILE).o
     
    5054
    5155clean: 
    52         rm -f $(OUTFILE) $(OUTFILE).s $(OUTFILE).o
     56        rm -f $(OUTFILE) $(OUTFILE).s $(OUTFILE).o id div2 logbase2 div2_logbase2 bit_byte $(OUTFILEJ)
     57
     58asm:  $(SRCFILE)
     59        $(CC) -o $(OUTFILE).s $(SRCFILE) $(INCLUDES) $(AFLAGS) -S -fverbose-asm
     60
     61# Symbol Table perf targets
     62
     63all_profiling_perf: buffer_profiling_perf id_profiling_perf div2_profiling_perf logbase2_profiling_perf div2_logbase2_profiling_perf bit_byte_profiling_perf
     64
     65buffer_profiling_perf: $(SRCFILE)
     66        $(CC) -o $(OUTFILE) $(SRCFILE) $(INCLUDES) $(AFLAGS) -DBUFFER_PROFILING  -DPERF_SCRIPT
     67
     68id_profiling_perf: $(ID_SRCFILE)
     69        $(CC) -o id $(ID_SRCFILE) $(INCLUDES) $(AFLAGS) -DID_STRATEGY -DBUFFER_PROFILING -DPERF_SCRIPT
     70
     71div2_profiling_perf: $(DIV2_SRCFILE)
     72        $(CC) -o div2 $(DIV2_SRCFILE) $(INCLUDES) $(AFLAGS) -DDIV2_STRATEGY -DBUFFER_PROFILING -DPERF_SCRIPT
     73
     74logbase2_profiling_perf: $(LOGBASE2_SRCFILE)
     75        $(CC) -o logbase2 $(LOGBASE2_SRCFILE) $(INCLUDES) $(AFLAGS) -DLOGBASE2_STRATEGY -DBUFFER_PROFILING -DPERF_SCRIPT
     76
     77div2_logbase2_profiling_perf: $(DIV2_LOGBASE2_SRCFILE)
     78        $(CC) -o div2_logbase2 $(DIV2_LOGBASE2_SRCFILE) $(INCLUDES) $(AFLAGS) -DDIV2_LOGBASE2_STRATEGY -DBUFFER_PROFILING -DPERF_SCRIPT
     79
     80bit_byte_profiling_perf: $(BIT_BYTE_SRCFILE)
     81        $(CC) -o bit_byte $(BIT_BYTE_SRCFILE) $(INCLUDES) $(AFLAGS) -DBIT_BYTE_STRATEGY -DBUFFER_PROFILING  -DPERF_SCRIPT
     82
     83# Symbol Table debug targets
     84
     85all_debug: id_debug div2_debug logbase2_debug div2_logbase2_debug bit_byte_debug
     86
     87id_debug: $(ID_SRCFILE)
     88        $(CC) -o id $(ID_SRCFILE) $(INCLUDES) $(AFLAGS) -DID_STRATEGY  -DDUMP_SYMBOLS
     89
     90div2_debug: $(DIV2_SRCFILE)
     91        $(CC) -o div2 $(DIV2_SRCFILE) $(INCLUDES) $(AFLAGS) -DDIV2_STRATEGY -DDUMP_SYMBOLS
     92
     93logbase2_debug: $(LOGBASE2_SRCFILE)
     94        $(CC) -o logbase2 $(LOGBASE2_SRCFILE) $(INCLUDES) $(AFLAGS) -DLOGBASE2_STRATEGY -DDUMP_SYMBOLS
     95
     96div2_logbase2_debug: $(DIV2_LOGBASE2_SRCFILE)
     97        $(CC) -o div2_logbase2 $(DIV2_LOGBASE2_SRCFILE) $(INCLUDES) $(AFLAGS) -DDIV2_LOGBASE2_STRATEGY -DDUMP_SYMBOLS
     98
     99bit_byte_debug: $(BIT_BYTE_SRCFILE)
     100        $(CC) -o bit_byte $(BIT_BYTE_SRCFILE) $(INCLUDES) $(AFLAGS) -DBIT_BYTE_STRATEGY -DDUMP_SYMBOLS
     101
  • proto/xmlschema/src/xmlwf.cpp

    r2351 r3219  
     1
     2#include <simd-lib/bitblock.hpp>
     3#include <simd-lib/carryQ.hpp>
     4#include <simd-lib/pabloSupport.hpp>
     5
     6#define LocalCarryDeclare(name, count)\
     7CarryArray<count, 0> name;\
     8
     9#define assert_0_error(errkind, errstrm)
     10
     11BitBlock EOF_mask = simd<1>::constant<1>();
     12
     13//#define USE_FUNCTION_TEMPLATES
     14//#define TEMPLATED_SIMD_LIB
     15
     16#define DEBUG 1
     17//#define BLOCK_SIZE (sizeof(SIMD_type) * 8)
     18//#define SEGMENT_BLOCKS 12
     19//#define BUFFER_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
     20//#define OVERLAP_BUFSIZE 2*(sizeof(SIMD_type))
     21
     22#define VECTOR_WIDTH 8
     23#define VECTOR_TYPE (char)
     24
    125#include <stdio.h>
    226#include <stdlib.h>
     
    428#include <sys/types.h>
    529#include <sys/stat.h>
    6 
    7 #define LocalCarryDeclare(name, count)\
    8 CarryArray<count, 0> name;\
    9 
    10 #include "../lib/bitblock.hpp"
    11 #include "../lib/carryQ.hpp"
    12 #include "../lib/bitblock_iterator.hpp"
    13 #include "../lib/s2p.hpp"
    14 #include "../lib/perflib/perfsec.h"
    15 
    16 #define VECTOR_WIDTH 8
    17 #define VECTOR_TYPE (char)
    18 #define DEBUG 0
    19 
    20 #define ScanBlock long long int
    21 
    22 
     30#include <limits.h>
     31
     32#include <simd-lib/s2p.hpp>
     33#include <simd-lib/buffer.hpp>
     34#include <simd-lib/bitblock_iterator.hpp>
     35#include <simd-lib/perflib/perfsec.h>
     36
     37//#include "../lib/lib_simd.h"
     38#include "../symtab/pbgs_identity_symbol_table.h"
     39
     40#include <stack>
     41#include <map>
     42#include <vector>
     43
     44#include <queue>
     45#include <string>
     46
     47typedef long ScanBlock;
     48typedef BytePack BitBlock;
     49
     50using namespace std;
     51
     52//#include "../lib/carryQ.h"
    2353#include <xmldecl.h>
     54//#include "xml_error.c"
     55//#include "xmldecl.c"
    2456#include <namechars.h>
    25 #include <TagMatcher.hpp>
    2657#include <LineColTracker.hpp>
    27 #include <ErrorUtil.h>
    28 #include <ErrorTracker.h>
    2958#include <XMLTestSuiteError.h>
    30 
    31 #include "../symtab/pbgs_identity_symbol_table.h"
    32 #include "../lib/buffer.hpp"
    33 
    34 #define OVERLAP_BUFSIZE PADDING_SIZE //sizeof(BitBlock)
    3559
    3660// Define the mappings for pablo.assert_0(strm, errkind) statements which
    3761// compile to the the form assert_0_error(errkind, strm)
     62#include <ErrorTracker.h>
     63
     64ErrorTracker error_tracker;
    3865#define assert_0_error(errkind, errstrm) error_tracker.NoteError(errkind, errstrm);
    3966
     67#ifndef STL_ALIGNED_VECTOR
     68#include <TagMatcher.hpp>
     69#endif
     70
     71#ifdef STL_ALIGNED_VECTOR // experimental, comment out TagMatcher code to use STL aligned memory vector
     72#warning "Compiling with experimental STL_ALIGNED_VECTOR."
     73#include <vector>
     74#include <simd-lib/stl_aligned_allocator.hpp>
     75#endif
     76
    4077#ifdef BUFFER_PROFILING
    41     BOM_Table * parser_timer;
     78        BOM_Table * parser_timer;
     79
    4280#elif CODE_CLOCKER
    43     //#define NUM_EVENTS 1
    44     //int Events[NUM_EVENTS] = {PAPI_TOT_CYC};
    45     //int Events[NUM_EVENTS] = {PAPI_L2_DCM};
    46     #define NUM_EVENTS 2
    47     int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
    48     int cal_size = 20;
    49     CC * parser_timer = new CC(Events,NUM_EVENTS,cal_size);
     81        #define NUM_EVENTS 1
     82        int Events[NUM_EVENTS] = {PAPI_TOT_CYC};
     83        //int Events[NUM_EVENTS] = {PAPI_L2_DCM};
     84        //int Events[NUM_EVENTS] = {PAPI_TOT_CYC, PAPI_BR_MSP};
     85        int cal_size = 20;
     86        CC * parser_timer = new CC(Events,NUM_EVENTS,cal_size);
    5087#else
    51     void * parser_timer;
     88        void * parser_timer;
    5289#endif
    53 
    54 
    55 int block_base=0;
    56 int buffer_base=0;
    57 char * source;
    58 int previous_block_last_elem_start;
    59 
    60 ErrorTracker error_tracker;
    61 BitBlock EOF_mask = simd<1>::constant<1>();
    62 
    63 BitBlock elem_starts;
    64 BytePack hashvalues[2];
    65 
    66 vector <int> gids;
    67 PBGSIdentitySymbolTable pbgs_symbol_table;
    68 
    69 stack <vector<char>* > elem_vec_stack;
    70 map<char, vector<char>* > elem_vec_map;
    71 map<char, vector<int>* > elem_pos_map;
    72 
    73 //file of all the gids
    74 FILE * ofile;
    75 
    76 //buffer for double_list data validation
    77 int val_buf_curser = 0;
    78 int content_start_pos = 0;
    79 int content_start_flag = 0; //-1 if the content_start is not in the current block, need to search for the comming blocks; 1 if content_start is in the former block; 0 content_start and content_end are both in current block
    80 int content_end_flag = 0; // content_end flag
    81 int val_buf_avail = sizeof(BitBlock)*8; // char available of val_buf
    82 //BytePack val_buf[(BUFFER_SIZE+BLOCK_SIZE+OVERLAP_BUFSIZE*2)/sizeof(BitBlock)];
    83 char val_buf[sizeof(BitBlock)*8];
    84 
    85 int counter = 0;
    86 
    87 static inline int ScanBackwardPos(BitBlock * block, int pos,int previous_block_last_elem_start);
    88 static inline int compute_hash_value (int lgth, int start);
    89 static inline char ElemEnd_grouping(int pos);
    90 //static inline int StreamScanLengthGrouping(ScanBlock * stream, ScanBlock * stream2, int blk_count);
    91 static inline int StreamScanLengthGrouping(Tag_Callouts & tag_Callouts, int blk_count);
    92 static inline void print_vecs();
    93 static inline void print_elem_vec(vector<char>* vec, int size);
    94 static inline void validation_do_block(Lex & lex, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail);
    95 static inline void print_basis_bits(Basis_bits & basis_bits);
    96 static inline void validate_city_gml();
    97 static inline void print_streams(char * s1,BitBlock & block, char * name);
    98 static inline void print_buffer(char * block, char * name);
    99 static inline void val_double_list(Tag_Callouts & tag_Callouts,int end_pos,int block_pos,int blk);
    100 static inline void val_dl_block();
    101 
    102 struct Citygml citygml;
    103 Validation_Vec_2 validation_Vec_2;
    10490
    10591//////////////////////////////////////////////////////////////////////////////////////////
     
    116102#define BUFFER_SIZE (COPYBACK_SIZE + SEGMENT_SIZE + LOOKAHEAD_SIZE + PADDING_SIZE)
    117103
    118 //////////////////////////////////////////////////////////////////////////////////////////
    119 // @ global depends on 'error_tracker' and 'EOF_mask' definitions.
    120 //////////////////////////////////////////////////////////////////////////////////////////
     104
     105
     106//int block_base=0;
     107//int buffer_base=0;
     108int buffer_last;
     109FILE *gid_writein;
     110#define ELEMENT_BUFFER_SIZE 30
     111char element_name_buffer[ELEMENT_BUFFER_SIZE];
     112int cross_buffer_flag = 0;
     113//char * source;
     114LineColTracker tracker;
     115
     116static inline int cfzl(long x){
     117        unsigned long ux = x;
     118        if (x<0){
     119                ux = ULONG_MAX + x + 1;
     120        }
     121        if (ux==0)
     122                return 8*sizeof(long);
     123        int n;
     124        n = 0;
     125        while (1){
     126                if ((ux%2) == 1){
     127                        if(n==64){
     128                                cout << ux <<endl;
     129                        }
     130                        return n;               
     131                }
     132                n ++;
     133                ux = ux >> 1;
     134                //cout << "x = " << x << endl;
     135        }
     136}
     137
     138static inline void print_gid_vec();
     139/*
     140static inline int cfzl(long x){
     141 if (x ==0)
     142  return 32;
     143 int n=0;
     144 if ((x & 0xFFFF0000) == 0) { n += 16; x =x << 16;} //1111 1111 1111 1111 0000 0000 0000 0000 // 16 bits from left are zero! so we omit 16left bits
     145 if ((x & 0xFF000000) == 0){ n = n +  8; x = x <<  8;} // 8 left bits are 0
     146 if ((x & 0xF0000000) ==0){ n = n +  4; x = x <<  4;} // 4 left bits are 0
     147 if ((x & 0xC0000000) == 0){ n =n +  2, x = x <<  2;}  // 110000....0 2 left bits are zero
     148 if ((x & 0x80000000) == 0){n = n +  1, x = x <<  1;} // first left bit is zero
     149 return n;
     150
     151}
     152*/
     153static inline void ReportError(const char * error_msg, int error_pos_in_block) {
     154  int error_line, error_column;
     155  tracker.get_Line_and_Column(error_pos_in_block, error_line, error_column);
     156  fprintf(stderr, "%s at line %i, column %i\n", error_msg, error_line, error_column);
     157}
     158/*
     159class ErrorTracker {
     160public:
     161    ErrorTracker() { noted_pos_in_block = -1;}
     162
     163    inline void NoteError(const char * error_msg, BitBlock err_strm) {
     164      int pos_in_block = count_forward_zeroes(err_strm);
     165      if ((noted_pos_in_block == -1) || (noted_pos_in_block > pos_in_block)) {
     166        noted_pos_in_block = pos_in_block;
     167        noted_error = error_msg;
     168      }
     169    }
     170
     171    inline void If_Error_Report_First() {
     172      if (noted_pos_in_block > -1) {
     173              int error_line, error_column;
     174              ReportError(noted_error, noted_pos_in_block);
     175              exit(-1);
     176      }
     177    }
     178
     179private:
     180  const char * noted_error;
     181  int noted_pos_in_block;
     182};
     183*/
     184
     185
     186//BitBlock EOF_mask = simd_const_1(1);
     187
     188//ErrorTracker error_tracker;
     189BitBlock elem_starts;
     190int previous_block_last_elem_start;
     191BytePack hashvalues[2];
     192
     193vector <int> gids;
     194PBGSIdentitySymbolTable pbgs_symbol_table;
     195
     196vector <char> gid_vec;
     197
     198stack <vector<char>* > elem_vec_stack;
     199map<char, vector<char>* > elem_vec_map;
     200
     201static inline int ScanBackwardPos(BitBlock * block, int pos)
     202{
     203        cout << "pos " << pos << endl;
     204    BitBlock s = block[0];
     205    if (bitblock::any(block[0])){
     206        print_register("zero",block[0]);
     207        print_register("t",simd_not(simd<128>::sll(simd<1>::constant<1>(), convert(pos))) );
     208        }
     209    BitBlock temp = simd_and( s, simd_not(simd<128>::sll(simd<1>::constant<1>(), convert(pos))) );
     210//      print_register("zero",simd_not(simd<128>::sll(simd<1>::constant<1>());
     211                print_register("temp",temp);
     212    if (bitblock::any(temp))
     213    {
     214        // sizeof (BitBlock)*8 - cbzl( s & ~(~0 << pos)) - 1;
     215//      cout << "block_size" << BLOCK_SIZE << endl;
     216        return BLOCK_SIZE - count_reverse_zeroes(temp) - 1;
     217    }
     218    else
     219    {
     220        //handle boundary case
     221        return previous_block_last_elem_start - 1;
     222    }
     223}
     224
     225static inline int StreamScan(ScanBlock * stream, int blk_count, int ProcessPos(int)) {
     226        int blk;
     227        int block_pos = 0;
     228        for (blk = 0; blk < blk_count; blk++) {
     229                ScanBlock s = stream[blk];
     230                while(s) {
     231                        int code = (ProcessPos(cfzl(s) + block_pos));
     232                        if (code) return code;
     233                        s = s & (s-1);  // clear rightmost bit.
     234                }
     235                block_pos += 8 * sizeof(ScanBlock);
     236        }
     237        return 0;
     238}
     239
     240static inline int compute_hash_value (int lgth, int start)
     241{
     242    unsigned int offset_bit = start + 128;
     243    uint64_t stream = *((uint64_t*)(((uint32_t*)hashvalues)+(offset_bit>>5)));
     244    return stream >> (offset_bit & 0x1F) & ~(~0 << lgth);
     245}
     246/*
     247template <int L>
     248static inline int ElemEnd_grouping(int pos) {
     249    int end = block_base + pos;
     250    int start = end - L;
     251    int hashvalue = compute_hash_value(L, start - block_base);
     252    int gid = pbgs_symbol_table.Lookup_or_Insert_Name<L>(source + start, hashvalue);
     253    gids.push_back(gid);
     254#if DEBUG
     255    char* symbol = new char[L+1];
     256    strncpy ( symbol, source + start, L );
     257    symbol[L] ='\0';
     258    printf ("%s | start: %i[%i] | end: %i[%i] | gid: %i | hashvalue: %i | symbol: %s\n", __FUNCTION__, start, start-buffer_base, end, end-buffer_base, gid, hashvalue, symbol );
     259    delete symbol; symbol = 0;
     260#endif
     261    return gid;
     262}
     263
     264template<>
     265inline int ElemEnd_grouping<17>(int pos) {
     266    int end = block_base + pos;
     267    int start = ScanBackwardPos (&elem_starts, pos) + block_base;
     268    int lgth = end - start;
     269    int hashvalue = compute_hash_value(lgth, start - block_base);
     270    int gid = 0;
     271
     272//    if (lgth < 32)
     273//    {
     274//      gid = pbgs_symbol_table.Lookup_or_Insert_Name_32(source + start, hashvalue, lgth);
     275//    }
     276//    else
     277    {
     278        gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
     279    }
     280    gids.push_back(gid);
     281#if DEBUG
     282    char* symbol = new char[lgth+1];
     283    strncpy ( symbol, source + start, lgth );
     284    symbol[lgth] ='\0';
     285    printf ("%s | start: %i[%i] | end: %i[%i] | lgth: %i | hashvalue: %i | gid: %i | symbol: %s\n", __FUNCTION__, start, start - block_base, end, end - block_base, lgth, hashvalue, gid, symbol);
     286#endif
     287    return gid;
     288}
     289*/
     290
     291inline char ElemEnd_grouping(char *source,int block_base,int pos) {
     292    int end = block_base + pos;
     293    int start = ScanBackwardPos(&elem_starts, pos) + block_base;
     294    ///////////////////////
     295    //start could be negitive
     296    //the element name falls in two buffers
     297    ///////////////////////
     298    char *name = source + start;
     299    char name_buffer[50];
     300   
     301    if (start<0){
     302        memmove (&name_buffer[0],&element_name_buffer[ELEMENT_BUFFER_SIZE+start],-start);       
     303        memmove (&element_name_buffer[ELEMENT_BUFFER_SIZE+start],&source[0],50+start);
     304        name = name_buffer;
     305    }
     306//    cout << end << "  " << start << endl;
     307    int lgth = end - start;
     308    int hashvalue = compute_hash_value(lgth, start - block_base);
     309    char gid = 0;
     310
     311
     312//    if (lgth < 32)
     313//    {
     314//      gid = pbgs_symbol_table.Lookup_or_Insert_Name_32(source + start, hashvalue, lgth);
     315//    }
     316//    else
     317
     318        switch (lgth)
     319    {
     320    case 1:
     321        gid = pbgs_symbol_table.Lookup_or_Insert_Name<1>(source + start, hashvalue);
     322        break;
     323    case 2:
     324        gid = pbgs_symbol_table.Lookup_or_Insert_Name<2>(source + start, hashvalue);
     325        break;
     326    case 3:
     327        gid = pbgs_symbol_table.Lookup_or_Insert_Name<3>(source + start, hashvalue);
     328        break;
     329    case 4:
     330        gid = pbgs_symbol_table.Lookup_or_Insert_Name<4>(source + start, hashvalue);
     331        break;
     332    case 5:
     333        gid = pbgs_symbol_table.Lookup_or_Insert_Name<5>(source + start, hashvalue);
     334        break;
     335    case 6:
     336        gid = pbgs_symbol_table.Lookup_or_Insert_Name<6>(source + start, hashvalue);
     337        break;
     338    case 7:
     339        gid = pbgs_symbol_table.Lookup_or_Insert_Name<7>(source + start, hashvalue);
     340        break;
     341    case 8:
     342        gid = pbgs_symbol_table.Lookup_or_Insert_Name<8>(source + start, hashvalue);
     343        break;
     344    case 9:
     345        gid = pbgs_symbol_table.Lookup_or_Insert_Name<9>(source + start, hashvalue);
     346        break;
     347    case 10:
     348        gid = pbgs_symbol_table.Lookup_or_Insert_Name<10>(source + start, hashvalue);
     349        break;
     350    case 11:
     351        gid = pbgs_symbol_table.Lookup_or_Insert_Name<11>(source + start, hashvalue);
     352        break;
     353    case 12:
     354        gid = pbgs_symbol_table.Lookup_or_Insert_Name<12>(source + start, hashvalue);
     355        break;
     356    case 13:
     357        gid = pbgs_symbol_table.Lookup_or_Insert_Name<13>(source + start, hashvalue);
     358        break;
     359    case 14:
     360        gid = pbgs_symbol_table.Lookup_or_Insert_Name<14>(source + start, hashvalue);
     361        break;
     362    case 15:
     363        gid = pbgs_symbol_table.Lookup_or_Insert_Name<15>(source + start, hashvalue);
     364        break;
     365    case 16:
     366        gid = pbgs_symbol_table.Lookup_or_Insert_Name<16>(source + start, hashvalue);
     367        break;
     368    default:
     369        gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
     370        break;
     371    }
     372/*
     373    {
     374        gid = (char)pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
     375    }
     376*/
     377    gids.push_back(gid);
     378#if DEBUG
     379    char* symbol = new char[lgth+1];
     380    strncpy ( symbol, source + start, lgth );
     381    symbol[lgth] ='\0';
     382    printf ("%s | start: %i[%i] | end: %i[%i] | lgth: %i | hashvalue: %i | gid: %i | symbol: %s\n", __FUNCTION__, start, start - block_base, end, end - block_base, lgth, hashvalue, gid, symbol);
     383    fprintf (gid_writein,"%s | start: %i[%i] | end: %i[%i] | lgth: %i | hashvalue: %i | gid: %i | symbol: %s\n", __FUNCTION__, start, start - block_base, end, end - block_base, lgth, hashvalue, gid, symbol);
     384#endif
     385    return gid;
     386}
     387
     388//template <int L>
     389static inline int StreamScanLengthGrouping(char *source,int block_base,ScanBlock * stream, ScanBlock * stream2, int blk_count) {
     390    int blk;
     391    int block_pos = 0;
     392//    cout << "cfzl():" << cfzl() << endl;
     393//      long test = 6;
     394//      test = (test & (test - 1));
     395//      cout << test << endl;
     396    for (blk = 0; blk < blk_count; blk++) {
     397        cout << "blk:" << blk << endl;
     398   
     399                ScanBlock s = stream[blk];
     400                ScanBlock s2 = stream2[blk];
     401//              cout << s << "  " << s2 << endl;
     402//              print_register("s:",((BitBlock*)stream)[0]);
     403                while(s | s2) {
     404                        cout << "cfzl(s):" << cfzl(s)<<endl;
     405                    int end_pos = cfzl(s) + block_pos;
     406//                  cout << "s2 " << s2 << endl;
     407                    int end_tag = cfzl(s2) + block_pos;
     408//              cout << "cfzl(s):" << cfzl(s)<<endl;
     409//              cout << "cfzl(s2):" << cfzl(s2)<<endl;
     410//                      cout << (end_tag < end_pos && s2 != 0)<< "||" <<(s == 0 && s2 != 0)  << endl;
     411                        if ((end_tag < end_pos && s2 != 0) | (s == 0 && s2 != 0)){
     412                        cout << "here" << endl;
     413                        cout << "closing tag" << endl;
     414//                      cout << "endtag:" << end_tag << " " << end_pos<<endl;
     415//                      cout << "1"<< endl;
     416//                      cout << elem_vec_stack.size()<<endl;
     417//                      cout << "2" << endl;
     418//                      cout << elem_vec_stack.top()<<endl;
     419                                if(!(gid_vec.empty())){
     420//                                      if(gid_vec.back() != 0){
     421                                                gid_vec.push_back(0);
     422//                                      }
     423                                }
     424                                cout << "before" << s2 << endl;
     425                                s2 = s2 & (s2 - 1);
     426                                cout << "after" << s2 << endl;
     427//                      cout << "s2 = " << s2 << endl;
     428                }else{
     429                        cout << " end_pos == " << end_pos<<endl;
     430                        cout << " end_tag_pos == " << end_tag << endl;
     431                        char gid = ElemEnd_grouping(source,block_base,end_pos);
     432                        if (s!=0){
     433                                s = s & (s-1);  // clear rightmost bit.
     434                        }
     435                        //else{
     436                        //      s2 = s2 & (s2 - 1);
     437                        //}
     438                                if (gid != 0){
     439                                        gid_vec.push_back(gid);                 
     440                                }
     441                                cout << "here1" << endl;
     442                }
     443                }
     444                block_pos += 8 * sizeof(ScanBlock);
     445                print_gid_vec();
     446    }
     447    return 0;
     448}
     449
     450void print_gid_vec(){
     451        for (int i = 0; i < gid_vec.size(); i++){
     452                cout << (int)gid_vec[i] << " ";
     453        }
     454        cout << endl;
     455/*     
     456        int i = 1;
     457        int lst[256] = {0};
     458        cout << elem_vec_map.size() << " vectors" << endl;
     459        for (int i = 0;i < gids.size(); i++){
     460                if (lst[gids[i]] == 0){
     461                        cout << "gid: " << gids[i] << endl;
     462                        lst[gids[i]] = 1;
     463                        vector<char> *tmp = elem_vec_map[gids[i]];
     464                        cout << "vector size:"<< (*tmp).size()<<endl;
     465                        if ((*tmp).size()!=0){
     466                                cout << "elements: ";
     467                                for (int j = 0; j < (*tmp).size();j++){
     468                                        cout << (int)(*tmp)[j] << " ";
     469                                }
     470                                cout << endl;
     471                        }
     472                        cout << endl;
     473                }
     474        }
     475*/
     476}
     477/*
     478static inline int NameStrt_check(int pos) {
     479        int block_pos = block_base + pos;
     480        if(XML_10_UTF8_NameStrt_bytes((unsigned char*)&source[block_pos]) == 0){
     481              ReportError("name start error", pos);
     482              exit(-1);
     483        }
     484        return 0;
     485}
     486
     487static inline int Name_check(int pos) {
     488        int block_pos = block_base + pos;
     489        if(XML_10_UTF8_NameChar_bytes((unsigned char*)&source[block_pos]) == 0){
     490              ReportError("name error", pos);
     491              exit(-1);
     492        }
     493        return 0;
     494}
     495
     496static inline int PIName_check(int pos) {
     497        int block_pos = block_base + pos;
     498        int file_pos = block_pos+buffer_base;
     499        printf ("%s:%i\n",__FUNCTION__,pos);
     500        if (at_XxMmLll<ASCII>((unsigned char*)&source[block_pos]) && (source[block_pos+3]=='?' || source[block_pos+3]<= ' ')) {
     501              // "<?xml" legal at start of file.
     502              if ((file_pos == 2) && at_XmlDecl_start<ASCII>((unsigned char*)&source[0])) return 0;
     503              ReportError("[Xx][Mm][Ll] illegal as PI name", pos);
     504              exit(-1);
     505        }
     506        return 0;
     507}
     508
     509static inline int CD_check(int pos) {
     510        int block_pos = block_base + pos;
     511        if (!at_CDATA1<ASCII>((unsigned char*)&source[block_pos])){
     512              ReportError("CDATA error", pos);
     513              exit(-1);
     514        }
     515        return 0;
     516}
     517
     518static inline int GenRef_check(int pos) {
     519        int block_pos = block_base + pos;
     520        unsigned char* s = (unsigned char*)&source[block_pos];
     521        if (!(at_Ref_gt<ASCII>(s)||at_Ref_lt<ASCII>(s)||at_Ref_amp<ASCII>(s)||at_Ref_quot<ASCII>(s)||at_Ref_apos<ASCII>(s))){
     522                        cout << pos << endl;
     523              ReportError("Undefined reference", pos);
     524              exit(-1);
     525        }
     526        return 0;
     527}
     528
     529static inline int HexRef_check(int pos) {
     530        int block_pos = block_base + pos;
     531        unsigned char* s = (unsigned char*)&source[block_pos];
     532        int ch_val = 0;
     533        while(at_HexDigit<ASCII>(s)){
     534          ch_val = HexVal<ASCII>(s[0]) + (ch_val<<4);
     535          if (ch_val> 0x10FFFF ){
     536            ReportError("Illegal character reference", pos);
     537            exit(-1);
     538          }
     539          s++;
     540        }
     541        if ((ch_val == 0x0) || ((ch_val | 0x7FF) == 0xDFFF)|| ((ch_val | 0x1) == 0xFFFF)){
     542          ReportError("Illegal character reference", pos);
     543          exit(-1);
     544        }
     545        else if (((ch_val < 0x20) && (ch_val != 0x9) && (ch_val != 0xD) && (ch_val != 0xA))){
     546          ReportError("Illegal XML 1.0 character reference", pos);
     547          exit(-1);
     548        }
     549        return 0;
     550}
     551
     552static inline int DecRef_check(int pos) {
     553        int block_pos = block_base + pos;
     554        unsigned char* s = (unsigned char*)&source[block_pos];
     555        int ch_val = 0;
     556        while(at_HexDigit<ASCII>(s)){
     557          ch_val = DigitVal<ASCII>(s[0]) + ch_val*10;
     558          if (ch_val> 0x10FFFF ){
     559            ReportError("Illegal character reference", pos);
     560            exit(-1);
     561          }
     562          s++;
     563        }
     564        if ((ch_val == 0x0) || ((ch_val | 0x7FF) == 0xDFFF)|| ((ch_val | 0x1) == 0xFFFF)){
     565          ReportError("Illegal character reference", pos);
     566          exit(-1);
     567        }
     568        else if (((ch_val < 0x20) && (ch_val != 0x9) && (ch_val != 0xD) && (ch_val != 0xA))){
     569          ReportError("Illegal XML 1.0 character reference", pos);
     570          exit(-1);
     571        }
     572        return 0;
     573}
     574
     575static inline int AttRef_check(int pos) {
     576        int block_pos = block_base + pos;
     577        unsigned char* s = (unsigned char*)&source[block_pos];
     578        int ch_val = 0;
     579        if(s[0]=='#'){
     580          s++;
     581          if(s[0]=='x' || s[0]=='X'){
     582            s++;
     583            while(at_HexDigit<ASCII>(s)){
     584              ch_val = HexVal<ASCII>(s[0]) + (ch_val<<4);
     585              s++;
     586            }
     587          }
     588          else{
     589            while(at_HexDigit<ASCII>(s)){
     590              ch_val = DigitVal<ASCII>(s[0]) + ch_val*10;
     591              s++;
     592            }
     593          }
     594          if (ch_val==60){
     595            ReportError("Attribute values contain '<' characters after reference expansion", pos);
     596            exit(-1);
     597          }
     598        }
     599        else if(at_Ref_lt<ASCII>(s)){
     600          ReportError("Attribute values contain '<' characters after reference expansion", pos);
     601          exit(-1);
     602        }
     603        return 0;
     604}
     605*/
     606
     607
     608
    121609  struct U8 {
    122610  BitBlock unibyte;
     
    189677};
    190678
     679  struct Marker {
     680  BitBlock LAngle_scope;
     681  BitBlock Ref_opener;
     682  BitBlock CD_closer;
     683};
     684
    191685  struct CtCDPI_Callouts {
    192686  BitBlock CD_end;
     
    242736  BitBlock EmptyTag_marks;
    243737  BitBlock EndTag_marks;
    244   BitBlock ElemName_starts_no_ns;
    245738  BitBlock Tag_closing;
    246   BitBlock ElemContent_starts;
    247   BitBlock ElemContent_ends;
    248   BitBlock stends;
    249739};
    250740
     
    273763};
    274764
    275   struct Citygml {
    276   BitBlock null;
    277   BitBlock appearanceMember;
    278   BitBlock Appearance;
    279   BitBlock surfaceDataMember;
    280   BitBlock ParameterizedTexture;
    281   BitBlock imageURI;
    282   BitBlock textureType;
    283   BitBlock wrapMode;
    284   BitBlock borderColor;
    285   BitBlock target;
    286   BitBlock TexCoordList;
    287   BitBlock textureCoordinates;
     765  struct Elem {
     766  BitBlock purchaseOrder;
     767  BitBlock shipTo;
     768  BitBlock name;
     769  BitBlock street;
     770  BitBlock city;
     771  BitBlock state;
     772  BitBlock zip;
     773  BitBlock billTo;
     774  BitBlock comment;
     775  BitBlock items;
     776  BitBlock item;
     777  BitBlock productName;
     778  BitBlock quantity;
     779  BitBlock USPrice;
     780  BitBlock shipDate;
    288781};
    289782
     
    291784  Parse_refs() {
    292785 }
    293   IDISA_INLINE void do_block(Lex & lex, Scope1 & scope1, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts) {
    294                 BitBlock ref_error, Ref1, NumRef2, NumRef3, HexRef3, ref_error1, ref_error2;
    295                 BitBlock ref_ends, ref_error3;
     786  IDISA_INLINE void do_block(Lex & lex, Marker & marker, Ref_Callouts & ref_Callouts) {
     787                BitBlock ref_error, Ref_scope, NumRef2, NumRef3, HexRef3, ref_error1;
     788                BitBlock ref_error2, ref_ends, ref_error3;
     789
     790
    296791
    297792
     
    303798        ref_Callouts.HexRef_ends = simd<1>::constant<0>();
    304799        ref_error = simd<1>::constant<0>();
    305         Ref1 = simd_andc(lex.RefStart, ctCDPI_Callouts.CtCDPI_mask);
    306         if ((bitblock::any(Ref1) || carryQ.CarryTest(0, 6))) {
    307           scope1.RefStart = carryQ.BitBlock_advance_ci_co(Ref1, carryQ.get_carry_in(0), 0);
    308           NumRef2 = simd_and(scope1.RefStart, lex.Hash);
    309           ref_Callouts.GenRef_starts = simd_andc(scope1.RefStart, lex.Hash);
    310           NumRef3 = carryQ.BitBlock_advance_ci_co(NumRef2, carryQ.get_carry_in(1), 1);
     800        if ((bitblock::any(marker.Ref_opener) || carryQ.CarryTest(0, 6))) {
     801          carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(marker.Ref_opener, carryQ.get_carry_in(0), Ref_scope));
     802          NumRef2 = simd_and(Ref_scope, lex.Hash);
     803          ref_Callouts.GenRef_starts = simd_andc(Ref_scope, lex.Hash);
     804          carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(NumRef2, carryQ.get_carry_in(1), NumRef3));
    311805          HexRef3 = simd_and(NumRef3, lex.x);
    312806          ref_Callouts.DecRef_starts = simd_andc(NumRef3, lex.x);
    313           ref_Callouts.HexRef_starts = carryQ.BitBlock_advance_ci_co(HexRef3, carryQ.get_carry_in(2), 2);
    314           ref_Callouts.GenRef_ends = carryQ.BitBlock_scanthru_ci_co(ref_Callouts.GenRef_starts, lex.NameScan, carryQ.get_carry_in(3), 3);
    315           ref_Callouts.DecRef_ends = carryQ.BitBlock_scanthru_ci_co(ref_Callouts.DecRef_starts, lex.Digit, carryQ.get_carry_in(4), 4);
    316           ref_Callouts.HexRef_ends = carryQ.BitBlock_scanthru_ci_co(ref_Callouts.HexRef_starts, lex.Hex, carryQ.get_carry_in(5), 5);
     807          carryQ.cq[2] = bitblock::srli<127>(pablo_blk_Advance(HexRef3, carryQ.get_carry_in(2), ref_Callouts.HexRef_starts));
     808          carryQ.cq[3] = bitblock::srli<127>(pablo_blk_ScanThru(ref_Callouts.GenRef_starts, lex.NameScan, carryQ.get_carry_in(3), ref_Callouts.GenRef_ends));
     809          carryQ.cq[4] = bitblock::srli<127>(pablo_blk_ScanThru(ref_Callouts.DecRef_starts, lex.Digit, carryQ.get_carry_in(4), ref_Callouts.DecRef_ends));
     810          carryQ.cq[5] = bitblock::srli<127>(pablo_blk_ScanThru(ref_Callouts.HexRef_starts, lex.Hex, carryQ.get_carry_in(5), ref_Callouts.HexRef_ends));
    317811          ref_error1 = simd_andc(ref_Callouts.DecRef_starts, lex.Digit);
    318812          ref_error2 = simd_andc(ref_Callouts.HexRef_starts, lex.Hex);
    319813          ref_ends = simd_or(simd_or(ref_Callouts.GenRef_ends, ref_Callouts.DecRef_ends), ref_Callouts.HexRef_ends);
    320814          ref_error3 = simd_andc(ref_ends, lex.Semicolon);
    321           ref_error = simd_or(simd_or(ref_error1, ref_error2), ref_error3);
    322           if (bitblock::any(ref_error)) {
    323             error_tracker.NoteError("Reference error found", ref_error);
     815          if (bitblock::any(simd_or(simd_or(ref_error1, ref_error2), ref_error3))) {
     816            assert_0_error("Reference error found", simd_or(simd_or(ref_error1, ref_error2), ref_error3));
    324817          }
    325818        }
     
    329822        carryQ.CarryQ_Adjust(6);
    330823  }
    331   void do_final_block(Lex & lex, Scope1 & scope1, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, BitBlock EOF_mask) {
    332                 BitBlock ref_error, Ref1, NumRef2, NumRef3, HexRef3, ref_error1, ref_error2;
    333                 BitBlock ref_ends, ref_error3;
     824  void do_final_block(Lex & lex, Marker & marker, Ref_Callouts & ref_Callouts, BitBlock EOF_mask) {
     825                BitBlock ref_error, Ref_scope, NumRef2, NumRef3, HexRef3, ref_error1;
     826                BitBlock ref_error2, ref_ends, ref_error3;
     827
     828
    334829
    335830
     
    341836        ref_Callouts.HexRef_ends = simd<1>::constant<0>();
    342837        ref_error = simd<1>::constant<0>();
    343         Ref1 = simd_andc(lex.RefStart, ctCDPI_Callouts.CtCDPI_mask);
    344         if ((bitblock::any(Ref1) || carryQ.CarryTest(0, 6))) {
    345           scope1.RefStart = carryQ.BitBlock_advance_ci_co(Ref1, carryQ.get_carry_in(0), 0);
    346           NumRef2 = simd_and(scope1.RefStart, lex.Hash);
    347           ref_Callouts.GenRef_starts = simd_andc(scope1.RefStart, lex.Hash);
    348           NumRef3 = carryQ.BitBlock_advance_ci_co(NumRef2, carryQ.get_carry_in(1), 1);
     838        if ((bitblock::any(marker.Ref_opener) || carryQ.CarryTest(0, 6))) {
     839          carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(marker.Ref_opener, carryQ.get_carry_in(0), Ref_scope));
     840          NumRef2 = simd_and(Ref_scope, lex.Hash);
     841          ref_Callouts.GenRef_starts = simd_andc(Ref_scope, lex.Hash);
     842          carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(NumRef2, carryQ.get_carry_in(1), NumRef3));
    349843          HexRef3 = simd_and(NumRef3, lex.x);
    350844          ref_Callouts.DecRef_starts = simd_andc(NumRef3, lex.x);
    351           ref_Callouts.HexRef_starts = carryQ.BitBlock_advance_ci_co(HexRef3, carryQ.get_carry_in(2), 2);
    352           ref_Callouts.GenRef_ends = carryQ.BitBlock_scanthru_ci_co(ref_Callouts.GenRef_starts, lex.NameScan, carryQ.get_carry_in(3), 3);
    353           ref_Callouts.DecRef_ends = carryQ.BitBlock_scanthru_ci_co(ref_Callouts.DecRef_starts, lex.Digit, carryQ.get_carry_in(4), 4);
    354           ref_Callouts.HexRef_ends = carryQ.BitBlock_scanthru_ci_co(ref_Callouts.HexRef_starts, lex.Hex, carryQ.get_carry_in(5), 5);
     845          carryQ.cq[2] = bitblock::srli<127>(pablo_blk_Advance(HexRef3, carryQ.get_carry_in(2), ref_Callouts.HexRef_starts));
     846          carryQ.cq[3] = bitblock::srli<127>(pablo_blk_ScanThru(ref_Callouts.GenRef_starts, lex.NameScan, carryQ.get_carry_in(3), ref_Callouts.GenRef_ends));
     847          carryQ.cq[4] = bitblock::srli<127>(pablo_blk_ScanThru(ref_Callouts.DecRef_starts, lex.Digit, carryQ.get_carry_in(4), ref_Callouts.DecRef_ends));
     848          carryQ.cq[5] = bitblock::srli<127>(pablo_blk_ScanThru(ref_Callouts.HexRef_starts, lex.Hex, carryQ.get_carry_in(5), ref_Callouts.HexRef_ends));
    355849          ref_error1 = simd_andc(ref_Callouts.DecRef_starts, lex.Digit);
    356850          ref_error2 = simd_andc(ref_Callouts.HexRef_starts, lex.Hex);
    357851          ref_ends = simd_or(simd_or(ref_Callouts.GenRef_ends, ref_Callouts.DecRef_ends), ref_Callouts.HexRef_ends);
    358852          ref_error3 = simd_andc(ref_ends, lex.Semicolon);
    359           ref_error = simd_or(simd_or(ref_error1, ref_error2), ref_error3);
    360           if (bitblock::any(ref_error)) {
    361             error_tracker.NoteError("Reference error found", ref_error);
     853          if (bitblock::any(simd_or(simd_or(ref_error1, ref_error2), ref_error3))) {
     854            assert_0_error("Reference error found", simd_or(simd_or(ref_error1, ref_error2), ref_error3));
    362855          }
    363856        }
     
    366859        }
    367860  }
    368   void do_segment(Lex lex[], Scope1 scope1[], CtCDPI_Callouts ctCDPI_Callouts[], Ref_Callouts ref_Callouts[], int segment_blocks) {
    369   int i;
    370   for (i = 0; i < segment_blocks; i++)
    371     do_block(lex[i], scope1[i], ctCDPI_Callouts[i], ref_Callouts[i]);
     861  CarryArray<6, 0> carryQ;
     862  };
     863
     864  struct Validate_Elem_Vec {
     865 
     866  IDISA_INLINE void do_block(Elem & elem, Basis_bits & basis_bits) {
     867                BitBlock temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10;
     868                BitBlock temp11, temp12, temp13, temp14, temp15, temp16, temp17, temp18;
     869                BitBlock temp19, temp20, temp21, temp22, temp23, temp24, temp25, temp26;
     870
     871
     872
     873
     874        temp1 = simd_or(basis_bits.bit_0, basis_bits.bit_1);
     875        temp2 = simd_or(basis_bits.bit_2, basis_bits.bit_3);
     876        temp3 = simd_or(temp1, temp2);
     877        temp4 = simd_or(basis_bits.bit_4, basis_bits.bit_5);
     878        temp5 = simd_andc(basis_bits.bit_7, basis_bits.bit_6);
     879        temp6 = simd_andc(temp5, temp4);
     880        elem.purchaseOrder = simd_andc(temp6, temp3);
     881        temp7 = simd_andc(basis_bits.bit_6, basis_bits.bit_7);
     882        temp8 = simd_andc(temp7, temp4);
     883        elem.shipTo = simd_andc(temp8, temp3);
     884        temp9 = simd_and(basis_bits.bit_6, basis_bits.bit_7);
     885        temp10 = simd_andc(temp9, temp4);
     886        elem.name = simd_andc(temp10, temp3);
     887        temp11 = simd_andc(basis_bits.bit_5, basis_bits.bit_4);
     888        temp12 = simd_or(basis_bits.bit_6, basis_bits.bit_7);
     889        temp13 = simd_andc(temp11, temp12);
     890        elem.street = simd_andc(temp13, temp3);
     891        temp14 = simd_and(temp11, temp5);
     892        elem.city = simd_andc(temp14, temp3);
     893        temp15 = simd_and(temp11, temp7);
     894        elem.state = simd_andc(temp15, temp3);
     895        temp16 = simd_and(temp11, temp9);
     896        elem.zip = simd_andc(temp16, temp3);
     897        temp17 = simd_andc(basis_bits.bit_4, basis_bits.bit_5);
     898        temp18 = simd_andc(temp17, temp12);
     899        elem.billTo = simd_andc(temp18, temp3);
     900        temp19 = simd_and(temp17, temp5);
     901        elem.comment = simd_andc(temp19, temp3);
     902        temp20 = simd_and(temp17, temp7);
     903        elem.items = simd_andc(temp20, temp3);
     904        temp21 = simd_and(temp17, temp9);
     905        elem.item = simd_andc(temp21, temp3);
     906        temp22 = simd_and(basis_bits.bit_4, basis_bits.bit_5);
     907        temp23 = simd_andc(temp22, temp12);
     908        elem.productName = simd_andc(temp23, temp3);
     909        temp24 = simd_and(temp22, temp5);
     910        elem.quantity = simd_andc(temp24, temp3);
     911        temp25 = simd_and(temp22, temp7);
     912        elem.USPrice = simd_andc(temp25, temp3);
     913        temp26 = simd_and(temp22, temp9);
     914        elem.shipDate = simd_andc(temp26, temp3);
    372915  }
    373   CarryArray<6> carryQ;
     916  void do_final_block(Elem & elem, Basis_bits & basis_bits, BitBlock EOF_mask) {
     917                BitBlock temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10;
     918                BitBlock temp11, temp12, temp13, temp14, temp15, temp16, temp17, temp18;
     919                BitBlock temp19, temp20, temp21, temp22, temp23, temp24, temp25, temp26;
     920
     921
     922
     923
     924        temp1 = simd_or(basis_bits.bit_0, basis_bits.bit_1);
     925        temp2 = simd_or(basis_bits.bit_2, basis_bits.bit_3);
     926        temp3 = simd_or(temp1, temp2);
     927        temp4 = simd_or(basis_bits.bit_4, basis_bits.bit_5);
     928        temp5 = simd_andc(basis_bits.bit_7, basis_bits.bit_6);
     929        temp6 = simd_andc(temp5, temp4);
     930        elem.purchaseOrder = simd_andc(temp6, temp3);
     931        temp7 = simd_andc(basis_bits.bit_6, basis_bits.bit_7);
     932        temp8 = simd_andc(temp7, temp4);
     933        elem.shipTo = simd_andc(temp8, temp3);
     934        temp9 = simd_and(basis_bits.bit_6, basis_bits.bit_7);
     935        temp10 = simd_andc(temp9, temp4);
     936        elem.name = simd_andc(temp10, temp3);
     937        temp11 = simd_andc(basis_bits.bit_5, basis_bits.bit_4);
     938        temp12 = simd_or(basis_bits.bit_6, basis_bits.bit_7);
     939        temp13 = simd_andc(temp11, temp12);
     940        elem.street = simd_andc(temp13, temp3);
     941        temp14 = simd_and(temp11, temp5);
     942        elem.city = simd_andc(temp14, temp3);
     943        temp15 = simd_and(temp11, temp7);
     944        elem.state = simd_andc(temp15, temp3);
     945        temp16 = simd_and(temp11, temp9);
     946        elem.zip = simd_andc(temp16, temp3);
     947        temp17 = simd_andc(basis_bits.bit_4, basis_bits.bit_5);
     948        temp18 = simd_andc(temp17, temp12);
     949        elem.billTo = simd_andc(temp18, temp3);
     950        temp19 = simd_and(temp17, temp5);
     951        elem.comment = simd_andc(temp19, temp3);
     952        temp20 = simd_and(temp17, temp7);
     953        elem.items = simd_andc(temp20, temp3);
     954        temp21 = simd_and(temp17, temp9);
     955        elem.item = simd_andc(temp21, temp3);
     956        temp22 = simd_and(basis_bits.bit_4, basis_bits.bit_5);
     957        temp23 = simd_andc(temp22, temp12);
     958        elem.productName = simd_andc(temp23, temp3);
     959        temp24 = simd_and(temp22, temp5);
     960        elem.quantity = simd_andc(temp24, temp3);
     961        temp25 = simd_and(temp22, temp7);
     962        elem.USPrice = simd_andc(temp25, temp3);
     963        temp26 = simd_and(temp22, temp9);
     964        elem.shipDate = simd_andc(temp26, temp3);
     965  }
     966 
    374967  };
    375968
     
    377970  Parse_tags() {
    378971 }
    379   IDISA_INLINE void do_block(Lex & lex, Scope1 & scope1, CtCDPI_Callouts & ctCDPI_Callouts, Tag_Callouts & tag_Callouts) {
    380                 BitBlock DQuoteDelim, SQuoteDelim, AttListDelim, LAngleFollow, ColonInElemName;
    381                 BitBlock ParseError, EqToCheck, AttValEnds, AfterWS, AttListEnd, AttNameStart;
    382                 BitBlock AttNameFollow, EqExpected, AttValPos, DQuoteAttVal, SQuoteAttVal;
    383                 BitBlock DQuoteAttEnd, SQuoteAttEnd, AttValEnd, AttValFollow, STagEnds;
    384                 BitBlock advanced, none_emptyelem_con_start, scanfrom, scan_to, EndTagEnds;
    385 
    386 
     972  IDISA_INLINE void do_block(Lex & lex, Marker & marker, Tag_Callouts & tag_Callouts) {
     973                BitBlock EqExpected, AttListEnd, DQuoteDelim, SQuoteDelim, AttListDelim;
     974                BitBlock ParseError, AfterWS, AttNameStart, AttNameFollow, AttValPos;
     975                BitBlock DQuoteAttVal, SQuoteAttVal, DQuoteAttEnd, SQuoteAttEnd, AttValEnd;
     976                BitBlock AttValFollow, STagEnds, EndTagEnds;
     977
     978
     979
     980
     981        EqExpected = simd<1>::constant<0>();
     982        AttListEnd = simd<1>::constant<0>();
    387983        DQuoteDelim = simd_or(lex.DQuote, lex.LAngle);
    388984        SQuoteDelim = simd_or(lex.SQuote, lex.LAngle);
    389985        AttListDelim = simd_or(lex.Slash, lex.RAngle);
    390         LAngleFollow = simd_andc(scope1.LAngle, ctCDPI_Callouts.CtCDPI_mask);
    391         tag_Callouts.ElemName_starts = simd_andc(LAngleFollow, lex.Slash);
    392         tag_Callouts.EndTag_marks = simd_and(LAngleFollow, lex.Slash);
    393         ColonInElemName = simd_and(lex.Colon, lex.NameScan);
    394         tag_Callouts.ElemName_starts_no_ns = simd_or(carryQ.BitBlock_advance_ci_co(ColonInElemName, carryQ.get_carry_in(0), 0), tag_Callouts.ElemName_starts);
    395         tag_Callouts.ElemName_ends = carryQ.BitBlock_scanthru_ci_co(tag_Callouts.ElemName_starts, lex.NameScan, carryQ.get_carry_in(1), 1);
     986        tag_Callouts.ElemName_starts = simd_andc(marker.LAngle_scope, lex.Slash);
     987        tag_Callouts.EndTag_marks = simd_and(marker.LAngle_scope, lex.Slash);
     988        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_ScanThru(tag_Callouts.ElemName_starts, lex.NameScan, carryQ.get_carry_in(0), tag_Callouts.ElemName_ends));
    396989        ParseError = simd_and(tag_Callouts.ElemName_starts, tag_Callouts.ElemName_ends);
    397990        tag_Callouts.AttName_starts = simd<1>::constant<0>();
    398991        tag_Callouts.AttName_ends = simd<1>::constant<0>();
    399         EqToCheck = simd<1>::constant<0>();
    400992        tag_Callouts.AttVal_starts = simd<1>::constant<0>();
    401         AttValEnds = simd<1>::constant<0>();
    402993        tag_Callouts.AttVal_ends = simd<1>::constant<0>();
    403         AfterWS = carryQ.BitBlock_scanthru_ci_co(tag_Callouts.ElemName_ends, lex.WS, carryQ.get_carry_in(2), 2);
    404         AttListEnd = simd_and(AfterWS, AttListDelim);
    405         AttNameStart = simd_andc(AfterWS, AttListDelim);
    406         ParseError = simd_or(ParseError, simd_and(tag_Callouts.ElemName_ends, AttNameStart));
    407         if ((bitblock::any(AttNameStart) || carryQ.CarryTest(3, 8))) {
    408           tag_Callouts.AttName_starts = simd_or(tag_Callouts.AttName_starts, AttNameStart);
    409           AttNameFollow = carryQ.BitBlock_scanthru_ci_co(AttNameStart, lex.NameScan, carryQ.get_carry_in(3), 3);
    410           tag_Callouts.AttName_ends = simd_or(tag_Callouts.AttName_ends, AttNameFollow);
    411           if ((bitblock::any(simd_and(AttNameFollow, lex.WS)) || carryQ.CarryTest(4, 1))) {
    412             EqExpected = carryQ.BitBlock_scanthru_ci_co(AttNameFollow, lex.WS, carryQ.get_carry_in(4), 4);
    413           }
    414           else {
    415             EqExpected = AttNameFollow;
    416             carryQ.CarryDequeueEnqueue(4, 1);
    417           }
    418           EqToCheck = simd_or(EqToCheck, EqExpected);
    419           AttValPos = carryQ.BitBlock_scanthru_ci_co(EqExpected, simd_or(EqExpected, lex.WS), carryQ.get_carry_in(5), 5);
    420           tag_Callouts.AttVal_starts = simd_or(tag_Callouts.AttVal_starts, AttValPos);
    421           DQuoteAttVal = simd_and(AttValPos, lex.DQuote);
    422           SQuoteAttVal = simd_and(AttValPos, lex.SQuote);
    423           DQuoteAttEnd = carryQ.BitBlock_scanthru_ci_co(DQuoteAttVal, simd_not(simd_andc(DQuoteDelim, DQuoteAttVal)), carryQ.get_carry_in(6), 6);
    424           SQuoteAttEnd = carryQ.BitBlock_scanthru_ci_co(SQuoteAttVal, simd_not(simd_andc(SQuoteDelim, SQuoteAttVal)), carryQ.get_carry_in(7), 7);
    425           AttValEnd = simd_or(DQuoteAttEnd, SQuoteAttEnd);
    426           AttValEnds = simd_or(AttValEnds, AttValEnd);
    427           AttValFollow = carryQ.BitBlock_advance_ci_co(AttValEnd, carryQ.get_carry_in(8), 8);
    428           tag_Callouts.AttVal_ends = simd_or(tag_Callouts.AttVal_ends, AttValFollow);
    429           if ((bitblock::any(simd_and(AttValFollow, lex.WS)) || carryQ.CarryTest(9, 1))) {
    430             AfterWS = carryQ.BitBlock_scanthru_ci_co(AttValFollow, lex.WS, carryQ.get_carry_in(9), 9);
    431           }
    432           else {
    433             AfterWS = AttValFollow;
    434             carryQ.CarryDequeueEnqueue(9, 1);
    435           }
    436           AfterWS = carryQ.BitBlock_scanthru_ci_co(AttValFollow, lex.WS, carryQ.get_carry_in(10), 10);
    437           AttListEnd = simd_or(AttListEnd, simd_and(AfterWS, AttListDelim));
     994        if ((bitblock::any(simd_and(tag_Callouts.ElemName_ends, lex.WS)) || carryQ.CarryTest(1, 8))) {
     995          carryQ.cq[1] = bitblock::srli<127>(pablo_blk_ScanThru(tag_Callouts.ElemName_ends, lex.WS, carryQ.get_carry_in(1), AfterWS));
     996          AttListEnd = simd_and(AfterWS, AttListDelim);
    438997          AttNameStart = simd_andc(AfterWS, AttListDelim);
    439           while (bitblock::any(AttNameStart)) {
    440             LocalCarryDeclare(subcarryQ, 8);
     998          if ((bitblock::any(AttNameStart) || carryQ.CarryTest(2, 7))) {
     999            ParseError = simd_or(ParseError, simd_andc(AttNameStart, lex.NameScan));
    4411000            tag_Callouts.AttName_starts = simd_or(tag_Callouts.AttName_starts, AttNameStart);
    442             AttNameFollow = subcarryQ.BitBlock_scanthru_ci_co(AttNameStart, lex.NameScan, simd<1>::constant<0>(), 0);
     1001            carryQ.cq[2] = bitblock::srli<127>(pablo_blk_ScanThru(AttNameStart, lex.NameScan, carryQ.get_carry_in(2), AttNameFollow));
    4431002            tag_Callouts.AttName_ends = simd_or(tag_Callouts.AttName_ends, AttNameFollow);
    444             if (bitblock::any(simd_and(AttNameFollow, lex.WS))) {
    445               EqExpected = subcarryQ.BitBlock_scanthru_ci_co(AttNameFollow, lex.WS, simd<1>::constant<0>(), 1);
     1003            if ((bitblock::any(simd_and(AttNameFollow, lex.WS)) || carryQ.CarryTest(3, 1))) {
     1004              carryQ.cq[3] = bitblock::srli<127>(pablo_blk_ScanThru(AttNameFollow, lex.WS, carryQ.get_carry_in(3), EqExpected));
    4461005            }
    4471006            else {
    4481007              EqExpected = AttNameFollow;
     1008              carryQ.CarryDequeueEnqueue(3, 1);
    4491009            }
    450             EqToCheck = simd_or(EqToCheck, EqExpected);
    451             AttValPos = subcarryQ.BitBlock_scanthru_ci_co(EqExpected, simd_or(EqExpected, lex.WS), simd<1>::constant<0>(), 2);
     1010            ParseError = simd_or(ParseError, simd_andc(EqExpected, lex.Equals));
     1011            carryQ.cq[4] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(EqExpected, lex.WS, carryQ.get_carry_in(4), AttValPos));
    4521012            tag_Callouts.AttVal_starts = simd_or(tag_Callouts.AttVal_starts, AttValPos);
    4531013            DQuoteAttVal = simd_and(AttValPos, lex.DQuote);
    4541014            SQuoteAttVal = simd_and(AttValPos, lex.SQuote);
    455             DQuoteAttEnd = subcarryQ.BitBlock_scanthru_ci_co(DQuoteAttVal, simd_not(simd_andc(DQuoteDelim, DQuoteAttVal)), simd<1>::constant<0>(), 3);
    456             SQuoteAttEnd = subcarryQ.BitBlock_scanthru_ci_co(SQuoteAttVal, simd_not(simd_andc(SQuoteDelim, SQuoteAttVal)), simd<1>::constant<0>(), 4);
     1015            carryQ.cq[5] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(DQuoteAttVal, simd_not(DQuoteDelim), carryQ.get_carry_in(5), DQuoteAttEnd));
     1016            carryQ.cq[6] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(SQuoteAttVal, simd_not(SQuoteDelim), carryQ.get_carry_in(6), SQuoteAttEnd));
    4571017            AttValEnd = simd_or(DQuoteAttEnd, SQuoteAttEnd);
    458             AttValEnds = simd_or(AttValEnds, AttValEnd);
    459             AttValFollow = subcarryQ.BitBlock_advance_ci_co(AttValEnd, simd<1>::constant<0>(), 5);
     1018            ParseError = simd_or(ParseError, simd_andc(simd_or(AttValPos, AttValEnd), simd_or(lex.DQuote, lex.SQuote)));
     1019            carryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(AttValEnd, carryQ.get_carry_in(7), AttValFollow));
    4601020            tag_Callouts.AttVal_ends = simd_or(tag_Callouts.AttVal_ends, AttValFollow);
    461             if (bitblock::any(simd_and(AttValFollow, lex.WS))) {
    462               AfterWS = subcarryQ.BitBlock_scanthru_ci_co(AttValFollow, lex.WS, simd<1>::constant<0>(), 6);
     1021            if ((bitblock::any(simd_and(AttValFollow, lex.WS)) || carryQ.CarryTest(8, 1))) {
     1022              carryQ.cq[8] = bitblock::srli<127>(pablo_blk_ScanThru(AttValFollow, lex.WS, carryQ.get_carry_in(8), AfterWS));
     1023              AttListEnd = simd_or(AttListEnd, simd_and(AfterWS, AttListDelim));
     1024              AttNameStart = simd_andc(AfterWS, AttListDelim);
    4631025            }
    4641026            else {
    465               AfterWS = AttValFollow;
     1027              AttListEnd = simd_or(AttListEnd, simd_and(AttValFollow, AttListDelim));
     1028              AttNameStart = simd_andc(AttValFollow, AttListDelim);
     1029              carryQ.CarryDequeueEnqueue(8, 1);
    4661030            }
    467             AfterWS = subcarryQ.BitBlock_scanthru_ci_co(AttValFollow, lex.WS, simd<1>::constant<0>(), 7);
    468             AttListEnd = simd_or(AttListEnd, simd_and(AfterWS, AttListDelim));
    469             AttNameStart = simd_andc(AfterWS, AttListDelim);
    470             carryQ.CarryCombine((ICarryQueue *) &subcarryQ, 3, 8);
     1031            ParseError = simd_or(ParseError, simd_and(AttValFollow, AttNameStart));
     1032            while (bitblock::any(AttNameStart)) {
     1033              LocalCarryDeclare(subcarryQ, 7);
     1034              ParseError = simd_or(ParseError, simd_andc(AttNameStart, lex.NameScan));
     1035              tag_Callouts.AttName_starts = simd_or(tag_Callouts.AttName_starts, AttNameStart);
     1036              subcarryQ.cq[0] = bitblock::srli<127>(pablo_blk_ScanThru(AttNameStart, lex.NameScan, simd<1>::constant<0>(), AttNameFollow));
     1037              tag_Callouts.AttName_ends = simd_or(tag_Callouts.AttName_ends, AttNameFollow);
     1038              if (bitblock::any(simd_and(AttNameFollow, lex.WS))) {
     1039                subcarryQ.cq[1] = bitblock::srli<127>(pablo_blk_ScanThru(AttNameFollow, lex.WS, simd<1>::constant<0>(), EqExpected));
     1040              }
     1041              else {
     1042                EqExpected = AttNameFollow;
     1043                subcarryQ.CarryDequeueEnqueue(3, 1);
     1044              }
     1045              ParseError = simd_or(ParseError, simd_andc(EqExpected, lex.Equals));
     1046              subcarryQ.cq[2] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(EqExpected, lex.WS, simd<1>::constant<0>(), AttValPos));
     1047              tag_Callouts.AttVal_starts = simd_or(tag_Callouts.AttVal_starts, AttValPos);
     1048              DQuoteAttVal = simd_and(AttValPos, lex.DQuote);
     1049              SQuoteAttVal = simd_and(AttValPos, lex.SQuote);
     1050              subcarryQ.cq[3] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(DQuoteAttVal, simd_not(DQuoteDelim), simd<1>::constant<0>(), DQuoteAttEnd));
     1051              subcarryQ.cq[4] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(SQuoteAttVal, simd_not(SQuoteDelim), simd<1>::constant<0>(), SQuoteAttEnd));
     1052              AttValEnd = simd_or(DQuoteAttEnd, SQuoteAttEnd);
     1053              ParseError = simd_or(ParseError, simd_andc(simd_or(AttValPos, AttValEnd), simd_or(lex.DQuote, lex.SQuote)));
     1054              subcarryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(AttValEnd, simd<1>::constant<0>(), AttValFollow));
     1055              tag_Callouts.AttVal_ends = simd_or(tag_Callouts.AttVal_ends, AttValFollow);
     1056              if (bitblock::any(simd_and(AttValFollow, lex.WS))) {
     1057                subcarryQ.cq[6] = bitblock::srli<127>(pablo_blk_ScanThru(AttValFollow, lex.WS, simd<1>::constant<0>(), AfterWS));
     1058                AttListEnd = simd_or(AttListEnd, simd_and(AfterWS, AttListDelim));
     1059                AttNameStart = simd_andc(AfterWS, AttListDelim);
     1060              }
     1061              else {
     1062                AttListEnd = simd_or(AttListEnd, simd_and(AttValFollow, AttListDelim));
     1063                AttNameStart = simd_andc(AttValFollow, AttListDelim);
     1064                subcarryQ.CarryDequeueEnqueue(8, 1);
     1065              }
     1066              ParseError = simd_or(ParseError, simd_and(AttValFollow, AttNameStart));
     1067              carryQ.CarryCombine(subcarryQ.cq, 2, 7);
     1068            }
    4711069          }
     1070          else {
     1071            carryQ.CarryDequeueEnqueue(2, 7);
     1072          }
    4721073        }
    4731074        else {
    474           carryQ.CarryDequeueEnqueue(3, 8);
     1075          AttListEnd = simd_and(tag_Callouts.ElemName_ends, AttListDelim);
     1076          ParseError = simd_or(ParseError, simd_andc(tag_Callouts.ElemName_ends, AttListDelim));
     1077          carryQ.CarryDequeueEnqueue(1, 8);
    4751078        }
    4761079        STagEnds = simd_and(AttListEnd, lex.RAngle);
    477         tag_Callouts.EmptyTag_marks = carryQ.BitBlock_advance_ci_co(simd_and(AttListEnd, lex.Slash), carryQ.get_carry_in(11), 11);
    478         tag_Callouts.Tag_closing = simd_or(tag_Callouts.EmptyTag_marks, tag_Callouts.EndTag_marks);
    479         ParseError = simd_or(ParseError, simd_and(tag_Callouts.AttVal_ends, tag_Callouts.AttName_starts));
    480         ParseError = simd_or(ParseError, simd_and(tag_Callouts.AttName_starts, tag_Callouts.AttName_ends));
    481         ParseError = simd_or(ParseError, simd_andc(EqToCheck, lex.Equals));
    482         ParseError = simd_or(ParseError, simd_andc(tag_Callouts.AttVal_starts, simd_or(lex.DQuote, lex.SQuote)));
    483         ParseError = simd_or(ParseError, simd_andc(AttValEnds, simd_or(lex.DQuote, lex.SQuote)));
     1080        carryQ.cq[9] = bitblock::srli<127>(pablo_blk_Advance(simd_and(AttListEnd, lex.Slash), carryQ.get_carry_in(9), tag_Callouts.EmptyTag_marks));
    4841081        ParseError = simd_or(ParseError, simd_andc(tag_Callouts.EmptyTag_marks, lex.RAngle));
    485         advanced = carryQ.BitBlock_advance_ci_co(STagEnds, carryQ.get_carry_in(12), 12);
    486         tag_Callouts.stends = STagEnds;
    487         none_emptyelem_con_start = carryQ.BitBlock_scanthru_ci_co(STagEnds, lex.WS, carryQ.get_carry_in(13), 13);
    488         tag_Callouts.ElemContent_starts = simd_andc(simd_andc(simd_andc(carryQ.BitBlock_advance_ci_co(none_emptyelem_con_start, carryQ.get_carry_in(14), 14), lex.LAngle), lex.CR), lex.LF);
    489         scanfrom = tag_Callouts.ElemContent_starts;
    490         scan_to = lex.LAngle;
    491         tag_Callouts.ElemContent_ends = carryQ.BitBlock_scanthru_ci_co(scanfrom, simd_not(scan_to), carryQ.get_carry_in(15), 15);
    492         EndTagEnds = carryQ.BitBlock_scanthru_ci_co(tag_Callouts.EndTag_marks, simd_or(tag_Callouts.EndTag_marks, lex.NameScan), carryQ.get_carry_in(16), 16);
    493         if ((bitblock::any(simd_and(EndTagEnds, lex.WS)) || carryQ.CarryTest(17, 1))) {
    494           EndTagEnds = carryQ.BitBlock_scanthru_ci_co(EndTagEnds, lex.WS, carryQ.get_carry_in(17), 17);
     1082        carryQ.cq[10] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(tag_Callouts.EndTag_marks, lex.NameScan, carryQ.get_carry_in(10), EndTagEnds));
     1083        if ((bitblock::any(simd_and(EndTagEnds, lex.WS)) || carryQ.CarryTest(11, 1))) {
     1084          carryQ.cq[11] = bitblock::srli<127>(pablo_blk_ScanThru(EndTagEnds, lex.WS, carryQ.get_carry_in(11), EndTagEnds));
    4951085        }
    4961086        else {
    497           carryQ.CarryDequeueEnqueue(17, 1);
     1087          carryQ.CarryDequeueEnqueue(11, 1);
    4981088        }
    4991089        ParseError = simd_or(ParseError, simd_andc(EndTagEnds, lex.RAngle));
    5001090        if (bitblock::any(ParseError)) {
    501           error_tracker.NoteError("Tag parsing error found", ParseError);
    502         }
    503         tag_Callouts.AttVal_spans = carryQ.BitBlock_sub_ci_co(tag_Callouts.AttVal_ends, tag_Callouts.AttVal_starts, carryQ.get_carry_in(18), 18);
    504         carryQ.CarryQ_Adjust(19);
     1091          assert_0_error("Tag parsing error found", ParseError);
     1092        }
     1093        carryQ.cq[12] = bitblock::srli<127>(pablo_blk_SpanUpTo(tag_Callouts.AttVal_starts, tag_Callouts.AttVal_ends, carryQ.get_carry_in(12), tag_Callouts.AttVal_spans));
     1094        tag_Callouts.Tag_closing = simd_or(tag_Callouts.EmptyTag_marks, tag_Callouts.EndTag_marks);
     1095        carryQ.CarryQ_Adjust(13);
    5051096  }
    506   void do_final_block(Lex & lex, Scope1 & scope1, CtCDPI_Callouts & ctCDPI_Callouts, Tag_Callouts & tag_Callouts, BitBlock EOF_mask) {
    507                 BitBlock DQuoteDelim, SQuoteDelim, AttListDelim, LAngleFollow, ColonInElemName;
    508                 BitBlock ParseError, EqToCheck, AttValEnds, AfterWS, AttListEnd, AttNameStart;
    509                 BitBlock AttNameFollow, EqExpected, AttValPos, DQuoteAttVal, SQuoteAttVal;
    510                 BitBlock DQuoteAttEnd, SQuoteAttEnd, AttValEnd, AttValFollow, STagEnds;
    511                 BitBlock advanced, none_emptyelem_con_start, scanfrom, scan_to, EndTagEnds;
    512 
    513 
     1097  void do_final_block(Lex & lex, Marker & marker, Tag_Callouts & tag_Callouts, BitBlock EOF_mask) {
     1098                BitBlock EqExpected, AttListEnd, DQuoteDelim, SQuoteDelim, AttListDelim;
     1099                BitBlock ParseError, AfterWS, AttNameStart, AttNameFollow, AttValPos;
     1100                BitBlock DQuoteAttVal, SQuoteAttVal, DQuoteAttEnd, SQuoteAttEnd, AttValEnd;
     1101                BitBlock AttValFollow, STagEnds, EndTagEnds;
     1102
     1103
     1104
     1105
     1106        EqExpected = simd<1>::constant<0>();
     1107        AttListEnd = simd<1>::constant<0>();
    5141108        DQuoteDelim = simd_or(lex.DQuote, lex.LAngle);
    5151109        SQuoteDelim = simd_or(lex.SQuote, lex.LAngle);
    5161110        AttListDelim = simd_or(lex.Slash, lex.RAngle);
    517         LAngleFollow = simd_andc(scope1.LAngle, ctCDPI_Callouts.CtCDPI_mask);
    518         tag_Callouts.ElemName_starts = simd_andc(LAngleFollow, lex.Slash);
    519         tag_Callouts.EndTag_marks = simd_and(LAngleFollow, lex.Slash);
    520         ColonInElemName = simd_and(lex.Colon, lex.NameScan);
    521         tag_Callouts.ElemName_starts_no_ns = simd_or(carryQ.BitBlock_advance_ci_co(ColonInElemName, carryQ.get_carry_in(0), 0), tag_Callouts.ElemName_starts);
    522         tag_Callouts.ElemName_ends = carryQ.BitBlock_scanthru_ci_co(tag_Callouts.ElemName_starts, lex.NameScan, carryQ.get_carry_in(1), 1);
     1111        tag_Callouts.ElemName_starts = simd_andc(marker.LAngle_scope, lex.Slash);
     1112        tag_Callouts.EndTag_marks = simd_and(marker.LAngle_scope, lex.Slash);
     1113        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_ScanThru(tag_Callouts.ElemName_starts, lex.NameScan, carryQ.get_carry_in(0), tag_Callouts.ElemName_ends));
    5231114        ParseError = simd_and(tag_Callouts.ElemName_starts, tag_Callouts.ElemName_ends);
    5241115        tag_Callouts.AttName_starts = simd<1>::constant<0>();
    5251116        tag_Callouts.AttName_ends = simd<1>::constant<0>();
    526         EqToCheck = simd<1>::constant<0>();
    5271117        tag_Callouts.AttVal_starts = simd<1>::constant<0>();
    528         AttValEnds = simd<1>::constant<0>();
    5291118        tag_Callouts.AttVal_ends = simd<1>::constant<0>();
    530         AfterWS = carryQ.BitBlock_scanthru_ci_co(tag_Callouts.ElemName_ends, lex.WS, carryQ.get_carry_in(2), 2);
    531         AttListEnd = simd_and(AfterWS, AttListDelim);
    532         AttNameStart = simd_andc(AfterWS, AttListDelim);
    533         ParseError = simd_or(ParseError, simd_and(tag_Callouts.ElemName_ends, AttNameStart));
    534         if ((bitblock::any(simd_and(AttNameStart, EOF_mask)) || carryQ.CarryTest(3, 8))) {
    535           tag_Callouts.AttName_starts = simd_or(tag_Callouts.AttName_starts, AttNameStart);
    536           AttNameFollow = carryQ.BitBlock_scanthru_ci_co(AttNameStart, lex.NameScan, carryQ.get_carry_in(3), 3);
    537           tag_Callouts.AttName_ends = simd_or(tag_Callouts.AttName_ends, AttNameFollow);
    538           if ((bitblock::any(simd_and(AttNameFollow, lex.WS)) || carryQ.CarryTest(4, 1))) {
    539             EqExpected = carryQ.BitBlock_scanthru_ci_co(AttNameFollow, lex.WS, carryQ.get_carry_in(4), 4);
    540           }
    541           else {
    542             EqExpected = AttNameFollow;
    543             carryQ.CarryDequeueEnqueue(4, 1);
    544           }
    545           EqToCheck = simd_or(EqToCheck, EqExpected);
    546           AttValPos = carryQ.BitBlock_scanthru_ci_co(EqExpected, simd_or(EqExpected, lex.WS), carryQ.get_carry_in(5), 5);
    547           tag_Callouts.AttVal_starts = simd_or(tag_Callouts.AttVal_starts, AttValPos);
    548           DQuoteAttVal = simd_and(AttValPos, lex.DQuote);
    549           SQuoteAttVal = simd_and(AttValPos, lex.SQuote);
    550           DQuoteAttEnd = carryQ.BitBlock_scanthru_ci_co(DQuoteAttVal, simd_andc(EOF_mask, simd_andc(DQuoteDelim, DQuoteAttVal)), carryQ.get_carry_in(6), 6);
    551           SQuoteAttEnd = carryQ.BitBlock_scanthru_ci_co(SQuoteAttVal, simd_andc(EOF_mask, simd_andc(SQuoteDelim, SQuoteAttVal)), carryQ.get_carry_in(7), 7);
    552           AttValEnd = simd_or(DQuoteAttEnd, SQuoteAttEnd);
    553           AttValEnds = simd_or(AttValEnds, AttValEnd);
    554           AttValFollow = carryQ.BitBlock_advance_ci_co(AttValEnd, carryQ.get_carry_in(8), 8);
    555           tag_Callouts.AttVal_ends = simd_or(tag_Callouts.AttVal_ends, AttValFollow);
    556           if ((bitblock::any(simd_and(AttValFollow, lex.WS)) || carryQ.CarryTest(9, 1))) {
    557             AfterWS = carryQ.BitBlock_scanthru_ci_co(AttValFollow, lex.WS, carryQ.get_carry_in(9), 9);
    558           }
    559           else {
    560             AfterWS = AttValFollow;
    561             carryQ.CarryDequeueEnqueue(9, 1);
    562           }
    563           AfterWS = carryQ.BitBlock_scanthru_ci_co(AttValFollow, lex.WS, carryQ.get_carry_in(10), 10);
    564           AttListEnd = simd_or(AttListEnd, simd_and(AfterWS, AttListDelim));
     1119        if ((bitblock::any(simd_and(tag_Callouts.ElemName_ends, lex.WS)) || carryQ.CarryTest(1, 8))) {
     1120          carryQ.cq[1] = bitblock::srli<127>(pablo_blk_ScanThru(tag_Callouts.ElemName_ends, lex.WS, carryQ.get_carry_in(1), AfterWS));
     1121          AttListEnd = simd_and(AfterWS, AttListDelim);
    5651122          AttNameStart = simd_andc(AfterWS, AttListDelim);
    566           while (bitblock::any(simd_and(AttNameStart, EOF_mask))) {
    567             LocalCarryDeclare(subcarryQ, 8);
     1123          if ((bitblock::any(simd_and(AttNameStart, EOF_mask)) || carryQ.CarryTest(2, 7))) {
     1124            ParseError = simd_or(ParseError, simd_andc(AttNameStart, lex.NameScan));
    5681125            tag_Callouts.AttName_starts = simd_or(tag_Callouts.AttName_starts, AttNameStart);
    569             AttNameFollow = subcarryQ.BitBlock_scanthru_ci_co(AttNameStart, lex.NameScan, simd<1>::constant<0>(), 0);
     1126            carryQ.cq[2] = bitblock::srli<127>(pablo_blk_ScanThru(AttNameStart, lex.NameScan, carryQ.get_carry_in(2), AttNameFollow));
    5701127            tag_Callouts.AttName_ends = simd_or(tag_Callouts.AttName_ends, AttNameFollow);
    571             if (bitblock::any(simd_and(AttNameFollow, lex.WS))) {
    572               EqExpected = subcarryQ.BitBlock_scanthru_ci_co(AttNameFollow, lex.WS, simd<1>::constant<0>(), 1);
     1128            if ((bitblock::any(simd_and(AttNameFollow, lex.WS)) || carryQ.CarryTest(3, 1))) {
     1129              carryQ.cq[3] = bitblock::srli<127>(pablo_blk_ScanThru(AttNameFollow, lex.WS, carryQ.get_carry_in(3), EqExpected));
    5731130            }
    5741131            else {
    5751132              EqExpected = AttNameFollow;
     1133              carryQ.CarryDequeueEnqueue(3, 1);
    5761134            }
    577             EqToCheck = simd_or(EqToCheck, EqExpected);
    578             AttValPos = subcarryQ.BitBlock_scanthru_ci_co(EqExpected, simd_or(EqExpected, lex.WS), simd<1>::constant<0>(), 2);
     1135            ParseError = simd_or(ParseError, simd_andc(EqExpected, lex.Equals));
     1136            carryQ.cq[4] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(EqExpected, lex.WS, carryQ.get_carry_in(4), AttValPos));
    5791137            tag_Callouts.AttVal_starts = simd_or(tag_Callouts.AttVal_starts, AttValPos);
    5801138            DQuoteAttVal = simd_and(AttValPos, lex.DQuote);
    5811139            SQuoteAttVal = simd_and(AttValPos, lex.SQuote);
    582             DQuoteAttEnd = subcarryQ.BitBlock_scanthru_ci_co(DQuoteAttVal, simd_andc(EOF_mask, simd_andc(DQuoteDelim, DQuoteAttVal)), simd<1>::constant<0>(), 3);
    583             SQuoteAttEnd = subcarryQ.BitBlock_scanthru_ci_co(SQuoteAttVal, simd_andc(EOF_mask, simd_andc(SQuoteDelim, SQuoteAttVal)), simd<1>::constant<0>(), 4);
     1140            carryQ.cq[5] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(DQuoteAttVal, simd_andc(EOF_mask, DQuoteDelim), carryQ.get_carry_in(5), DQuoteAttEnd));
     1141            carryQ.cq[6] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(SQuoteAttVal, simd_andc(EOF_mask, SQuoteDelim), carryQ.get_carry_in(6), SQuoteAttEnd));
    5841142            AttValEnd = simd_or(DQuoteAttEnd, SQuoteAttEnd);
    585             AttValEnds = simd_or(AttValEnds, AttValEnd);
    586             AttValFollow = subcarryQ.BitBlock_advance_ci_co(AttValEnd, simd<1>::constant<0>(), 5);
     1143            ParseError = simd_or(ParseError, simd_andc(simd_or(AttValPos, AttValEnd), simd_or(lex.DQuote, lex.SQuote)));
     1144            carryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(AttValEnd, carryQ.get_carry_in(7), AttValFollow));
    5871145            tag_Callouts.AttVal_ends = simd_or(tag_Callouts.AttVal_ends, AttValFollow);
    588             if (bitblock::any(simd_and(AttValFollow, lex.WS))) {
    589               AfterWS = subcarryQ.BitBlock_scanthru_ci_co(AttValFollow, lex.WS, simd<1>::constant<0>(), 6);
     1146            if ((bitblock::any(simd_and(AttValFollow, lex.WS)) || carryQ.CarryTest(8, 1))) {
     1147              carryQ.cq[8] = bitblock::srli<127>(pablo_blk_ScanThru(AttValFollow, lex.WS, carryQ.get_carry_in(8), AfterWS));
     1148              AttListEnd = simd_or(AttListEnd, simd_and(AfterWS, AttListDelim));
     1149              AttNameStart = simd_andc(AfterWS, AttListDelim);
    5901150            }
    5911151            else {
    592               AfterWS = AttValFollow;
     1152              AttListEnd = simd_or(AttListEnd, simd_and(AttValFollow, AttListDelim));
     1153              AttNameStart = simd_andc(AttValFollow, AttListDelim);
     1154              carryQ.CarryDequeueEnqueue(8, 1);
    5931155            }
    594             AfterWS = subcarryQ.BitBlock_scanthru_ci_co(AttValFollow, lex.WS, simd<1>::constant<0>(), 7);
    595             AttListEnd = simd_or(AttListEnd, simd_and(AfterWS, AttListDelim));
    596             AttNameStart = simd_andc(AfterWS, AttListDelim);
    597             carryQ.CarryCombine((ICarryQueue *) &subcarryQ, 3, 8);
     1156            ParseError = simd_or(ParseError, simd_and(AttValFollow, AttNameStart));
     1157            while (bitblock::any(simd_and(AttNameStart, EOF_mask))) {
     1158              LocalCarryDeclare(subcarryQ, 7);
     1159              ParseError = simd_or(ParseError, simd_andc(AttNameStart, lex.NameScan));
     1160              tag_Callouts.AttName_starts = simd_or(tag_Callouts.AttName_starts, AttNameStart);
     1161              subcarryQ.cq[0] = bitblock::srli<127>(pablo_blk_ScanThru(AttNameStart, lex.NameScan, simd<1>::constant<0>(), AttNameFollow));
     1162              tag_Callouts.AttName_ends = simd_or(tag_Callouts.AttName_ends, AttNameFollow);
     1163              if (bitblock::any(simd_and(AttNameFollow, lex.WS))) {
     1164                subcarryQ.cq[1] = bitblock::srli<127>(pablo_blk_ScanThru(AttNameFollow, lex.WS, simd<1>::constant<0>(), EqExpected));
     1165              }
     1166              else {
     1167                EqExpected = AttNameFollow;
     1168                subcarryQ.CarryDequeueEnqueue(3, 1);
     1169              }
     1170              ParseError = simd_or(ParseError, simd_andc(EqExpected, lex.Equals));
     1171              subcarryQ.cq[2] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(EqExpected, lex.WS, simd<1>::constant<0>(), AttValPos));
     1172              tag_Callouts.AttVal_starts = simd_or(tag_Callouts.AttVal_starts, AttValPos);
     1173              DQuoteAttVal = simd_and(AttValPos, lex.DQuote);
     1174              SQuoteAttVal = simd_and(AttValPos, lex.SQuote);
     1175              subcarryQ.cq[3] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(DQuoteAttVal, simd_andc(EOF_mask, DQuoteDelim), simd<1>::constant<0>(), DQuoteAttEnd));
     1176              subcarryQ.cq[4] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(SQuoteAttVal, simd_andc(EOF_mask, SQuoteDelim), simd<1>::constant<0>(), SQuoteAttEnd));
     1177              AttValEnd = simd_or(DQuoteAttEnd, SQuoteAttEnd);
     1178              ParseError = simd_or(ParseError, simd_andc(simd_or(AttValPos, AttValEnd), simd_or(lex.DQuote, lex.SQuote)));
     1179              subcarryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(AttValEnd, simd<1>::constant<0>(), AttValFollow));
     1180              tag_Callouts.AttVal_ends = simd_or(tag_Callouts.AttVal_ends, AttValFollow);
     1181              if (bitblock::any(simd_and(AttValFollow, lex.WS))) {
     1182                subcarryQ.cq[6] = bitblock::srli<127>(pablo_blk_ScanThru(AttValFollow, lex.WS, simd<1>::constant<0>(), AfterWS));
     1183                AttListEnd = simd_or(AttListEnd, simd_and(AfterWS, AttListDelim));
     1184                AttNameStart = simd_andc(AfterWS, AttListDelim);
     1185              }
     1186              else {
     1187                AttListEnd = simd_or(AttListEnd, simd_and(AttValFollow, AttListDelim));
     1188                AttNameStart = simd_andc(AttValFollow, AttListDelim);
     1189                subcarryQ.CarryDequeueEnqueue(8, 1);
     1190              }
     1191              ParseError = simd_or(ParseError, simd_and(AttValFollow, AttNameStart));
     1192              carryQ.CarryCombine(subcarryQ.cq, 2, 7);
     1193            }
    5981194          }
     1195          else {
     1196            carryQ.CarryDequeueEnqueue(2, 7);
     1197          }
    5991198        }
    6001199        else {
    601           carryQ.CarryDequeueEnqueue(3, 8);
     1200          AttListEnd = simd_and(tag_Callouts.ElemName_ends, AttListDelim);
     1201          ParseError = simd_or(ParseError, simd_andc(tag_Callouts.ElemName_ends, AttListDelim));
     1202          carryQ.CarryDequeueEnqueue(1, 8);
    6021203        }
    6031204        STagEnds = simd_and(AttListEnd, lex.RAngle);
    604         tag_Callouts.EmptyTag_marks = carryQ.BitBlock_advance_ci_co(simd_and(AttListEnd, lex.Slash), carryQ.get_carry_in(11), 11);
    605         tag_Callouts.Tag_closing = simd_or(tag_Callouts.EmptyTag_marks, tag_Callouts.EndTag_marks);
    606         ParseError = simd_or(ParseError, simd_and(tag_Callouts.AttVal_ends, tag_Callouts.AttName_starts));
    607         ParseError = simd_or(ParseError, simd_and(tag_Callouts.AttName_starts, tag_Callouts.AttName_ends));
    608         ParseError = simd_or(ParseError, simd_andc(EqToCheck, lex.Equals));
    609         ParseError = simd_or(ParseError, simd_andc(tag_Callouts.AttVal_starts, simd_or(lex.DQuote, lex.SQuote)));
    610         ParseError = simd_or(ParseError, simd_andc(AttValEnds, simd_or(lex.DQuote, lex.SQuote)));
     1205        carryQ.cq[9] = bitblock::srli<127>(pablo_blk_Advance(simd_and(AttListEnd, lex.Slash), carryQ.get_carry_in(9), tag_Callouts.EmptyTag_marks));
    6111206        ParseError = simd_or(ParseError, simd_andc(tag_Callouts.EmptyTag_marks, lex.RAngle));
    612         advanced = carryQ.BitBlock_advance_ci_co(STagEnds, carryQ.get_carry_in(12), 12);
    613         tag_Callouts.stends = STagEnds;
    614         none_emptyelem_con_start = carryQ.BitBlock_scanthru_ci_co(STagEnds, lex.WS, carryQ.get_carry_in(13), 13);
    615         tag_Callouts.ElemContent_starts = simd_andc(simd_andc(simd_andc(carryQ.BitBlock_advance_ci_co(none_emptyelem_con_start, carryQ.get_carry_in(14), 14), lex.LAngle), lex.CR), lex.LF);
    616         scanfrom = tag_Callouts.ElemContent_starts;
    617         scan_to = lex.LAngle;
    618         tag_Callouts.ElemContent_ends = carryQ.BitBlock_scanthru_ci_co(scanfrom, simd_andc(EOF_mask, scan_to), carryQ.get_carry_in(15), 15);
    619         EndTagEnds = carryQ.BitBlock_scanthru_ci_co(tag_Callouts.EndTag_marks, simd_or(tag_Callouts.EndTag_marks, lex.NameScan), carryQ.get_carry_in(16), 16);
    620         if ((bitblock::any(simd_and(EndTagEnds, lex.WS)) || carryQ.CarryTest(17, 1))) {
    621           EndTagEnds = carryQ.BitBlock_scanthru_ci_co(EndTagEnds, lex.WS, carryQ.get_carry_in(17), 17);
     1207        carryQ.cq[10] = bitblock::srli<127>(pablo_blk_AdvanceThenScanThru(tag_Callouts.EndTag_marks, lex.NameScan, carryQ.get_carry_in(10), EndTagEnds));
     1208        if ((bitblock::any(simd_and(EndTagEnds, lex.WS)) || carryQ.CarryTest(11, 1))) {
     1209          carryQ.cq[11] = bitblock::srli<127>(pablo_blk_ScanThru(EndTagEnds, lex.WS, carryQ.get_carry_in(11), EndTagEnds));
    6221210        }
    6231211        else {
    624           carryQ.CarryDequeueEnqueue(17, 1);
     1212          carryQ.CarryDequeueEnqueue(11, 1);
    6251213        }
    6261214        ParseError = simd_or(ParseError, simd_andc(EndTagEnds, lex.RAngle));
    6271215        if (bitblock::any(ParseError)) {
    628           error_tracker.NoteError("Tag parsing error found", ParseError);
    629         }
    630         tag_Callouts.AttVal_spans = carryQ.BitBlock_sub_ci_co(tag_Callouts.AttVal_ends, tag_Callouts.AttVal_starts, carryQ.get_carry_in(18), 18);
     1216          assert_0_error("Tag parsing error found", ParseError);
     1217        }
     1218        carryQ.cq[12] = bitblock::srli<127>(pablo_blk_SpanUpTo(tag_Callouts.AttVal_starts, tag_Callouts.AttVal_ends, carryQ.get_carry_in(12), tag_Callouts.AttVal_spans));
     1219        tag_Callouts.Tag_closing = simd_or(tag_Callouts.EmptyTag_marks, tag_Callouts.EndTag_marks);
    6311220  }
    632   void do_segment(Lex lex[], Scope1 scope1[], CtCDPI_Callouts ctCDPI_Callouts[], Tag_Callouts tag_Callouts[], int segment_blocks) {
    633   int i;
    634   for (i = 0; i < segment_blocks; i++)
    635     do_block(lex[i], scope1[i], ctCDPI_Callouts[i], tag_Callouts[i]);
    636   }
    637   CarryArray<19> carryQ;
     1221  CarryArray<13, 0> carryQ;
    6381222  };
    6391223
     
    6541238                BitBlock u8lastscope, u8error1, u8error2, u8error3, u8error4, EF_BF_pending;
    6551239                BitBlock u8mismatch;
     1240
     1241
    6561242
    6571243
     
    7711357          u8.badprefix = simd_or(temp68, temp71);
    7721358          u8_error = u8.badprefix;
    773           u8.scope22 = carryQ.BitBlock_advance_ci_co(u8.prefix2, carryQ.get_carry_in(0), 0);
     1359          carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(u8.prefix2, carryQ.get_carry_in(0), u8.scope22));
    7741360          u8anyscope = u8.scope22;
    7751361          if ((bitblock::any(simd_or(u8.prefix3, u8.prefix4)) || carryQ.CarryTest(1, 11))) {
     
    7871373            u8.xBF = simd_and(temp73, temp23);
    7881374            u8.xBE = simd_and(temp73, temp15);
    789             u8.xE0_scope = carryQ.BitBlock_advance_ci_co(xE0, carryQ.get_carry_in(1), 1);
    790             u8.xED_scope = carryQ.BitBlock_advance_ci_co(xED, carryQ.get_carry_in(2), 2);
    791             u8.xF0_scope = carryQ.BitBlock_advance_ci_co(xF0, carryQ.get_carry_in(3), 3);
    792             u8.xF4_scope = carryQ.BitBlock_advance_ci_co(xF4, carryQ.get_carry_in(4), 4);
    793             u8.xEF_scope = carryQ.BitBlock_advance_ci_co(xEF, carryQ.get_carry_in(5), 5);
    794             u8.scope32 = carryQ.BitBlock_advance_ci_co(u8.prefix3, carryQ.get_carry_in(6), 6);
    795             u8.scope33 = carryQ.BitBlock_advance_ci_co(u8.scope32, carryQ.get_carry_in(7), 7);
    796             u8.scope42 = carryQ.BitBlock_advance_ci_co(u8.prefix4, carryQ.get_carry_in(8), 8);
    797             u8.scope43 = carryQ.BitBlock_advance_ci_co(u8.scope42, carryQ.get_carry_in(9), 9);
    798             u8.scope44 = carryQ.BitBlock_advance_ci_co(u8.scope43, carryQ.get_carry_in(10), 10);
     1375            carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(xE0, carryQ.get_carry_in(1), u8.xE0_scope));
     1376            carryQ.cq[2] = bitblock::srli<127>(pablo_blk_Advance(xED, carryQ.get_carry_in(2), u8.xED_scope));
     1377            carryQ.cq[3] = bitblock::srli<127>(pablo_blk_Advance(xF0, carryQ.get_carry_in(3), u8.xF0_scope));
     1378            carryQ.cq[4] = bitblock::srli<127>(pablo_blk_Advance(xF4, carryQ.get_carry_in(4), u8.xF4_scope));
     1379            carryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(xEF, carryQ.get_carry_in(5), u8.xEF_scope));
     1380            carryQ.cq[6] = bitblock::srli<127>(pablo_blk_Advance(u8.prefix3, carryQ.get_carry_in(6), u8.scope32));
     1381            carryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(u8.scope32, carryQ.get_carry_in(7), u8.scope33));
     1382            carryQ.cq[8] = bitblock::srli<127>(pablo_blk_Advance(u8.prefix4, carryQ.get_carry_in(8), u8.scope42));
     1383            carryQ.cq[9] = bitblock::srli<127>(pablo_blk_Advance(u8.scope42, carryQ.get_carry_in(9), u8.scope43));
     1384            carryQ.cq[10] = bitblock::srli<127>(pablo_blk_Advance(u8.scope43, carryQ.get_carry_in(10), u8.scope44));
    7991385            u8lastscope = simd_or(simd_or(u8.scope22, u8.scope33), u8.scope44);
    8001386            u8anyscope = simd_or(simd_or(simd_or(u8lastscope, u8.scope32), u8.scope42), u8.scope43);
     
    8041390            u8error4 = simd_and(u8.xF4_scope, u8.x90_xBF);
    8051391            u8_error = simd_or(u8_error, simd_or(simd_or(simd_or(u8error1, u8error2), u8error3), u8error4));
    806             EF_BF_pending = carryQ.BitBlock_advance_ci_co(simd_and(u8.xEF_scope, u8.xBF), carryQ.get_carry_in(11), 11);
     1392            carryQ.cq[11] = bitblock::srli<127>(pablo_blk_Advance(simd_and(u8.xEF_scope, u8.xBF), carryQ.get_carry_in(11), EF_BF_pending));
    8071393            u8_FFFE_FFFF = simd_and(EF_BF_pending, simd_or(u8.xBE, u8.xBF));
    8081394          }
     
    8341420                BitBlock u8lastscope, u8error1, u8error2, u8error3, u8error4, EF_BF_pending;
    8351421                BitBlock u8mismatch;
     1422
     1423
    8361424
    8371425
     
    9511539          u8.badprefix = simd_or(temp68, temp71);
    9521540          u8_error = u8.badprefix;
    953           u8.scope22 = carryQ.BitBlock_advance_ci_co(u8.prefix2, carryQ.get_carry_in(0), 0);
     1541          carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(u8.prefix2, carryQ.get_carry_in(0), u8.scope22));
    9541542          u8anyscope = u8.scope22;
    9551543          if ((bitblock::any(simd_or(u8.prefix3, u8.prefix4)) || carryQ.CarryTest(1, 11))) {
     
    9671555            u8.xBF = simd_and(temp73, temp23);
    9681556            u8.xBE = simd_and(temp73, temp15);
    969             u8.xE0_scope = carryQ.BitBlock_advance_ci_co(xE0, carryQ.get_carry_in(1), 1);
    970             u8.xED_scope = carryQ.BitBlock_advance_ci_co(xED, carryQ.get_carry_in(2), 2);
    971             u8.xF0_scope = carryQ.BitBlock_advance_ci_co(xF0, carryQ.get_carry_in(3), 3);
    972             u8.xF4_scope = carryQ.BitBlock_advance_ci_co(xF4, carryQ.get_carry_in(4), 4);
    973             u8.xEF_scope = carryQ.BitBlock_advance_ci_co(xEF, carryQ.get_carry_in(5), 5);
    974             u8.scope32 = carryQ.BitBlock_advance_ci_co(u8.prefix3, carryQ.get_carry_in(6), 6);
    975             u8.scope33 = carryQ.BitBlock_advance_ci_co(u8.scope32, carryQ.get_carry_in(7), 7);
    976             u8.scope42 = carryQ.BitBlock_advance_ci_co(u8.prefix4, carryQ.get_carry_in(8), 8);
    977             u8.scope43 = carryQ.BitBlock_advance_ci_co(u8.scope42, carryQ.get_carry_in(9), 9);
    978             u8.scope44 = carryQ.BitBlock_advance_ci_co(u8.scope43, carryQ.get_carry_in(10), 10);
     1557            carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(xE0, carryQ.get_carry_in(1), u8.xE0_scope));
     1558            carryQ.cq[2] = bitblock::srli<127>(pablo_blk_Advance(xED, carryQ.get_carry_in(2), u8.xED_scope));
     1559            carryQ.cq[3] = bitblock::srli<127>(pablo_blk_Advance(xF0, carryQ.get_carry_in(3), u8.xF0_scope));
     1560            carryQ.cq[4] = bitblock::srli<127>(pablo_blk_Advance(xF4, carryQ.get_carry_in(4), u8.xF4_scope));
     1561            carryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(xEF, carryQ.get_carry_in(5), u8.xEF_scope));
     1562            carryQ.cq[6] = bitblock::srli<127>(pablo_blk_Advance(u8.prefix3, carryQ.get_carry_in(6), u8.scope32));
     1563            carryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(u8.scope32, carryQ.get_carry_in(7), u8.scope33));
     1564            carryQ.cq[8] = bitblock::srli<127>(pablo_blk_Advance(u8.prefix4, carryQ.get_carry_in(8), u8.scope42));
     1565            carryQ.cq[9] = bitblock::srli<127>(pablo_blk_Advance(u8.scope42, carryQ.get_carry_in(9), u8.scope43));
     1566            carryQ.cq[10] = bitblock::srli<127>(pablo_blk_Advance(u8.scope43, carryQ.get_carry_in(10), u8.scope44));
    9791567            u8lastscope = simd_or(simd_or(u8.scope22, u8.scope33), u8.scope44);
    9801568            u8anyscope = simd_or(simd_or(simd_or(u8lastscope, u8.scope32), u8.scope42), u8.scope43);
     
    9841572            u8error4 = simd_and(u8.xF4_scope, u8.x90_xBF);
    9851573            u8_error = simd_or(u8_error, simd_or(simd_or(simd_or(u8error1, u8error2), u8error3), u8error4));
    986             EF_BF_pending = carryQ.BitBlock_advance_ci_co(simd_and(u8.xEF_scope, u8.xBF), carryQ.get_carry_in(11), 11);
     1574            carryQ.cq[11] = bitblock::srli<127>(pablo_blk_Advance(simd_and(u8.xEF_scope, u8.xBF), carryQ.get_carry_in(11), EF_BF_pending));
    9871575            u8_FFFE_FFFF = simd_and(EF_BF_pending, simd_or(u8.xBE, u8.xBF));
    9881576          }
     
    10001588        }
    10011589  }
    1002   void do_segment(Basis_bits basis_bits[], Lex lex[], U8 u8[], int segment_blocks) {
    1003   int i;
    1004   for (i = 0; i < segment_blocks; i++)
    1005     do_block(basis_bits[i], lex[i], u8[i]);
    1006   }
    1007   CarryArray<12> carryQ;
     1590  CarryArray<12, 0> carryQ;
    10081591  };
    10091592
    10101593  struct Parse_CtCDPI {
    10111594  Parse_CtCDPI() {
     1595carryQ.cq[4] = carryQ.carry_flip(carryQ.cq[4]);
    10121596 }
    1013   IDISA_INLINE void do_block(CtCDPI_Callouts & ctCDPI_Callouts, Lex & lex, Scope1 & scope1, Check_streams & check_streams) {
    1014                 BitBlock ctCDPI_error, CtCDPI_starts, Ct_errors, PI_start, CtCD_start;
    1015                 BitBlock CtCDPI_start, DoubleHyphen, PI_end, CtCDPI_Cursor, PI_Cursor;
    1016                 BitBlock CD_Ct_Cursor, CD_Cursor, Ct_Cursor, PI_name_end;
    1017 
    1018 
    1019         ctCDPI_Callouts.CD_end = simd<1>::constant<0>();
     1597  IDISA_INLINE void do_block(Lex & lex, Marker & marker, CtCDPI_Callouts & ctCDPI_Callouts, Check_streams & check_streams) {
     1598                BitBlock CtCDPI_starts, CtCDPI_ends, ctCDPI_mask, v, w, v1, w1, LAngle_scope;
     1599                BitBlock PI_opener, CtCD_opener, CtCDPI_opener, CD_closer, DoubleHyphen;
     1600                BitBlock DoubleRBracket, PI_closer, CtCDPI_Cursor, PI_Cursor, CD_Ct_Cursor;
     1601                BitBlock CD_Cursor, Ct_Cursor, PI_name_end, PI_error, PI_noWS, Ct_error;
     1602
     1603                BitBlock tempvar0, tempvar1, tempvar2, tempvar3, tempvar4, tempvar5;
     1604
     1605
    10201606        ctCDPI_Callouts.Ct_starts = simd<1>::constant<0>();
    10211607        ctCDPI_Callouts.Ct_ends = simd<1>::constant<0>();
     
    10261612        ctCDPI_Callouts.PI_name_ends = simd<1>::constant<0>();
    10271613        ctCDPI_Callouts.PI_ends = simd<1>::constant<0>();
    1028         ctCDPI_Callouts.CtCDPI_mask = simd<1>::constant<0>();
    1029         ctCDPI_error = simd<1>::constant<0>();
    10301614        CtCDPI_starts = simd<1>::constant<0>();
    1031         Ct_errors = simd<1>::constant<0>();
    1032         if ((bitblock::any(lex.RBracket) || carryQ.CarryTest(0, 2))) {
    1033           scope1.RBracket = carryQ.BitBlock_advance_ci_co(lex.RBracket, carryQ.get_carry_in(0), 0);
    1034           ctCDPI_Callouts.CD_end = simd_and(carryQ.BitBlock_advance_ci_co(simd_and(scope1.RBracket, lex.RBracket), carryQ.get_carry_in(1), 1), lex.RAngle);
     1615        CtCDPI_ends = simd<1>::constant<0>();
     1616        ctCDPI_mask = simd<1>::constant<0>();
     1617        v = simd_or(lex.LAngle, lex.Hyphen);
     1618        w = simd_or(lex.Hyphen, lex.QMark);
     1619        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(v, carryQ.get_carry_in(0), v1));
     1620        carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(w, carryQ.get_carry_in(1), w1));
     1621        LAngle_scope = simd_andc(v1, w1);
     1622        PI_opener = simd_and(LAngle_scope, lex.QMark);
     1623        CtCD_opener = simd_and(LAngle_scope, lex.Exclam);
     1624        CtCDPI_opener = simd_or(PI_opener, CtCD_opener);
     1625        CD_closer = simd<1>::constant<0>();
     1626        DoubleHyphen = simd_and(simd_and(v1, w1), lex.Hyphen);
     1627        if ((bitblock::any(lex.RBracket) || carryQ.CarryTest(2, 2))) {
     1628          carryQ.cq[2] = bitblock::srli<127>(pablo_blk_Advance(lex.RBracket, carryQ.get_carry_in(2), tempvar0));
     1629          DoubleRBracket = simd_and(tempvar0, lex.RBracket);
     1630          carryQ.cq[3] = bitblock::srli<127>(pablo_blk_Advance(DoubleRBracket, carryQ.get_carry_in(3), tempvar1));
     1631          CD_closer = simd_and(tempvar1, lex.RAngle);
    10351632        }
    10361633        else {
    1037           carryQ.CarryDequeueEnqueue(0, 2);
    1038         }
    1039         PI_start = simd_and(scope1.LAngle, lex.QMark);
    1040         CtCD_start = simd_and(scope1.LAngle, lex.Exclam);
    1041         CtCDPI_start = simd_or(PI_start, CtCD_start);
    1042         DoubleHyphen = simd_and(scope1.Hyphen, lex.Hyphen);
    1043         PI_end = simd_and(scope1.QMark, lex.RAngle);
    1044         CtCDPI_Cursor = carryQ.BitBlock_scantofirst(CtCDPI_start, carryQ.get_carry_in(2), 2);
    1045         if ((bitblock::any(CtCDPI_Cursor) || carryQ.CarryTest(3, 14))) {
     1634          carryQ.CarryDequeueEnqueue(2, 2);
     1635        }
     1636        PI_closer = simd_and(simd_andc(w1, v1), lex.RAngle);
     1637        carryQ.cq[4] = bitblock::srli<127>(pablo_blk_ScanToFirst(CtCDPI_opener, carryQ.get_carry_in(4), CtCDPI_Cursor));
     1638        if ((bitblock::any(CtCDPI_Cursor) || carryQ.CarryTest(5, 13))) {
    10461639          CtCDPI_starts = simd_or(CtCDPI_starts, CtCDPI_Cursor);
    1047           PI_Cursor = simd_and(CtCDPI_Cursor, PI_start);
    1048           CD_Ct_Cursor = carryQ.BitBlock_advance_ci_co(simd_andc(CtCDPI_Cursor, PI_Cursor), carryQ.get_carry_in(3), 3);
     1640          PI_Cursor = simd_and(CtCDPI_Cursor, PI_opener);
     1641          carryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(simd_andc(CtCDPI_Cursor, PI_Cursor), carryQ.get_carry_in(5), CD_Ct_Cursor));
    10491642          CD_Cursor = simd_and(CD_Ct_Cursor, lex.LBracket);
    10501643          Ct_Cursor = simd_and(CD_Ct_Cursor, lex.Hyphen);
    1051           ctCDPI_Callouts.PI_starts = simd_or(ctCDPI_Callouts.PI_starts, PI_Cursor);
    1052           ctCDPI_Callouts.CD_starts = simd_or(ctCDPI_Callouts.CD_starts, CD_Cursor);
    1053           ctCDPI_Callouts.Ct_starts = simd_or(ctCDPI_Callouts.Ct_starts, Ct_Cursor);
    1054           Ct_Cursor = carryQ.BitBlock_advance_ci_co(Ct_Cursor, carryQ.get_carry_in(4), 4);
    1055           Ct_errors = simd_or(Ct_errors, simd_andc(Ct_Cursor, lex.Hyphen));
    1056           Ct_Cursor = carryQ.BitBlock_advance_ci_co(carryQ.BitBlock_advance_ci_co(Ct_Cursor, carryQ.get_carry_in(5), 5), carryQ.get_carry_in(6), 6);
    1057           PI_Cursor = carryQ.BitBlock_advance_ci_co(PI_Cursor, carryQ.get_carry_in(7), 7);
    1058           ctCDPI_Callouts.PI_name_starts = simd_or(ctCDPI_Callouts.PI_name_starts, PI_Cursor);
    1059           PI_name_end = carryQ.BitBlock_scanthru_ci_co(PI_Cursor, lex.NameScan, carryQ.get_carry_in(8), 8);
    1060           ctCDPI_Callouts.PI_name_ends = simd_or(ctCDPI_Callouts.PI_name_ends, PI_name_end);
    1061           PI_Cursor = carryQ.BitBlock_scanthru_ci_co(PI_name_end, simd_not(PI_end), carryQ.get_carry_in(9), 9);
    1062           CD_Cursor = carryQ.BitBlock_scanthru_ci_co(CD_Cursor, simd_not(ctCDPI_Callouts.CD_end), carryQ.get_carry_in(10), 10);
    1063           Ct_Cursor = carryQ.BitBlock_advance_ci_co(carryQ.BitBlock_scanthru_ci_co(Ct_Cursor, simd_not(DoubleHyphen), carryQ.get_carry_in(11), 11), carryQ.get_carry_in(12), 12);
    1064           ctCDPI_Callouts.PI_ends = simd_or(ctCDPI_Callouts.PI_ends, PI_Cursor);
    1065           ctCDPI_Callouts.CD_ends = simd_or(ctCDPI_Callouts.CD_ends, CD_Cursor);
    1066           ctCDPI_Callouts.Ct_ends = simd_or(ctCDPI_Callouts.Ct_ends, Ct_Cursor);
     1644          if ((bitblock::any(PI_Cursor) || carryQ.CarryTest(6, 4))) {
     1645            ctCDPI_Callouts.PI_starts = simd_or(ctCDPI_Callouts.PI_starts, PI_Cursor);
     1646            carryQ.cq[6] = bitblock::srli<127>(pablo_blk_Advance(PI_Cursor, carryQ.get_carry_in(6), PI_Cursor));
     1647            ctCDPI_Callouts.PI_name_starts = simd_or(ctCDPI_Callouts.PI_name_starts, PI_Cursor);
     1648            carryQ.cq[7] = bitblock::srli<127>(pablo_blk_ScanThru(PI_Cursor, lex.NameScan, carryQ.get_carry_in(7), PI_name_end));
     1649            PI_error = simd_and(PI_Cursor, PI_name_end);
     1650            PI_noWS = simd_andc(PI_name_end, lex.WS);
     1651            carryQ.cq[8] = bitblock::srli<127>(pablo_blk_Advance(PI_noWS, carryQ.get_carry_in(8), tempvar2));
     1652            PI_error = simd_or(PI_error, simd_or(simd_andc(PI_noWS, lex.QMark), simd_andc(tempvar2, PI_closer)));
     1653            if (bitblock::any(PI_error)) {
     1654              assert_0_error("Error in PI syntax", PI_error);
     1655            }
     1656            ctCDPI_Callouts.PI_name_ends = simd_or(ctCDPI_Callouts.PI_name_ends, PI_name_end);
     1657            carryQ.cq[9] = bitblock::srli<127>(pablo_blk_ScanThru(PI_name_end, simd_not(PI_closer), carryQ.get_carry_in(9), PI_Cursor));
     1658            ctCDPI_Callouts.PI_ends = simd_or(ctCDPI_Callouts.PI_ends, PI_Cursor);
     1659            CtCDPI_ends = simd_or(CtCDPI_ends, PI_Cursor);
     1660          }
     1661          else {
     1662            carryQ.CarryDequeueEnqueue(6, 4);
     1663          }
     1664          if ((bitblock::any(CD_Cursor) || carryQ.CarryTest(10, 1))) {
     1665            ctCDPI_Callouts.CD_starts = simd_or(ctCDPI_Callouts.CD_starts, CD_Cursor);
     1666            carryQ.cq[10] = bitblock::srli<127>(pablo_blk_ScanThru(CD_Cursor, simd_not(CD_closer), carryQ.get_carry_in(10), CD_Cursor));
     1667            ctCDPI_Callouts.CD_ends = simd_or(ctCDPI_Callouts.CD_ends, CD_Cursor);
     1668            CtCDPI_ends = simd_or(CtCDPI_ends, CD_Cursor);
     1669          }
     1670          else {
     1671            carryQ.CarryDequeueEnqueue(10, 1);
     1672          }
     1673          if ((bitblock::any(Ct_Cursor) || carryQ.CarryTest(11, 5))) {
     1674            ctCDPI_Callouts.Ct_starts = simd_or(ctCDPI_Callouts.Ct_starts, Ct_Cursor);
     1675            carryQ.cq[11] = bitblock::srli<127>(pablo_blk_Advance(Ct_Cursor, carryQ.get_carry_in(11), Ct_Cursor));
     1676            Ct_error = simd_andc(Ct_Cursor, lex.Hyphen);
     1677            carryQ.cq[12] = bitblock::srli<127>(pablo_blk_Advance(Ct_Cursor, carryQ.get_carry_in(12), tempvar3));
     1678            carryQ.cq[13] = bitblock::srli<127>(pablo_blk_Advance(tempvar3, carryQ.get_carry_in(13), Ct_Cursor));
     1679            carryQ.cq[14] = bitblock::srli<127>(pablo_blk_ScanThru(Ct_Cursor, simd_not(DoubleHyphen), carryQ.get_carry_in(14), tempvar4));
     1680            carryQ.cq[15] = bitblock::srli<127>(pablo_blk_Advance(tempvar4, carryQ.get_carry_in(15), Ct_Cursor));
     1681            if (bitblock::any(simd_or(Ct_error, simd_andc(Ct_Cursor, lex.RAngle)))) {
     1682              assert_0_error("Error in comment syntax", simd_or(Ct_error, simd_andc(Ct_Cursor, lex.RAngle)));
     1683            }
     1684            ctCDPI_Callouts.Ct_ends = simd_or(ctCDPI_Callouts.Ct_ends, Ct_Cursor);
     1685            CtCDPI_ends = simd_or(CtCDPI_ends, Ct_Cursor);
     1686          }
     1687          else {
     1688            carryQ.CarryDequeueEnqueue(11, 5);
     1689          }
    10671690          CtCDPI_Cursor = simd_or(simd_or(PI_Cursor, CD_Cursor), Ct_Cursor);
    1068           CtCDPI_Cursor = carryQ.BitBlock_scanthru_ci_co(CtCDPI_Cursor, simd_not(CtCDPI_start), carryQ.get_carry_in(13), 13);
    1069           ctCDPI_Callouts.CtCDPI_mask = carryQ.BitBlock_sub_ci_co(carryQ.BitBlock_advance_ci_co(simd_or(simd_or(ctCDPI_Callouts.CD_ends, ctCDPI_Callouts.Ct_ends), ctCDPI_Callouts.PI_ends), carryQ.get_carry_in(14), 14), CtCDPI_starts, carryQ.get_carry_in(15), 15);
    1070           ctCDPI_error = simd_or(Ct_errors, simd_andc(ctCDPI_Callouts.Ct_ends, lex.RAngle));
    1071           ctCDPI_error = simd_or(ctCDPI_error, simd_andc(carryQ.BitBlock_advance_ci_co(simd_andc(ctCDPI_Callouts.PI_name_ends, lex.WS), carryQ.get_carry_in(16), 16), PI_end));
    1072           ctCDPI_error = simd_or(ctCDPI_error, simd_and(ctCDPI_Callouts.PI_name_starts, ctCDPI_Callouts.PI_name_ends));
    1073           ctCDPI_error = simd_or(ctCDPI_error, simd_andc(ctCDPI_Callouts.CtCDPI_mask, EOF_mask));
     1691          carryQ.cq[16] = bitblock::srli<127>(pablo_blk_InclusiveSpan(CtCDPI_starts, CtCDPI_ends, carryQ.get_carry_in(16), ctCDPI_mask));
     1692          if (bitblock::any(simd<1>::constant<0>())) {
     1693            assert_0_error("Error in comment, CDATA or processing instruction syntax", simd<1>::constant<0>());
     1694          }
     1695          carryQ.cq[17] = bitblock::srli<127>(pablo_blk_ScanThru(CtCDPI_Cursor, simd_not(CtCDPI_opener), carryQ.get_carry_in(17), CtCDPI_Cursor));
    10741696          while (bitblock::any(CtCDPI_Cursor)) {
    1075             LocalCarryDeclare(subcarryQ, 14);
     1697            LocalCarryDeclare(subcarryQ, 13);
    10761698            CtCDPI_starts = simd_or(CtCDPI_starts, CtCDPI_Cursor);
    1077             PI_Cursor = simd_and(CtCDPI_Cursor, PI_start);
    1078             CD_Ct_Cursor = subcarryQ.BitBlock_advance_ci_co(simd_andc(CtCDPI_Cursor, PI_Cursor), simd<1>::constant<0>(), 0);
     1699            PI_Cursor = simd_and(CtCDPI_Cursor, PI_opener);
     1700            subcarryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(simd_andc(CtCDPI_Cursor, PI_Cursor), simd<1>::constant<0>(), CD_Ct_Cursor));
    10791701            CD_Cursor = simd_and(CD_Ct_Cursor, lex.LBracket);
    10801702            Ct_Cursor = simd_and(CD_Ct_Cursor, lex.Hyphen);
    1081             ctCDPI_Callouts.PI_starts = simd_or(ctCDPI_Callouts.PI_starts, PI_Cursor);
    1082             ctCDPI_Callouts.CD_starts = simd_or(ctCDPI_Callouts.CD_starts, CD_Cursor);
    1083             ctCDPI_Callouts.Ct_starts = simd_or(ctCDPI_Callouts.Ct_starts, Ct_Cursor);
    1084             Ct_Cursor = subcarryQ.BitBlock_advance_ci_co(Ct_Cursor, simd<1>::constant<0>(), 1);
    1085             Ct_errors = simd_or(Ct_errors, simd_andc(Ct_Cursor, lex.Hyphen));
    1086             Ct_Cursor = subcarryQ.BitBlock_advance_ci_co(subcarryQ.BitBlock_advance_ci_co(Ct_Cursor, simd<1>::constant<0>(), 2), simd<1>::constant<0>(), 3);
    1087             PI_Cursor = subcarryQ.BitBlock_advance_ci_co(PI_Cursor, simd<1>::constant<0>(), 4);
    1088             ctCDPI_Callouts.PI_name_starts = simd_or(ctCDPI_Callouts.PI_name_starts, PI_Cursor);
    1089             PI_name_end = subcarryQ.BitBlock_scanthru_ci_co(PI_Cursor, lex.NameScan, simd<1>::constant<0>(), 5);
    1090             ctCDPI_Callouts.PI_name_ends = simd_or(ctCDPI_Callouts.PI_name_ends, PI_name_end);
    1091             PI_Cursor = subcarryQ.BitBlock_scanthru_ci_co(PI_name_end, simd_not(PI_end), simd<1>::constant<0>(), 6);
    1092             CD_Cursor = subcarryQ.BitBlock_scanthru_ci_co(CD_Cursor, simd_not(ctCDPI_Callouts.CD_end), simd<1>::constant<0>(), 7);
    1093             Ct_Cursor = subcarryQ.BitBlock_advance_ci_co(subcarryQ.BitBlock_scanthru_ci_co(Ct_Cursor, simd_not(DoubleHyphen), simd<1>::constant<0>(), 8), simd<1>::constant<0>(), 9);
    1094             ctCDPI_Callouts.PI_ends = simd_or(ctCDPI_Callouts.PI_ends, PI_Cursor);
    1095             ctCDPI_Callouts.CD_ends = simd_or(ctCDPI_Callouts.CD_ends, CD_Cursor);
    1096             ctCDPI_Callouts.Ct_ends = simd_or(ctCDPI_Callouts.Ct_ends, Ct_Cursor);
     1703            if (bitblock::any(PI_Cursor)) {
     1704              ctCDPI_Callouts.PI_starts = simd_or(ctCDPI_Callouts.PI_starts, PI_Cursor);
     1705              subcarryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(PI_Cursor, simd<1>::constant<0>(), PI_Cursor));
     1706              ctCDPI_Callouts.PI_name_starts = simd_or(ctCDPI_Callouts.PI_name_starts, PI_Cursor);
     1707              subcarryQ.cq[2] = bitblock::srli<127>(pablo_blk_ScanThru(PI_Cursor, lex.NameScan, simd<1>::constant<0>(), PI_name_end));
     1708              PI_error = simd_and(PI_Cursor, PI_name_end);
     1709              PI_noWS = simd_andc(PI_name_end, lex.WS);
     1710              subcarryQ.cq[3] = bitblock::srli<127>(pablo_blk_Advance(PI_noWS, simd<1>::constant<0>(), tempvar2));
     1711              PI_error = simd_or(PI_error, simd_or(simd_andc(PI_noWS, lex.QMark), simd_andc(tempvar2, PI_closer)));
     1712              if (bitblock::any(PI_error)) {
     1713                assert_0_error("Error in PI syntax", PI_error);
     1714              }
     1715              ctCDPI_Callouts.PI_name_ends = simd_or(ctCDPI_Callouts.PI_name_ends, PI_name_end);
     1716              subcarryQ.cq[4] = bitblock::srli<127>(pablo_blk_ScanThru(PI_name_end, simd_not(PI_closer), simd<1>::constant<0>(), PI_Cursor));
     1717              ctCDPI_Callouts.PI_ends = simd_or(ctCDPI_Callouts.PI_ends, PI_Cursor);
     1718              CtCDPI_ends = simd_or(CtCDPI_ends, PI_Cursor);
     1719            }
     1720            else {
     1721              subcarryQ.CarryDequeueEnqueue(6, 4);
     1722            }
     1723            if (bitblock::any(CD_Cursor)) {
     1724              ctCDPI_Callouts.CD_starts = simd_or(ctCDPI_Callouts.CD_starts, CD_Cursor);
     1725              subcarryQ.cq[5] = bitblock::srli<127>(pablo_blk_ScanThru(CD_Cursor, simd_not(CD_closer), simd<1>::constant<0>(), CD_Cursor));
     1726              ctCDPI_Callouts.CD_ends = simd_or(ctCDPI_Callouts.CD_ends, CD_Cursor);
     1727              CtCDPI_ends = simd_or(CtCDPI_ends, CD_Cursor);
     1728            }
     1729            else {
     1730              subcarryQ.CarryDequeueEnqueue(10, 1);
     1731            }
     1732            if (bitblock::any(Ct_Cursor)) {
     1733              ctCDPI_Callouts.Ct_starts = simd_or(ctCDPI_Callouts.Ct_starts, Ct_Cursor);
     1734              subcarryQ.cq[6] = bitblock::srli<127>(pablo_blk_Advance(Ct_Cursor, simd<1>::constant<0>(), Ct_Cursor));
     1735              Ct_error = simd_andc(Ct_Cursor, lex.Hyphen);
     1736              subcarryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(Ct_Cursor, simd<1>::constant<0>(), tempvar3));
     1737              subcarryQ.cq[8] = bitblock::srli<127>(pablo_blk_Advance(tempvar3, simd<1>::constant<0>(), Ct_Cursor));
     1738              subcarryQ.cq[9] = bitblock::srli<127>(pablo_blk_ScanThru(Ct_Cursor, simd_not(DoubleHyphen), simd<1>::constant<0>(), tempvar4));
     1739              subcarryQ.cq[10] = bitblock::srli<127>(pablo_blk_Advance(tempvar4, simd<1>::constant<0>(), Ct_Cursor));
     1740              if (bitblock::any(simd_or(Ct_error, simd_andc(Ct_Cursor, lex.RAngle)))) {
     1741                assert_0_error("Error in comment syntax", simd_or(Ct_error, simd_andc(Ct_Cursor, lex.RAngle)));
     1742              }
     1743              ctCDPI_Callouts.Ct_ends = simd_or(ctCDPI_Callouts.Ct_ends, Ct_Cursor);
     1744              CtCDPI_ends = simd_or(CtCDPI_ends, Ct_Cursor);
     1745            }
     1746            else {
     1747              subcarryQ.CarryDequeueEnqueue(11, 5);
     1748            }
    10971749            CtCDPI_Cursor = simd_or(simd_or(PI_Cursor, CD_Cursor), Ct_Cursor);
    1098             CtCDPI_Cursor = subcarryQ.BitBlock_scanthru_ci_co(CtCDPI_Cursor, simd_not(CtCDPI_start), simd<1>::constant<0>(), 10);
    1099             ctCDPI_Callouts.CtCDPI_mask = subcarryQ.BitBlock_sub_ci_co(subcarryQ.BitBlock_advance_ci_co(simd_or(simd_or(ctCDPI_Callouts.CD_ends, ctCDPI_Callouts.Ct_ends), ctCDPI_Callouts.PI_ends), simd<1>::constant<0>(), 11), CtCDPI_starts, simd<1>::constant<0>(), 12);
    1100             ctCDPI_error = simd_or(Ct_errors, simd_andc(ctCDPI_Callouts.Ct_ends, lex.RAngle));
    1101             ctCDPI_error = simd_or(ctCDPI_error, simd_andc(subcarryQ.BitBlock_advance_ci_co(simd_andc(ctCDPI_Callouts.PI_name_ends, lex.WS), simd<1>::constant<0>(), 13), PI_end));
    1102             ctCDPI_error = simd_or(ctCDPI_error, simd_and(ctCDPI_Callouts.PI_name_starts, ctCDPI_Callouts.PI_name_ends));
    1103             ctCDPI_error = simd_or(ctCDPI_error, simd_andc(ctCDPI_Callouts.CtCDPI_mask, EOF_mask));
    1104             carryQ.CarryCombine((ICarryQueue *) &subcarryQ, 3, 14);
     1750            subcarryQ.cq[11] = bitblock::srli<127>(pablo_blk_InclusiveSpan(CtCDPI_starts, CtCDPI_ends, simd<1>::constant<0>(), ctCDPI_mask));
     1751            if (bitblock::any(simd<1>::constant<0>())) {
     1752              assert_0_error("Error in comment, CDATA or processing instruction syntax", simd<1>::constant<0>());
     1753            }
     1754            subcarryQ.cq[12] = bitblock::srli<127>(pablo_blk_ScanThru(CtCDPI_Cursor, simd_not(CtCDPI_opener), simd<1>::constant<0>(), CtCDPI_Cursor));
     1755            carryQ.CarryCombine(subcarryQ.cq, 5, 13);
    11051756          }
    11061757        }
    11071758        else {
    1108           carryQ.CarryDequeueEnqueue(3, 14);
    1109         }
    1110         if (bitblock::any(ctCDPI_error)) {
    1111           error_tracker.NoteError("Error in comment, CDATA or processing instruction syntax", ctCDPI_error);
    1112         }
    1113         check_streams.misc_mask = simd_and(simd_or(simd_or(simd_or(lex.WS, lex.LAngle), carryQ.BitBlock_sub_ci_co(carryQ.BitBlock_advance_ci_co(simd_or(ctCDPI_Callouts.Ct_ends, ctCDPI_Callouts.PI_ends), carryQ.get_carry_in(17), 17), simd_or(ctCDPI_Callouts.Ct_starts, ctCDPI_Callouts.PI_starts), carryQ.get_carry_in(18), 18)), CtCDPI_starts), EOF_mask);
    1114         carryQ.CarryQ_Adjust(19);
     1759          carryQ.CarryDequeueEnqueue(5, 13);
     1760        }
     1761        carryQ.cq[18] = bitblock::srli<127>(pablo_blk_InclusiveSpan(simd_or(ctCDPI_Callouts.Ct_starts, ctCDPI_Callouts.PI_starts), simd_or(ctCDPI_Callouts.Ct_ends, ctCDPI_Callouts.PI_ends), carryQ.get_carry_in(18), tempvar5));
     1762        check_streams.misc_mask = simd_and(simd_or(simd_or(simd_or(lex.WS, lex.LAngle), tempvar5), CtCDPI_starts), EOF_mask);
     1763        marker.LAngle_scope = simd_andc(LAngle_scope, ctCDPI_mask);
     1764        marker.Ref_opener = simd_andc(lex.RefStart, ctCDPI_mask);
     1765        marker.CD_closer = simd_andc(CD_closer, ctCDPI_mask);
     1766        carryQ.CarryQ_Adjust(17);
    11151767  }
    1116   void do_final_block(CtCDPI_Callouts & ctCDPI_Callouts, Lex & lex, Scope1 & scope1, Check_streams & check_streams, BitBlock EOF_mask) {
    1117                 BitBlock ctCDPI_error, CtCDPI_starts, Ct_errors, PI_start, CtCD_start;
    1118                 BitBlock CtCDPI_start, DoubleHyphen, PI_end, CtCDPI_Cursor, PI_Cursor;
    1119                 BitBlock CD_Ct_Cursor, CD_Cursor, Ct_Cursor, PI_name_end;
    1120 
    1121 
    1122         ctCDPI_Callouts.CD_end = simd<1>::constant<0>();
     1768  void do_final_block(Lex & lex, Marker & marker, CtCDPI_Callouts & ctCDPI_Callouts, Check_streams & check_streams, BitBlock EOF_mask) {
     1769                BitBlock CtCDPI_starts, CtCDPI_ends, ctCDPI_mask, v, w, v1, w1, LAngle_scope;
     1770                BitBlock PI_opener, CtCD_opener, CtCDPI_opener, CD_closer, DoubleHyphen;
     1771                BitBlock DoubleRBracket, PI_closer, CtCDPI_Cursor, PI_Cursor, CD_Ct_Cursor;
     1772                BitBlock CD_Cursor, Ct_Cursor, PI_name_end, PI_error, PI_noWS, Ct_error;
     1773
     1774                BitBlock tempvar0, tempvar1, tempvar2, tempvar3, tempvar4, tempvar5;
     1775
     1776
    11231777        ctCDPI_Callouts.Ct_starts = simd<1>::constant<0>();
    11241778        ctCDPI_Callouts.Ct_ends = simd<1>::constant<0>();
     
    11291783        ctCDPI_Callouts.PI_name_ends = simd<1>::constant<0>();
    11301784        ctCDPI_Callouts.PI_ends = simd<1>::constant<0>();
    1131         ctCDPI_Callouts.CtCDPI_mask = simd<1>::constant<0>();
    1132         ctCDPI_error = simd<1>::constant<0>();
    11331785        CtCDPI_starts = simd<1>::constant<0>();
    1134         Ct_errors = simd<1>::constant<0>();
    1135         if ((bitblock::any(lex.RBracket) || carryQ.CarryTest(0, 2))) {
    1136           scope1.RBracket = carryQ.BitBlock_advance_ci_co(lex.RBracket, carryQ.get_carry_in(0), 0);
    1137           ctCDPI_Callouts.CD_end = simd_and(carryQ.BitBlock_advance_ci_co(simd_and(scope1.RBracket, lex.RBracket), carryQ.get_carry_in(1), 1), lex.RAngle);
     1786        CtCDPI_ends = simd<1>::constant<0>();
     1787        ctCDPI_mask = simd<1>::constant<0>();
     1788        v = simd_or(lex.LAngle, lex.Hyphen);
     1789        w = simd_or(lex.Hyphen, lex.QMark);
     1790        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(v, carryQ.get_carry_in(0), v1));
     1791        carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(w, carryQ.get_carry_in(1), w1));
     1792        LAngle_scope = simd_andc(v1, w1);
     1793        PI_opener = simd_and(LAngle_scope, lex.QMark);
     1794        CtCD_opener = simd_and(LAngle_scope, lex.Exclam);
     1795        CtCDPI_opener = simd_or(PI_opener, CtCD_opener);
     1796        CD_closer = simd<1>::constant<0>();
     1797        DoubleHyphen = simd_and(simd_and(v1, w1), lex.Hyphen);
     1798        if ((bitblock::any(lex.RBracket) || carryQ.CarryTest(2, 2))) {
     1799          carryQ.cq[2] = bitblock::srli<127>(pablo_blk_Advance(lex.RBracket, carryQ.get_carry_in(2), tempvar0));
     1800          DoubleRBracket = simd_and(tempvar0, lex.RBracket);
     1801          carryQ.cq[3] = bitblock::srli<127>(pablo_blk_Advance(DoubleRBracket, carryQ.get_carry_in(3), tempvar1));
     1802          CD_closer = simd_and(tempvar1, lex.RAngle);
    11381803        }
    11391804        else {
    1140           carryQ.CarryDequeueEnqueue(0, 2);
    1141         }
    1142         PI_start = simd_and(scope1.LAngle, lex.QMark);
    1143         CtCD_start = simd_and(scope1.LAngle, lex.Exclam);
    1144         CtCDPI_start = simd_or(PI_start, CtCD_start);
    1145         DoubleHyphen = simd_and(scope1.Hyphen, lex.Hyphen);
    1146         PI_end = simd_and(scope1.QMark, lex.RAngle);
    1147         CtCDPI_Cursor = carryQ.BitBlock_scantofirst(CtCDPI_start, carryQ.get_carry_in(2), 2);
    1148         if ((bitblock::any(simd_and(CtCDPI_Cursor, EOF_mask)) || carryQ.CarryTest(3, 14))) {
     1805          carryQ.CarryDequeueEnqueue(2, 2);
     1806        }
     1807        PI_closer = simd_and(simd_andc(w1, v1), lex.RAngle);
     1808        carryQ.cq[4] = bitblock::srli<127>(pablo_blk_ScanToFirst(CtCDPI_opener, carryQ.get_carry_in(4), CtCDPI_Cursor));
     1809        if ((bitblock::any(simd_and(CtCDPI_Cursor, EOF_mask)) || carryQ.CarryTest(5, 13))) {
    11491810          CtCDPI_starts = simd_or(CtCDPI_starts, CtCDPI_Cursor);
    1150           PI_Cursor = simd_and(CtCDPI_Cursor, PI_start);
    1151           CD_Ct_Cursor = carryQ.BitBlock_advance_ci_co(simd_andc(CtCDPI_Cursor, PI_Cursor), carryQ.get_carry_in(3), 3);
     1811          PI_Cursor = simd_and(CtCDPI_Cursor, PI_opener);
     1812          carryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(simd_andc(CtCDPI_Cursor, PI_Cursor), carryQ.get_carry_in(5), CD_Ct_Cursor));
    11521813          CD_Cursor = simd_and(CD_Ct_Cursor, lex.LBracket);
    11531814          Ct_Cursor = simd_and(CD_Ct_Cursor, lex.Hyphen);
    1154           ctCDPI_Callouts.PI_starts = simd_or(ctCDPI_Callouts.PI_starts, PI_Cursor);
    1155           ctCDPI_Callouts.CD_starts = simd_or(ctCDPI_Callouts.CD_starts, CD_Cursor);
    1156           ctCDPI_Callouts.Ct_starts = simd_or(ctCDPI_Callouts.Ct_starts, Ct_Cursor);
    1157           Ct_Cursor = carryQ.BitBlock_advance_ci_co(Ct_Cursor, carryQ.get_carry_in(4), 4);
    1158           Ct_errors = simd_or(Ct_errors, simd_andc(Ct_Cursor, lex.Hyphen));
    1159           Ct_Cursor = carryQ.BitBlock_advance_ci_co(carryQ.BitBlock_advance_ci_co(Ct_Cursor, carryQ.get_carry_in(5), 5), carryQ.get_carry_in(6), 6);
    1160           PI_Cursor = carryQ.BitBlock_advance_ci_co(PI_Cursor, carryQ.get_carry_in(7), 7);
    1161           ctCDPI_Callouts.PI_name_starts = simd_or(ctCDPI_Callouts.PI_name_starts, PI_Cursor);
    1162           PI_name_end = carryQ.BitBlock_scanthru_ci_co(PI_Cursor, lex.NameScan, carryQ.get_carry_in(8), 8);
    1163           ctCDPI_Callouts.PI_name_ends = simd_or(ctCDPI_Callouts.PI_name_ends, PI_name_end);
    1164           PI_Cursor = carryQ.BitBlock_scanthru_ci_co(PI_name_end, simd_andc(EOF_mask, PI_end), carryQ.get_carry_in(9), 9);
    1165           CD_Cursor = carryQ.BitBlock_scanthru_ci_co(CD_Cursor, simd_andc(EOF_mask, ctCDPI_Callouts.CD_end), carryQ.get_carry_in(10), 10);
    1166           Ct_Cursor = carryQ.BitBlock_advance_ci_co(carryQ.BitBlock_scanthru_ci_co(Ct_Cursor, simd_andc(EOF_mask, DoubleHyphen), carryQ.get_carry_in(11), 11), carryQ.get_carry_in(12), 12);
    1167           ctCDPI_Callouts.PI_ends = simd_or(ctCDPI_Callouts.PI_ends, PI_Cursor);
    1168           ctCDPI_Callouts.CD_ends = simd_or(ctCDPI_Callouts.CD_ends, CD_Cursor);
    1169           ctCDPI_Callouts.Ct_ends = simd_or(ctCDPI_Callouts.Ct_ends, Ct_Cursor);
     1815          if ((bitblock::any(PI_Cursor) || carryQ.CarryTest(6, 4))) {
     1816            ctCDPI_Callouts.PI_starts = simd_or(ctCDPI_Callouts.PI_starts, PI_Cursor);
     1817            carryQ.cq[6] = bitblock::srli<127>(pablo_blk_Advance(PI_Cursor, carryQ.get_carry_in(6), PI_Cursor));
     1818            ctCDPI_Callouts.PI_name_starts = simd_or(ctCDPI_Callouts.PI_name_starts, PI_Cursor);
     1819            carryQ.cq[7] = bitblock::srli<127>(pablo_blk_ScanThru(PI_Cursor, lex.NameScan, carryQ.get_carry_in(7), PI_name_end));
     1820            PI_error = simd_and(PI_Cursor, PI_name_end);
     1821            PI_noWS = simd_andc(PI_name_end, lex.WS);
     1822            carryQ.cq[8] = bitblock::srli<127>(pablo_blk_Advance(PI_noWS, carryQ.get_carry_in(8), tempvar2));
     1823            PI_error = simd_or(PI_error, simd_or(simd_andc(PI_noWS, lex.QMark), simd_andc(tempvar2, PI_closer)));
     1824            if (bitblock::any(PI_error)) {
     1825              assert_0_error("Error in PI syntax", PI_error);
     1826            }
     1827            ctCDPI_Callouts.PI_name_ends = simd_or(ctCDPI_Callouts.PI_name_ends, PI_name_end);
     1828            carryQ.cq[9] = bitblock::srli<127>(pablo_blk_ScanThru(PI_name_end, simd_andc(EOF_mask, PI_closer), carryQ.get_carry_in(9), PI_Cursor));
     1829            ctCDPI_Callouts.PI_ends = simd_or(ctCDPI_Callouts.PI_ends, PI_Cursor);
     1830            CtCDPI_ends = simd_or(CtCDPI_ends, PI_Cursor);
     1831          }
     1832          else {
     1833            carryQ.CarryDequeueEnqueue(6, 4);
     1834          }
     1835          if ((bitblock::any(CD_Cursor) || carryQ.CarryTest(10, 1))) {
     1836            ctCDPI_Callouts.CD_starts = simd_or(ctCDPI_Callouts.CD_starts, CD_Cursor);
     1837            carryQ.cq[10] = bitblock::srli<127>(pablo_blk_ScanThru(CD_Cursor, simd_andc(EOF_mask, CD_closer), carryQ.get_carry_in(10), CD_Cursor));
     1838            ctCDPI_Callouts.CD_ends = simd_or(ctCDPI_Callouts.CD_ends, CD_Cursor);
     1839            CtCDPI_ends = simd_or(CtCDPI_ends, CD_Cursor);
     1840          }
     1841          else {
     1842            carryQ.CarryDequeueEnqueue(10, 1);
     1843          }
     1844          if ((bitblock::any(Ct_Cursor) || carryQ.CarryTest(11, 5))) {
     1845            ctCDPI_Callouts.Ct_starts = simd_or(ctCDPI_Callouts.Ct_starts, Ct_Cursor);
     1846            carryQ.cq[11] = bitblock::srli<127>(pablo_blk_Advance(Ct_Cursor, carryQ.get_carry_in(11), Ct_Cursor));
     1847            Ct_error = simd_andc(Ct_Cursor, lex.Hyphen);
     1848            carryQ.cq[12] = bitblock::srli<127>(pablo_blk_Advance(Ct_Cursor, carryQ.get_carry_in(12), tempvar3));
     1849            carryQ.cq[13] = bitblock::srli<127>(pablo_blk_Advance(tempvar3, carryQ.get_carry_in(13), Ct_Cursor));
     1850            carryQ.cq[14] = bitblock::srli<127>(pablo_blk_ScanThru(Ct_Cursor, simd_andc(EOF_mask, DoubleHyphen), carryQ.get_carry_in(14), tempvar4));
     1851            carryQ.cq[15] = bitblock::srli<127>(pablo_blk_Advance(tempvar4, carryQ.get_carry_in(15), Ct_Cursor));
     1852            if (bitblock::any(simd_or(Ct_error, simd_andc(Ct_Cursor, lex.RAngle)))) {
     1853              assert_0_error("Error in comment syntax", simd_or(Ct_error, simd_andc(Ct_Cursor, lex.RAngle)));
     1854            }
     1855            ctCDPI_Callouts.Ct_ends = simd_or(ctCDPI_Callouts.Ct_ends, Ct_Cursor);
     1856            CtCDPI_ends = simd_or(CtCDPI_ends, Ct_Cursor);
     1857          }
     1858          else {
     1859            carryQ.CarryDequeueEnqueue(11, 5);
     1860          }
    11701861          CtCDPI_Cursor = simd_or(simd_or(PI_Cursor, CD_Cursor), Ct_Cursor);
    1171           CtCDPI_Cursor = carryQ.BitBlock_scanthru_ci_co(CtCDPI_Cursor, simd_andc(EOF_mask, CtCDPI_start), carryQ.get_carry_in(13), 13);
    1172           ctCDPI_Callouts.CtCDPI_mask = carryQ.BitBlock_sub_ci_co(carryQ.BitBlock_advance_ci_co(simd_or(simd_or(ctCDPI_Callouts.CD_ends, ctCDPI_Callouts.Ct_ends), ctCDPI_Callouts.PI_ends), carryQ.get_carry_in(14), 14), CtCDPI_starts, carryQ.get_carry_in(15), 15);
    1173           ctCDPI_error = simd_or(Ct_errors, simd_andc(ctCDPI_Callouts.Ct_ends, lex.RAngle));
    1174           ctCDPI_error = simd_or(ctCDPI_error, simd_andc(carryQ.BitBlock_advance_ci_co(simd_andc(ctCDPI_Callouts.PI_name_ends, lex.WS), carryQ.get_carry_in(16), 16), PI_end));
    1175           ctCDPI_error = simd_or(ctCDPI_error, simd_and(ctCDPI_Callouts.PI_name_starts, ctCDPI_Callouts.PI_name_ends));
    1176           ctCDPI_error = simd_or(ctCDPI_error, simd_andc(ctCDPI_Callouts.CtCDPI_mask, EOF_mask));
     1862          carryQ.cq[16] = bitblock::srli<127>(pablo_blk_InclusiveSpan(CtCDPI_starts, CtCDPI_ends, carryQ.get_carry_in(16), ctCDPI_mask));
     1863          if (bitblock::any(simd_andc(ctCDPI_mask, EOF_mask))) {
     1864            assert_0_error("Error in comment, CDATA or processing instruction syntax", simd_andc(ctCDPI_mask, EOF_mask));
     1865          }
     1866          carryQ.cq[17] = bitblock::srli<127>(pablo_blk_ScanThru(CtCDPI_Cursor, simd_andc(EOF_mask, CtCDPI_opener), carryQ.get_carry_in(17), CtCDPI_Cursor));
    11771867          while (bitblock::any(simd_and(CtCDPI_Cursor, EOF_mask))) {
    1178             LocalCarryDeclare(subcarryQ, 14);
     1868            LocalCarryDeclare(subcarryQ, 13);
    11791869            CtCDPI_starts = simd_or(CtCDPI_starts, CtCDPI_Cursor);
    1180             PI_Cursor = simd_and(CtCDPI_Cursor, PI_start);
    1181             CD_Ct_Cursor = subcarryQ.BitBlock_advance_ci_co(simd_andc(CtCDPI_Cursor, PI_Cursor), simd<1>::constant<0>(), 0);
     1870            PI_Cursor = simd_and(CtCDPI_Cursor, PI_opener);
     1871            subcarryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(simd_andc(CtCDPI_Cursor, PI_Cursor), simd<1>::constant<0>(), CD_Ct_Cursor));
    11821872            CD_Cursor = simd_and(CD_Ct_Cursor, lex.LBracket);
    11831873            Ct_Cursor = simd_and(CD_Ct_Cursor, lex.Hyphen);
    1184             ctCDPI_Callouts.PI_starts = simd_or(ctCDPI_Callouts.PI_starts, PI_Cursor);
    1185             ctCDPI_Callouts.CD_starts = simd_or(ctCDPI_Callouts.CD_starts, CD_Cursor);
    1186             ctCDPI_Callouts.Ct_starts = simd_or(ctCDPI_Callouts.Ct_starts, Ct_Cursor);
    1187             Ct_Cursor = subcarryQ.BitBlock_advance_ci_co(Ct_Cursor, simd<1>::constant<0>(), 1);
    1188             Ct_errors = simd_or(Ct_errors, simd_andc(Ct_Cursor, lex.Hyphen));
    1189             Ct_Cursor = subcarryQ.BitBlock_advance_ci_co(subcarryQ.BitBlock_advance_ci_co(Ct_Cursor, simd<1>::constant<0>(), 2), simd<1>::constant<0>(), 3);
    1190             PI_Cursor = subcarryQ.BitBlock_advance_ci_co(PI_Cursor, simd<1>::constant<0>(), 4);
    1191             ctCDPI_Callouts.PI_name_starts = simd_or(ctCDPI_Callouts.PI_name_starts, PI_Cursor);
    1192             PI_name_end = subcarryQ.BitBlock_scanthru_ci_co(PI_Cursor, lex.NameScan, simd<1>::constant<0>(), 5);
    1193             ctCDPI_Callouts.PI_name_ends = simd_or(ctCDPI_Callouts.PI_name_ends, PI_name_end);
    1194             PI_Cursor = subcarryQ.BitBlock_scanthru_ci_co(PI_name_end, simd_andc(EOF_mask, PI_end), simd<1>::constant<0>(), 6);
    1195             CD_Cursor = subcarryQ.BitBlock_scanthru_ci_co(CD_Cursor, simd_andc(EOF_mask, ctCDPI_Callouts.CD_end), simd<1>::constant<0>(), 7);
    1196             Ct_Cursor = subcarryQ.BitBlock_advance_ci_co(subcarryQ.BitBlock_scanthru_ci_co(Ct_Cursor, simd_andc(EOF_mask, DoubleHyphen), simd<1>::constant<0>(), 8), simd<1>::constant<0>(), 9);
    1197             ctCDPI_Callouts.PI_ends = simd_or(ctCDPI_Callouts.PI_ends, PI_Cursor);
    1198             ctCDPI_Callouts.CD_ends = simd_or(ctCDPI_Callouts.CD_ends, CD_Cursor);
    1199             ctCDPI_Callouts.Ct_ends = simd_or(ctCDPI_Callouts.Ct_ends, Ct_Cursor);
     1874            if (bitblock::any(PI_Cursor)) {
     1875              ctCDPI_Callouts.PI_starts = simd_or(ctCDPI_Callouts.PI_starts, PI_Cursor);
     1876              subcarryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(PI_Cursor, simd<1>::constant<0>(), PI_Cursor));
     1877              ctCDPI_Callouts.PI_name_starts = simd_or(ctCDPI_Callouts.PI_name_starts, PI_Cursor);
     1878              subcarryQ.cq[2] = bitblock::srli<127>(pablo_blk_ScanThru(PI_Cursor, lex.NameScan, simd<1>::constant<0>(), PI_name_end));
     1879              PI_error = simd_and(PI_Cursor, PI_name_end);
     1880              PI_noWS = simd_andc(PI_name_end, lex.WS);
     1881              subcarryQ.cq[3] = bitblock::srli<127>(pablo_blk_Advance(PI_noWS, simd<1>::constant<0>(), tempvar2));
     1882              PI_error = simd_or(PI_error, simd_or(simd_andc(PI_noWS, lex.QMark), simd_andc(tempvar2, PI_closer)));
     1883              if (bitblock::any(PI_error)) {
     1884                assert_0_error("Error in PI syntax", PI_error);
     1885              }
     1886              ctCDPI_Callouts.PI_name_ends = simd_or(ctCDPI_Callouts.PI_name_ends, PI_name_end);
     1887              subcarryQ.cq[4] = bitblock::srli<127>(pablo_blk_ScanThru(PI_name_end, simd_andc(EOF_mask, PI_closer), simd<1>::constant<0>(), PI_Cursor));
     1888              ctCDPI_Callouts.PI_ends = simd_or(ctCDPI_Callouts.PI_ends, PI_Cursor);
     1889              CtCDPI_ends = simd_or(CtCDPI_ends, PI_Cursor);
     1890            }
     1891            else {
     1892              subcarryQ.CarryDequeueEnqueue(6, 4);
     1893            }
     1894            if (bitblock::any(CD_Cursor)) {
     1895              ctCDPI_Callouts.CD_starts = simd_or(ctCDPI_Callouts.CD_starts, CD_Cursor);
     1896              subcarryQ.cq[5] = bitblock::srli<127>(pablo_blk_ScanThru(CD_Cursor, simd_andc(EOF_mask, CD_closer), simd<1>::constant<0>(), CD_Cursor));
     1897              ctCDPI_Callouts.CD_ends = simd_or(ctCDPI_Callouts.CD_ends, CD_Cursor);
     1898              CtCDPI_ends = simd_or(CtCDPI_ends, CD_Cursor);
     1899            }
     1900            else {
     1901              subcarryQ.CarryDequeueEnqueue(10, 1);
     1902            }
     1903            if (bitblock::any(Ct_Cursor)) {
     1904              ctCDPI_Callouts.Ct_starts = simd_or(ctCDPI_Callouts.Ct_starts, Ct_Cursor);
     1905              subcarryQ.cq[6] = bitblock::srli<127>(pablo_blk_Advance(Ct_Cursor, simd<1>::constant<0>(), Ct_Cursor));
     1906              Ct_error = simd_andc(Ct_Cursor, lex.Hyphen);
     1907              subcarryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(Ct_Cursor, simd<1>::constant<0>(), tempvar3));
     1908              subcarryQ.cq[8] = bitblock::srli<127>(pablo_blk_Advance(tempvar3, simd<1>::constant<0>(), Ct_Cursor));
     1909              subcarryQ.cq[9] = bitblock::srli<127>(pablo_blk_ScanThru(Ct_Cursor, simd_andc(EOF_mask, DoubleHyphen), simd<1>::constant<0>(), tempvar4));
     1910              subcarryQ.cq[10] = bitblock::srli<127>(pablo_blk_Advance(tempvar4, simd<1>::constant<0>(), Ct_Cursor));
     1911              if (bitblock::any(simd_or(Ct_error, simd_andc(Ct_Cursor, lex.RAngle)))) {
     1912                assert_0_error("Error in comment syntax", simd_or(Ct_error, simd_andc(Ct_Cursor, lex.RAngle)));
     1913              }
     1914              ctCDPI_Callouts.Ct_ends = simd_or(ctCDPI_Callouts.Ct_ends, Ct_Cursor);
     1915              CtCDPI_ends = simd_or(CtCDPI_ends, Ct_Cursor);
     1916            }
     1917            else {
     1918              subcarryQ.CarryDequeueEnqueue(11, 5);
     1919            }
    12001920            CtCDPI_Cursor = simd_or(simd_or(PI_Cursor, CD_Cursor), Ct_Cursor);
    1201             CtCDPI_Cursor = subcarryQ.BitBlock_scanthru_ci_co(CtCDPI_Cursor, simd_andc(EOF_mask, CtCDPI_start), simd<1>::constant<0>(), 10);
    1202             ctCDPI_Callouts.CtCDPI_mask = subcarryQ.BitBlock_sub_ci_co(subcarryQ.BitBlock_advance_ci_co(simd_or(simd_or(ctCDPI_Callouts.CD_ends, ctCDPI_Callouts.Ct_ends), ctCDPI_Callouts.PI_ends), simd<1>::constant<0>(), 11), CtCDPI_starts, simd<1>::constant<0>(), 12);
    1203             ctCDPI_error = simd_or(Ct_errors, simd_andc(ctCDPI_Callouts.Ct_ends, lex.RAngle));
    1204             ctCDPI_error = simd_or(ctCDPI_error, simd_andc(subcarryQ.BitBlock_advance_ci_co(simd_andc(ctCDPI_Callouts.PI_name_ends, lex.WS), simd<1>::constant<0>(), 13), PI_end));
    1205             ctCDPI_error = simd_or(ctCDPI_error, simd_and(ctCDPI_Callouts.PI_name_starts, ctCDPI_Callouts.PI_name_ends));
    1206             ctCDPI_error = simd_or(ctCDPI_error, simd_andc(ctCDPI_Callouts.CtCDPI_mask, EOF_mask));
    1207             carryQ.CarryCombine((ICarryQueue *) &subcarryQ, 3, 14);
     1921            subcarryQ.cq[11] = bitblock::srli<127>(pablo_blk_InclusiveSpan(CtCDPI_starts, CtCDPI_ends, simd<1>::constant<0>(), ctCDPI_mask));
     1922            if (bitblock::any(simd_andc(ctCDPI_mask, EOF_mask))) {
     1923              assert_0_error("Error in comment, CDATA or processing instruction syntax", simd_andc(ctCDPI_mask, EOF_mask));
     1924            }
     1925            subcarryQ.cq[12] = bitblock::srli<127>(pablo_blk_ScanThru(CtCDPI_Cursor, simd_andc(EOF_mask, CtCDPI_opener), simd<1>::constant<0>(), CtCDPI_Cursor));
     1926            carryQ.CarryCombine(subcarryQ.cq, 5, 13);
    12081927          }
    12091928        }
    12101929        else {
    1211           carryQ.CarryDequeueEnqueue(3, 14);
    1212         }
    1213         if (bitblock::any(ctCDPI_error)) {
    1214           error_tracker.NoteError("Error in comment, CDATA or processing instruction syntax", ctCDPI_error);
    1215         }
    1216         check_streams.misc_mask = simd_and(simd_or(simd_or(simd_or(lex.WS, lex.LAngle), carryQ.BitBlock_sub_ci_co(carryQ.BitBlock_advance_ci_co(simd_or(ctCDPI_Callouts.Ct_ends, ctCDPI_Callouts.PI_ends), carryQ.get_carry_in(17), 17), simd_or(ctCDPI_Callouts.Ct_starts, ctCDPI_Callouts.PI_starts), carryQ.get_carry_in(18), 18)), CtCDPI_starts), EOF_mask);
     1930          carryQ.CarryDequeueEnqueue(5, 13);
     1931        }
     1932        carryQ.cq[18] = bitblock::srli<127>(pablo_blk_InclusiveSpan(simd_or(ctCDPI_Callouts.Ct_starts, ctCDPI_Callouts.PI_starts), simd_or(ctCDPI_Callouts.Ct_ends, ctCDPI_Callouts.PI_ends), carryQ.get_carry_in(18), tempvar5));
     1933        check_streams.misc_mask = simd_and(simd_or(simd_or(simd_or(lex.WS, lex.LAngle), tempvar5), CtCDPI_starts), EOF_mask);
     1934        marker.LAngle_scope = simd_andc(LAngle_scope, ctCDPI_mask);
     1935        marker.Ref_opener = simd_andc(lex.RefStart, ctCDPI_mask);
     1936        marker.CD_closer = simd_andc(CD_closer, ctCDPI_mask);
    12171937  }
    1218   void do_segment(CtCDPI_Callouts ctCDPI_Callouts[], Lex lex[], Scope1 scope1[], Check_streams check_streams[], int segment_blocks) {
    1219   int i;
    1220   for (i = 0; i < segment_blocks; i++)
    1221     do_block(ctCDPI_Callouts[i], lex[i], scope1[i], check_streams[i]);
    1222   }
    1223   CarryArray<19> carryQ;
    1224   };
    1225 
    1226   struct Validation_Vec_2 {
    1227   Validation_Vec_2() {
    1228  }
    1229   IDISA_INLINE void do_block(Citygml & citygml, Basis_bits & basis_bits) {
    1230                 BitBlock temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10;
    1231                 BitBlock temp11, temp12, temp13, temp14, temp15, temp16, temp17;
    1232                 BitBlock apperance_scope, vec_2_error1;
    1233 
    1234 
    1235         temp1 = simd_or(basis_bits.bit_0, basis_bits.bit_1);
    1236         temp2 = simd_and(basis_bits.bit_2, basis_bits.bit_3);
    1237         temp3 = simd_andc(temp2, temp1);
    1238         temp4 = simd_or(basis_bits.bit_4, basis_bits.bit_5);
    1239         temp5 = simd_or(basis_bits.bit_6, basis_bits.bit_7);
    1240         temp6 = simd_or(temp4, temp5);
    1241         temp7 = simd_andc(temp3, temp6);
    1242         temp8 = simd_andc(basis_bits.bit_1, basis_bits.bit_0);
    1243         temp9 = simd_and(temp8, temp2);
    1244         temp10 = simd_andc(basis_bits.bit_4, basis_bits.bit_5);
    1245         temp11 = simd_andc(temp10, temp5);
    1246         temp12 = simd_and(temp9, temp11);
    1247         temp13 = simd_or(temp7, temp12);
    1248         temp14 = simd_or(temp13, temp7);
    1249         citygml.null = simd_or(temp14, temp7);
    1250         temp15 = simd_and(basis_bits.bit_6, basis_bits.bit_7);
    1251         temp16 = simd_andc(temp15, temp4);
    1252         temp17 = simd_and(temp3, temp16);
    1253         citygml.Appearance = simd_or(temp14, temp17);
    1254         apperance_scope = carryQ.BitBlock_advance_ci_co(citygml.Appearance, carryQ.get_carry_in(0), 0);
    1255         vec_2_error1 = simd_andc(apperance_scope, citygml.null);
    1256         if (bitblock::any(vec_2_error1)) {
    1257           error_tracker.NoteError("Vec_2_error found", vec_2_error1);
    1258         }
    1259         carryQ.CarryQ_Adjust(1);
    1260   }
    1261   void do_final_block(Citygml & citygml, Basis_bits & basis_bits, BitBlock EOF_mask) {
    1262                 BitBlock temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10;
    1263                 BitBlock temp11, temp12, temp13, temp14, temp15, temp16, temp17;
    1264                 BitBlock apperance_scope, vec_2_error1;
    1265 
    1266 
    1267         temp1 = simd_or(basis_bits.bit_0, basis_bits.bit_1);
    1268         temp2 = simd_and(basis_bits.bit_2, basis_bits.bit_3);
    1269         temp3 = simd_andc(temp2, temp1);
    1270         temp4 = simd_or(basis_bits.bit_4, basis_bits.bit_5);
    1271         temp5 = simd_or(basis_bits.bit_6, basis_bits.bit_7);
    1272         temp6 = simd_or(temp4, temp5);
    1273         temp7 = simd_andc(temp3, temp6);
    1274         temp8 = simd_andc(basis_bits.bit_1, basis_bits.bit_0);
    1275         temp9 = simd_and(temp8, temp2);
    1276         temp10 = simd_andc(basis_bits.bit_4, basis_bits.bit_5);
    1277         temp11 = simd_andc(temp10, temp5);
    1278         temp12 = simd_and(temp9, temp11);
    1279         temp13 = simd_or(temp7, temp12);
    1280         temp14 = simd_or(temp13, temp7);
    1281         citygml.null = simd_or(temp14, temp7);
    1282         temp15 = simd_and(basis_bits.bit_6, basis_bits.bit_7);
    1283         temp16 = simd_andc(temp15, temp4);
    1284         temp17 = simd_and(temp3, temp16);
    1285         citygml.Appearance = simd_or(temp14, temp17);
    1286         apperance_scope = carryQ.BitBlock_advance_ci_co(citygml.Appearance, carryQ.get_carry_in(0), 0);
    1287         vec_2_error1 = simd_andc(apperance_scope, citygml.null);
    1288         if (bitblock::any(vec_2_error1)) {
    1289           error_tracker.NoteError("Vec_2_error found", vec_2_error1);
    1290         }
    1291   }
    1292   void do_segment(Citygml citygml[], Basis_bits basis_bits[], int segment_blocks) {
    1293   int i;
    1294   for (i = 0; i < segment_blocks; i++)
    1295     do_block(citygml[i], basis_bits[i]);
    1296   }
    1297   CarryArray<1> carryQ;
     1938  CarryArray<19, 0> carryQ;
    12981939  };
    12991940
    13001941  struct Do_check_streams {
    13011942 
    1302   IDISA_INLINE void do_block(CtCDPI_Callouts & ctCDPI_Callouts, Tag_Callouts & tag_Callouts, Lex & lex, U8 & u8, Scope1 & scope1, Ref_Callouts & ref_Callouts, Xml_names & xml_names, Check_streams & check_streams) {
    1303                 BitBlock CD_end_error;
    1304 
    1305 
    1306         CD_end_error = simd_andc(ctCDPI_Callouts.CD_end, simd_or(ctCDPI_Callouts.CtCDPI_mask, tag_Callouts.AttVal_spans));
    1307         if (bitblock::any(CD_end_error)) {
    1308           error_tracker.NoteError("Error: ]]> in text", CD_end_error);
     1943  IDISA_INLINE void do_block(Marker & marker, Tag_Callouts & tag_Callouts, Check_streams & check_streams) {
     1944
     1945
     1946
     1947
     1948
     1949        if (bitblock::any(simd_andc(marker.CD_closer, tag_Callouts.AttVal_spans))) {
     1950          assert_0_error("Error: ]]> in text", simd_andc(marker.CD_closer, tag_Callouts.AttVal_spans));
    13091951        }
    13101952        check_streams.tag_marks = simd_or(simd_or(simd_or(tag_Callouts.EmptyTag_marks, tag_Callouts.ElemName_starts), tag_Callouts.EndTag_marks), tag_Callouts.AttName_starts);
    13111953        check_streams.name_follows = simd_or(tag_Callouts.ElemName_ends, tag_Callouts.AttName_ends);
    1312         check_streams.att_refs = simd_and(tag_Callouts.AttVal_spans, scope1.RefStart);
     1954        check_streams.att_refs = simd_and(tag_Callouts.AttVal_spans, marker.Ref_opener);
    13131955  }
    1314   void do_final_block(CtCDPI_Callouts & ctCDPI_Callouts, Tag_Callouts & tag_Callouts, Lex & lex, U8 & u8, Scope1 & scope1, Ref_Callouts & ref_Callouts, Xml_names & xml_names, Check_streams & check_streams, BitBlock EOF_mask) {
    1315                 BitBlock CD_end_error;
    1316 
    1317 
    1318         CD_end_error = simd_andc(ctCDPI_Callouts.CD_end, simd_or(ctCDPI_Callouts.CtCDPI_mask, tag_Callouts.AttVal_spans));
    1319         if (bitblock::any(CD_end_error)) {
    1320           error_tracker.NoteError("Error: ]]> in text", CD_end_error);
     1956  void do_final_block(Marker & marker, Tag_Callouts & tag_Callouts, Check_streams & check_streams, BitBlock EOF_mask) {
     1957
     1958
     1959
     1960
     1961
     1962        if (bitblock::any(simd_andc(marker.CD_closer, tag_Callouts.AttVal_spans))) {
     1963          assert_0_error("Error: ]]> in text", simd_andc(marker.CD_closer, tag_Callouts.AttVal_spans));
    13211964        }
    13221965        check_streams.tag_marks = simd_or(simd_or(simd_or(tag_Callouts.EmptyTag_marks, tag_Callouts.ElemName_starts), tag_Callouts.EndTag_marks), tag_Callouts.AttName_starts);
    13231966        check_streams.name_follows = simd_or(tag_Callouts.ElemName_ends, tag_Callouts.AttName_ends);
    1324         check_streams.att_refs = simd_and(tag_Callouts.AttVal_spans, scope1.RefStart);
    1325   }
    1326   void do_segment(CtCDPI_Callouts ctCDPI_Callouts[], Tag_Callouts tag_Callouts[], Lex lex[], U8 u8[], Scope1 scope1[], Ref_Callouts ref_Callouts[], Xml_names xml_names[], Check_streams check_streams[], int segment_blocks) {
    1327   int i;
    1328   for (i = 0; i < segment_blocks; i++)
    1329     do_block(ctCDPI_Callouts[i], tag_Callouts[i], lex[i], u8[i], scope1[i], ref_Callouts[i], xml_names[i], check_streams[i]);
     1967        check_streams.att_refs = simd_and(tag_Callouts.AttVal_spans, marker.Ref_opener);
    13301968  }
    13311969 
     
    13391977
    13401978
     1979
     1980
    13411981        remaining_starts = tag_Callouts.ElemName_starts;
    13421982        remaining_ends = tag_Callouts.ElemName_ends;
    13431983        temp = tag_Callouts.ElemName_starts;
    1344         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(0), 0);
     1984        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(0), temp));
    13451985        tag_Callouts.ElemName_ends_1 = simd_and(temp, remaining_ends);
    13461986        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_1);
    1347         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(1), 1);
     1987        carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(1), temp));
    13481988        tag_Callouts.ElemName_ends_2 = simd_and(temp, remaining_ends);
    13491989        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_2);
    1350         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(2), 2);
     1990        carryQ.cq[2] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(2), temp));
    13511991        tag_Callouts.ElemName_ends_3 = simd_and(temp, remaining_ends);
    13521992        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_3);
    1353         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(3), 3);
     1993        carryQ.cq[3] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(3), temp));
    13541994        tag_Callouts.ElemName_ends_4 = simd_and(temp, remaining_ends);
    13551995        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_4);
    1356         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(4), 4);
     1996        carryQ.cq[4] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(4), temp));
    13571997        tag_Callouts.ElemName_ends_5 = simd_and(temp, remaining_ends);
    13581998        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_5);
    1359         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(5), 5);
     1999        carryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(5), temp));
    13602000        tag_Callouts.ElemName_ends_6 = simd_and(temp, remaining_ends);
    13612001        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_6);
    1362         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(6), 6);
     2002        carryQ.cq[6] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(6), temp));
    13632003        tag_Callouts.ElemName_ends_7 = simd_and(temp, remaining_ends);
    13642004        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_7);
    1365         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(7), 7);
     2005        carryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(7), temp));
    13662006        tag_Callouts.ElemName_ends_8 = simd_and(temp, remaining_ends);
    13672007        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_8);
    1368         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(8), 8);
     2008        carryQ.cq[8] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(8), temp));
    13692009        tag_Callouts.ElemName_ends_9 = simd_and(temp, remaining_ends);
    13702010        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_9);
    1371         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(9), 9);
     2011        carryQ.cq[9] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(9), temp));
    13722012        tag_Callouts.ElemName_ends_10 = simd_and(temp, remaining_ends);
    13732013        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_10);
    1374         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(10), 10);
     2014        carryQ.cq[10] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(10), temp));
    13752015        tag_Callouts.ElemName_ends_11 = simd_and(temp, remaining_ends);
    13762016        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_11);
    1377         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(11), 11);
     2017        carryQ.cq[11] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(11), temp));
    13782018        tag_Callouts.ElemName_ends_12 = simd_and(temp, remaining_ends);
    13792019        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_12);
    1380         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(12), 12);
     2020        carryQ.cq[12] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(12), temp));
    13812021        tag_Callouts.ElemName_ends_13 = simd_and(temp, remaining_ends);
    13822022        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_13);
    1383         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(13), 13);
     2023        carryQ.cq[13] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(13), temp));
    13842024        tag_Callouts.ElemName_ends_14 = simd_and(temp, remaining_ends);
    13852025        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_14);
    1386         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(14), 14);
     2026        carryQ.cq[14] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(14), temp));
    13872027        tag_Callouts.ElemName_ends_15 = simd_and(temp, remaining_ends);
    13882028        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_15);
    1389         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(15), 15);
     2029        carryQ.cq[15] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(15), temp));
    13902030        tag_Callouts.ElemName_ends_16 = simd_and(temp, remaining_ends);
    13912031        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_16);
     
    13972037
    13982038
     2039
     2040
    13992041        remaining_starts = tag_Callouts.ElemName_starts;
    14002042        remaining_ends = tag_Callouts.ElemName_ends;
    14012043        temp = tag_Callouts.ElemName_starts;
    1402         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(0), 0);
     2044        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(0), temp));
    14032045        tag_Callouts.ElemName_ends_1 = simd_and(temp, remaining_ends);
    14042046        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_1);
    1405         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(1), 1);
     2047        carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(1), temp));
    14062048        tag_Callouts.ElemName_ends_2 = simd_and(temp, remaining_ends);
    14072049        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_2);
    1408         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(2), 2);
     2050        carryQ.cq[2] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(2), temp));
    14092051        tag_Callouts.ElemName_ends_3 = simd_and(temp, remaining_ends);
    14102052        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_3);
    1411         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(3), 3);
     2053        carryQ.cq[3] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(3), temp));
    14122054        tag_Callouts.ElemName_ends_4 = simd_and(temp, remaining_ends);
    14132055        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_4);
    1414         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(4), 4);
     2056        carryQ.cq[4] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(4), temp));
    14152057        tag_Callouts.ElemName_ends_5 = simd_and(temp, remaining_ends);
    14162058        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_5);
    1417         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(5), 5);
     2059        carryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(5), temp));
    14182060        tag_Callouts.ElemName_ends_6 = simd_and(temp, remaining_ends);
    14192061        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_6);
    1420         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(6), 6);
     2062        carryQ.cq[6] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(6), temp));
    14212063        tag_Callouts.ElemName_ends_7 = simd_and(temp, remaining_ends);
    14222064        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_7);
    1423         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(7), 7);
     2065        carryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(7), temp));
    14242066        tag_Callouts.ElemName_ends_8 = simd_and(temp, remaining_ends);
    14252067        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_8);
    1426         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(8), 8);
     2068        carryQ.cq[8] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(8), temp));
    14272069        tag_Callouts.ElemName_ends_9 = simd_and(temp, remaining_ends);
    14282070        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_9);
    1429         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(9), 9);
     2071        carryQ.cq[9] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(9), temp));
    14302072        tag_Callouts.ElemName_ends_10 = simd_and(temp, remaining_ends);
    14312073        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_10);
    1432         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(10), 10);
     2074        carryQ.cq[10] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(10), temp));
    14332075        tag_Callouts.ElemName_ends_11 = simd_and(temp, remaining_ends);
    14342076        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_11);
    1435         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(11), 11);
     2077        carryQ.cq[11] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(11), temp));
    14362078        tag_Callouts.ElemName_ends_12 = simd_and(temp, remaining_ends);
    14372079        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_12);
    1438         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(12), 12);
     2080        carryQ.cq[12] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(12), temp));
    14392081        tag_Callouts.ElemName_ends_13 = simd_and(temp, remaining_ends);
    14402082        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_13);
    1441         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(13), 13);
     2083        carryQ.cq[13] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(13), temp));
    14422084        tag_Callouts.ElemName_ends_14 = simd_and(temp, remaining_ends);
    14432085        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_14);
    1444         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(14), 14);
     2086        carryQ.cq[14] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(14), temp));
    14452087        tag_Callouts.ElemName_ends_15 = simd_and(temp, remaining_ends);
    14462088        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_15);
    1447         temp = carryQ.BitBlock_advance_ci_co(temp, carryQ.get_carry_in(15), 15);
     2089        carryQ.cq[15] = bitblock::srli<127>(pablo_blk_Advance(temp, carryQ.get_carry_in(15), temp));
    14482090        tag_Callouts.ElemName_ends_16 = simd_and(temp, remaining_ends);
    14492091        remaining_ends = simd_andc(remaining_ends, tag_Callouts.ElemName_ends_16);
    14502092        tag_Callouts.ElemName_ends_17_and_longer = remaining_ends;
    14512093  }
    1452   void do_segment(Tag_Callouts tag_Callouts[], int segment_blocks) {
    1453   int i;
    1454   for (i = 0; i < segment_blocks; i++)
    1455     do_block(tag_Callouts[i]);
     2094  CarryArray<16, 0> carryQ;
     2095  };
     2096
     2097  struct Validate_xml_names {
     2098  Validate_xml_names() {
     2099 }
     2100  IDISA_INLINE void do_block(CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Tag_Callouts & tag_Callouts, Lex & lex, U8 & u8, Check_streams & check_streams) {
     2101                BitBlock PI_names, GenRefs, ElemNames, AttNames, qname_stream, ncname_stream;
     2102                BitBlock name_stream, name_start, name_cursor, void_prefix_err, namespace_sep;
     2103                BitBlock local_part_start, local_part_err, colon2_err, ncname_err;
     2104                BitBlock namespace_error;
     2105
     2106                BitBlock tempvar0, tempvar1, tempvar2, tempvar3;
     2107
     2108
     2109        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_SpanUpTo(ctCDPI_Callouts.PI_name_starts, ctCDPI_Callouts.PI_name_ends, carryQ.get_carry_in(0), PI_names));
     2110        carryQ.cq[1] = bitblock::srli<127>(pablo_blk_SpanUpTo(ref_Callouts.GenRef_starts, ref_Callouts.GenRef_ends, carryQ.get_carry_in(1), GenRefs));
     2111        carryQ.cq[2] = bitblock::srli<127>(pablo_blk_SpanUpTo(tag_Callouts.ElemName_starts, tag_Callouts.ElemName_ends, carryQ.get_carry_in(2), ElemNames));
     2112        carryQ.cq[3] = bitblock::srli<127>(pablo_blk_SpanUpTo(tag_Callouts.AttName_starts, tag_Callouts.AttName_ends, carryQ.get_carry_in(3), AttNames));
     2113        qname_stream = simd_or(ElemNames, AttNames);
     2114        ncname_stream = simd_or(PI_names, GenRefs);
     2115        name_stream = simd_or(qname_stream, ncname_stream);
     2116        carryQ.cq[4] = bitblock::srli<127>(pablo_blk_Advance(name_stream, carryQ.get_carry_in(4), tempvar0));
     2117        name_start = simd_andc(name_stream, tempvar0);
     2118        carryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(name_stream, carryQ.get_carry_in(5), tempvar1));
     2119        name_cursor = simd_andc(name_stream, tempvar1);
     2120        void_prefix_err = simd_and(name_cursor, lex.Colon);
     2121        carryQ.cq[6] = bitblock::srli<127>(pablo_blk_ScanThru(name_cursor, simd_andc(lex.NameScan, lex.Colon), carryQ.get_carry_in(6), tempvar2));
     2122        namespace_sep = simd_and(tempvar2, lex.Colon);
     2123        carryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(namespace_sep, carryQ.get_carry_in(7), local_part_start));
     2124        local_part_err = simd_andc(local_part_start, lex.NameScan);
     2125        carryQ.cq[8] = bitblock::srli<127>(pablo_blk_ScanThru(local_part_start, simd_andc(lex.NameScan, lex.Colon), carryQ.get_carry_in(8), tempvar3));
     2126        colon2_err = simd_and(tempvar3, lex.Colon);
     2127        ncname_err = simd_and(ncname_stream, lex.Colon);
     2128        namespace_error = simd_or(simd_or(simd_or(void_prefix_err, local_part_err), colon2_err), ncname_err);
     2129        if (bitblock::any(namespace_error)) {
     2130          error_tracker.NoteError("error found", namespace_error);
     2131        }
     2132        check_streams.non_ascii_name_starts = simd_andc(name_start, lex.ASCII_name_start);
     2133        check_streams.non_ascii_names = simd_andc(simd_andc(simd_andc(name_stream, name_start), lex.ASCII_name_char), u8.suffix);
     2134        carryQ.CarryQ_Adjust(9);
    14562135  }
    1457   CarryArray<16> carryQ;
     2136  void do_final_block(CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Tag_Callouts & tag_Callouts, Lex & lex, U8 & u8, Check_streams & check_streams, BitBlock EOF_mask) {
     2137                BitBlock PI_names, GenRefs, ElemNames, AttNames, qname_stream, ncname_stream;
     2138                BitBlock name_stream, name_start, name_cursor, void_prefix_err, namespace_sep;
     2139                BitBlock local_part_start, local_part_err, colon2_err, ncname_err;
     2140                BitBlock namespace_error;
     2141
     2142                BitBlock tempvar0, tempvar1, tempvar2, tempvar3;
     2143
     2144
     2145        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_SpanUpTo(ctCDPI_Callouts.PI_name_starts, ctCDPI_Callouts.PI_name_ends, carryQ.get_carry_in(0), PI_names));
     2146        carryQ.cq[1] = bitblock::srli<127>(pablo_blk_SpanUpTo(ref_Callouts.GenRef_starts, ref_Callouts.GenRef_ends, carryQ.get_carry_in(1), GenRefs));
     2147        carryQ.cq[2] = bitblock::srli<127>(pablo_blk_SpanUpTo(tag_Callouts.ElemName_starts, tag_Callouts.ElemName_ends, carryQ.get_carry_in(2), ElemNames));
     2148        carryQ.cq[3] = bitblock::srli<127>(pablo_blk_SpanUpTo(tag_Callouts.AttName_starts, tag_Callouts.AttName_ends, carryQ.get_carry_in(3), AttNames));
     2149        qname_stream = simd_or(ElemNames, AttNames);
     2150        ncname_stream = simd_or(PI_names, GenRefs);
     2151        name_stream = simd_or(qname_stream, ncname_stream);
     2152        carryQ.cq[4] = bitblock::srli<127>(pablo_blk_Advance(name_stream, carryQ.get_carry_in(4), tempvar0));
     2153        name_start = simd_andc(name_stream, tempvar0);
     2154        carryQ.cq[5] = bitblock::srli<127>(pablo_blk_Advance(name_stream, carryQ.get_carry_in(5), tempvar1));
     2155        name_cursor = simd_andc(name_stream, tempvar1);
     2156        void_prefix_err = simd_and(name_cursor, lex.Colon);
     2157        carryQ.cq[6] = bitblock::srli<127>(pablo_blk_ScanThru(name_cursor, simd_andc(lex.NameScan, lex.Colon), carryQ.get_carry_in(6), tempvar2));
     2158        namespace_sep = simd_and(tempvar2, lex.Colon);
     2159        carryQ.cq[7] = bitblock::srli<127>(pablo_blk_Advance(namespace_sep, carryQ.get_carry_in(7), local_part_start));
     2160        local_part_err = simd_andc(local_part_start, lex.NameScan);
     2161        carryQ.cq[8] = bitblock::srli<127>(pablo_blk_ScanThru(local_part_start, simd_andc(lex.NameScan, lex.Colon), carryQ.get_carry_in(8), tempvar3));
     2162        colon2_err = simd_and(tempvar3, lex.Colon);
     2163        ncname_err = simd_and(ncname_stream, lex.Colon);
     2164        namespace_error = simd_or(simd_or(simd_or(void_prefix_err, local_part_err), colon2_err), ncname_err);
     2165        if (bitblock::any(namespace_error)) {
     2166          error_tracker.NoteError("error found", namespace_error);
     2167        }
     2168        check_streams.non_ascii_name_starts = simd_andc(name_start, lex.ASCII_name_start);
     2169        check_streams.non_ascii_names = simd_andc(simd_andc(simd_andc(name_stream, name_start), lex.ASCII_name_char), u8.suffix);
     2170  }
     2171  CarryArray<9, 0> carryQ;
     2172  };
     2173
     2174  struct Compute_Hash_Value_Bitstream {
     2175 
     2176  IDISA_INLINE void do_block(Hash_data & hash_data, Basis_bits & basis_bits) {
     2177
     2178
     2179
     2180
     2181
     2182        hash_data.Hash_value = simd_xor(simd_xor(basis_bits.bit_2, basis_bits.bit_4), basis_bits.bit_6);
     2183  }
     2184  void do_final_block(Hash_data & hash_data, Basis_bits & basis_bits, BitBlock EOF_mask) {
     2185
     2186
     2187
     2188
     2189
     2190        hash_data.Hash_value = simd_xor(simd_xor(basis_bits.bit_2, basis_bits.bit_4), basis_bits.bit_6);
     2191  }
     2192 
    14582193  };
    14592194
     
    14652200
    14662201
     2202
     2203
    14672204        v = simd_or(lex.LAngle, lex.Hyphen);
    14682205        w = simd_or(lex.Hyphen, lex.QMark);
    1469         v1 = carryQ.BitBlock_advance_ci_co(v, carryQ.get_carry_in(0), 0);
    1470         w1 = carryQ.BitBlock_advance_ci_co(w, carryQ.get_carry_in(1), 1);
     2206        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(v, carryQ.get_carry_in(0), v1));
     2207        carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(w, carryQ.get_carry_in(1), w1));
    14712208        scope1.LAngle = simd_andc(v1, w1);
    14722209        scope1.Hyphen = simd_and(v1, w1);
     
    14792216
    14802217
     2218
     2219
    14812220        v = simd_or(lex.LAngle, lex.Hyphen);
    14822221        w = simd_or(lex.Hyphen, lex.QMark);
    1483         v1 = carryQ.BitBlock_advance_ci_co(v, carryQ.get_carry_in(0), 0);
    1484         w1 = carryQ.BitBlock_advance_ci_co(w, carryQ.get_carry_in(1), 1);
     2222        carryQ.cq[0] = bitblock::srli<127>(pablo_blk_Advance(v, carryQ.get_carry_in(0), v1));
     2223        carryQ.cq[1] = bitblock::srli<127>(pablo_blk_Advance(w, carryQ.get_carry_in(1), w1));
    14852224        scope1.LAngle = simd_andc(v1, w1);
    14862225        scope1.Hyphen = simd_and(v1, w1);
     
    14882227        scope1.RefStart = simd<1>::constant<0>();
    14892228  }
    1490   void do_segment(Lex lex[], Scope1 scope1[], int segment_blocks) {
    1491   int i;
    1492   for (i = 0; i < segment_blocks; i++)
    1493     do_block(lex[i], scope1[i]);
    1494   }
    1495   CarryArray<2> carryQ;
     2229  CarryArray<2, 0> carryQ;
    14962230  };
    14972231
    1498   struct Compute_Hash_Value_Bitstream {
    1499  
    1500   IDISA_INLINE void do_block(Hash_data & hash_data, Basis_bits & basis_bits) {
    1501 
    1502 
    1503 
    1504         hash_data.Hash_value = simd_xor(simd_xor(basis_bits.bit_2, basis_bits.bit_4), basis_bits.bit_6);
    1505   }
    1506   void do_final_block(Hash_data & hash_data, Basis_bits & basis_bits, BitBlock EOF_mask) {
    1507 
    1508 
    1509 
    1510         hash_data.Hash_value = simd_xor(simd_xor(basis_bits.bit_2, basis_bits.bit_4), basis_bits.bit_6);
    1511   }
    1512   void do_segment(Hash_data hash_data[], Basis_bits basis_bits[], int segment_blocks) {
    1513   int i;
    1514   for (i = 0; i < segment_blocks; i++)
    1515     do_block(hash_data[i], basis_bits[i]);
    1516   }
    1517  
    1518   };
    1519 
    1520   struct Validate_xml_names {
    1521   Validate_xml_names() {
    1522  }
    1523   IDISA_INLINE void do_block(CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Tag_Callouts & tag_Callouts, Lex & lex, U8 & u8, Xml_names & xml_names, Check_streams & check_streams) {
    1524                 BitBlock PI_names, GenRefs, ElemNames, AttNames, qname_stream, ncname_stream;
    1525                 BitBlock name_stream, name_start, name_cursor, void_prefix_err, namespace_sep;
    1526                 BitBlock local_part_start, local_part_err, colon2_err, ncname_err;
    1527                 BitBlock namespace_error;
    1528 
    1529 
    1530         PI_names = carryQ.BitBlock_sub_ci_co(ctCDPI_Callouts.PI_name_ends, ctCDPI_Callouts.PI_name_starts, carryQ.get_carry_in(0), 0);
    1531         GenRefs = carryQ.BitBlock_sub_ci_co(ref_Callouts.GenRef_ends, ref_Callouts.GenRef_starts, carryQ.get_carry_in(1), 1);
    1532         ElemNames = carryQ.BitBlock_sub_ci_co(tag_Callouts.ElemName_ends, tag_Callouts.ElemName_starts, carryQ.get_carry_in(2), 2);
    1533         AttNames = carryQ.BitBlock_sub_ci_co(tag_Callouts.AttName_ends, tag_Callouts.AttName_starts, carryQ.get_carry_in(3), 3);
    1534         qname_stream = simd_or(ElemNames, AttNames);
    1535         ncname_stream = simd_or(PI_names, GenRefs);
    1536         name_stream = simd_or(qname_stream, ncname_stream);
    1537         name_start = simd_andc(name_stream, carryQ.BitBlock_advance_ci_co(name_stream, carryQ.get_carry_in(4), 4));
    1538         name_cursor = simd_andc(name_stream, carryQ.BitBlock_advance_ci_co(name_stream, carryQ.get_carry_in(5), 5));
    1539         void_prefix_err = simd_and(name_cursor, lex.Colon);
    1540         namespace_sep = simd_and(carryQ.BitBlock_scanthru_ci_co(name_cursor, simd_andc(lex.NameScan, lex.Colon), carryQ.get_carry_in(6), 6), lex.Colon);
    1541         local_part_start = carryQ.BitBlock_advance_ci_co(namespace_sep, carryQ.get_carry_in(7), 7);
    1542         local_part_err = simd_andc(local_part_start, lex.NameScan);
    1543         colon2_err = simd_and(carryQ.BitBlock_scanthru_ci_co(local_part_start, simd_andc(lex.NameScan, lex.Colon), carryQ.get_carry_in(8), 8), lex.Colon);
    1544         ncname_err = simd_and(ncname_stream, lex.Colon);
    1545         namespace_error = simd_or(simd_or(simd_or(void_prefix_err, local_part_err), colon2_err), ncname_err);
    1546         if (bitblock::any(namespace_error)) {
    1547           error_tracker.NoteError("error found", namespace_error);
    1548         }
    1549         check_streams.non_ascii_name_starts = simd_andc(name_start, lex.ASCII_name_start);
    1550         check_streams.non_ascii_names = simd_andc(simd_andc(simd_andc(name_stream, name_start), lex.ASCII_name_char), u8.suffix);
    1551         carryQ.CarryQ_Adjust(9);
    1552   }
    1553   void do_final_block(CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Tag_Callouts & tag_Callouts, Lex & lex, U8 & u8, Xml_names & xml_names, Check_streams & check_streams, BitBlock EOF_mask) {
    1554                 BitBlock PI_names, GenRefs, ElemNames, AttNames, qname_stream, ncname_stream;
    1555                 BitBlock name_stream, name_start, name_cursor, void_prefix_err, namespace_sep;
    1556                 BitBlock local_part_start, local_part_err, colon2_err, ncname_err;
    1557                 BitBlock namespace_error;
    1558 
    1559 
    1560         PI_names = carryQ.BitBlock_sub_ci_co(ctCDPI_Callouts.PI_name_ends, ctCDPI_Callouts.PI_name_starts, carryQ.get_carry_in(0), 0);
    1561         GenRefs = carryQ.BitBlock_sub_ci_co(ref_Callouts.GenRef_ends, ref_Callouts.GenRef_starts, carryQ.get_carry_in(1), 1);
    1562         ElemNames = carryQ.BitBlock_sub_ci_co(tag_Callouts.ElemName_ends, tag_Callouts.ElemName_starts, carryQ.get_carry_in(2), 2);
    1563         AttNames = carryQ.BitBlock_sub_ci_co(tag_Callouts.AttName_ends, tag_Callouts.AttName_starts, carryQ.get_carry_in(3), 3);
    1564         qname_stream = simd_or(ElemNames, AttNames);
    1565         ncname_stream = simd_or(PI_names, GenRefs);
    1566         name_stream = simd_or(qname_stream, ncname_stream);
    1567         name_start = simd_andc(name_stream, carryQ.BitBlock_advance_ci_co(name_stream, carryQ.get_carry_in(4), 4));
    1568         name_cursor = simd_andc(name_stream, carryQ.BitBlock_advance_ci_co(name_stream, carryQ.get_carry_in(5), 5));
    1569         void_prefix_err = simd_and(name_cursor, lex.Colon);
    1570         namespace_sep = simd_and(carryQ.BitBlock_scanthru_ci_co(name_cursor, simd_andc(lex.NameScan, lex.Colon), carryQ.get_carry_in(6), 6), lex.Colon);
    1571         local_part_start = carryQ.BitBlock_advance_ci_co(namespace_sep, carryQ.get_carry_in(7), 7);
    1572         local_part_err = simd_andc(local_part_start, lex.NameScan);
    1573         colon2_err = simd_and(carryQ.BitBlock_scanthru_ci_co(local_part_start, simd_andc(lex.NameScan, lex.Colon), carryQ.get_carry_in(8), 8), lex.Colon);
    1574         ncname_err = simd_and(ncname_stream, lex.Colon);
    1575         namespace_error = simd_or(simd_or(simd_or(void_prefix_err, local_part_err), colon2_err), ncname_err);
    1576         if (bitblock::any(namespace_error)) {
    1577           error_tracker.NoteError("error found", namespace_error);
    1578         }
    1579         check_streams.non_ascii_name_starts = simd_andc(name_start, lex.ASCII_name_start);
    1580         check_streams.non_ascii_names = simd_andc(simd_andc(simd_andc(name_stream, name_start), lex.ASCII_name_char), u8.suffix);
    1581   }
    1582   void do_segment(CtCDPI_Callouts ctCDPI_Callouts[], Ref_Callouts ref_Callouts[], Tag_Callouts tag_Callouts[], Lex lex[], U8 u8[], Xml_names xml_names[], Check_streams check_streams[], int segment_blocks) {
    1583   int i;
    1584   for (i = 0; i < segment_blocks; i++)
    1585     do_block(ctCDPI_Callouts[i], ref_Callouts[i], tag_Callouts[i], lex[i], u8[i], xml_names[i], check_streams[i]);
    1586   }
    1587   CarryArray<9> carryQ;
    1588   };
    1589 
    1590 
    1591 
    1592 //////////////////////////////////////////////////////////////////////////////////////////
    1593 // Headers that depend @ global stream struct types.
    1594 //////////////////////////////////////////////////////////////////////////////////////////
    1595 #include "../lib/transpose.hpp"
     2232
     2233
     2234#include <simd-lib/transpose.hpp>
    15962235#include <post_process.hpp>
    1597 
    1598 static void do_process(FILE *infile, FILE *outfile);
    1599 
    1600 int main(int argc, char * argv[]) {
    1601 
    1602         char * infilename, * outfilename;
    1603         FILE *infile, *outfile;
    1604         struct stat fileinfo;
    1605 
    1606         if (argc < 2) {
    1607                 printf("Usage: %s <filename> [<outputfile>]\n", argv[0]);
    1608                 exit(-1);
    1609         }
    1610 
    1611         infilename = argv[1];
    1612         stat(infilename, &fileinfo);
    1613         infile = fopen(infilename, "rb");
    1614         if (!infile) {
    1615                 fprintf(stderr, "Error: cannot open %s for input.\n", infilename);
    1616                 exit(-1);
    1617         }
    1618 
    1619         if (argc < 3) outfile = stdout;
    1620         else {
    1621                 outfilename = argv[2];
    1622                 outfile = fopen(outfilename, "wb");
    1623                 if (!outfile) {
    1624                         fprintf(stderr, "Error: cannot open %s for writing.\n", outfilename);
    1625                         exit(-1);
    1626                 }
    1627         }
    1628 
    1629         PERF_SEC_BIND(1);
    1630 
    1631         PERF_SEC_INIT(parser_timer);
    1632 
    1633         do_process(infile, outfile);
    1634 
    1635         PERF_SEC_DUMP(parser_timer);
    1636 
    1637         PERF_SEC_DESTROY(parser_timer);
    1638 
    1639         fclose(infile);
    1640         fclose(outfile);
    1641 
    1642         return(0);
     2236/*
     2237static inline void s2p_do_block(BytePack U8[], Basis_bits & basis_bits) {
     2238  s2p(U8[0], U8[1], U8[2], U8[3], U8[4], U8[5], U8[6], U8[7],
     2239        basis_bits.bit_0, basis_bits.bit_1, basis_bits.bit_2, basis_bits.bit_3, basis_bits.bit_4, basis_bits.bit_5, basis_bits.bit_6, basis_bits.bit_7);
    16432240}
    16442241
    1645 void do_process(FILE *infile, FILE *outfile) {
    1646 
    1647       struct U8 u8;
    1648 
    1649   struct Lex lex;
    1650 
    1651   struct Scope1 scope1;
    1652 
    1653   struct CtCDPI_Callouts ctCDPI_Callouts;
    1654 
    1655   struct Ref_Callouts ref_Callouts;
    1656 
    1657   struct Hash_data hash_data;
    1658 
    1659   struct Tag_Callouts tag_Callouts;
    1660 
    1661   struct Basis_bits basis_bits;
    1662 
    1663   struct Check_streams check_streams;
    1664 
    1665   struct Xml_names xml_names;
    1666 
    1667   struct Citygml citygml;
    1668 
    1669 
    1670 
    1671     LineColTracker tracker;
    1672         TagMatcher<SEGMENT_SIZE,LOOKAHEAD_SIZE> matcher;
    1673 
    1674     int block_base=0;
    1675         int block_pos = 0;
    1676     int buffer_base=0;
    1677     int buffer_pos = 0;
    1678     int chars_avail = 0;
    1679         int chars_read = 0;
    1680 
    1681     //////////////////////////////////////////////////////////////////////////////////////////
    1682     // Buffer Management
    1683     //////////////////////////////////////////////////////////////////////////////////////////
    1684         //      BitBlock buf[(BUFFER_SIZE)/sizeof(BitBlock)];
    1685         //      uint8_t * src_buffer = (uint8_t *)buf + COPYBACK_SIZE;
    1686 
    1687         #include "../lib/buffer.hpp"
    1688         uint8_t * COPYBACK;
    1689         uint8_t * src_buffer;
    1690         ALLOC_STATIC_ALIGNED_BYTE_BUFFER_WITH_COPYBACK(COPYBACK, src_buffer);
    1691        
    1692     //////////////////////////////////////////////////////////////////////////////////////////
    1693     // XML Validation / Content Model
    1694     //////////////////////////////////////////////////////////////////////////////////////////
    1695         chars_read = fread((void *)src_buffer, 1, SEGMENT_SIZE, infile);
    1696         chars_avail = chars_read;
    1697         if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
    1698 
    1699     if(chars_read<4){
    1700                 fprintf(stderr,"File is too short. Not well formed.\n");
    1701                 exit(-1);
    1702     }
    1703 
    1704     Entity_Info * e = new Entity_Info;
    1705         e->AnalyzeSignature((unsigned char *)src_buffer);
    1706 
    1707     if (e->code_unit_base == ASCII) {
    1708 
    1709                 XML_Decl_Parser<ASCII> decl_parser((unsigned char *)src_buffer);
    1710 
    1711                 decl_parser.ReadXMLInfo(*e);
    1712 
    1713                 if (e->code_unit_size != SingleByte || (e->has_encoding_decl && (!at_UTF_8(e->encoding)))){
    1714                         fprintf(stderr,"Sorry, this xmlwf demo only works for UTF-8.\n");
    1715                         exit(-1);
    1716                 }
    1717     }
    1718     else {
    1719         fprintf(stderr,"Sorry, this xmlwf demo does not process EBCDIC.\n");
    1720         exit(-1);
    1721     }
    1722 
    1723     if (e->content_start != 0) {
    1724 
    1725                 memmove(&src_buffer[0], &src_buffer[e->content_start], chars_avail - e->content_start);
    1726                 buffer_pos = e->content_start;
    1727                 if ((chars_avail-e->content_start) < SEGMENT_SIZE) {
    1728                         chars_read = chars_avail - e->content_start + fread(&src_buffer[chars_avail-e->content_start], 1, e->content_start, infile);
    1729                         chars_avail = chars_read;
    1730                 }
    1731                 if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
    1732     }
    1733 
    1734         //////////////////////////////////////////////////////////////////////////////////////////
    1735         // Read OVERLAP bytes to support post processing validation lookahead.
    1736         //////////////////////////////////////////////////////////////////////////////////////////
    1737         chars_read = chars_avail + fread(&src_buffer[chars_avail], 1, LOOKAHEAD_SIZE, infile);
    1738         chars_avail = chars_read;
    1739         if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
    1740 
    1741       Parse_refs parse_refs;
    1742   Parse_tags parse_tags;
    1743   Classify_bytes_Validate_utf8 classify_bytes_Validate_utf8;
    1744   Parse_CtCDPI parse_CtCDPI;
    1745   Validation_Vec_2 validation_Vec_2;
    1746   Do_check_streams do_check_streams;
    1747   Form_Length_Group_Bitstreams form_Length_Group_Bitstreams;
    1748   Add_scope_streams add_scope_streams;
    1749   Compute_Hash_Value_Bitstream compute_Hash_Value_Bitstream;
    1750   Validate_xml_names validate_xml_names;
    1751 
    1752 
    1753     //////////////////////////////////////////////////////////////////////////////////////////
    1754     // Full Segments
    1755     //////////////////////////////////////////////////////////////////////////////////////////
    1756         matcher.setSrc((char *)src_buffer);
    1757         while (chars_avail >= SEGMENT_SIZE) {
    1758       PERF_SEC_START(parser_timer);
    1759       for (int blk = 0; blk < SEGMENT_BLOCKS; blk++) {
    1760           block_base = blk*BLOCK_SIZE;
    1761                   s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits);
    1762 
    1763          
    1764   classify_bytes_Validate_utf8.do_block(basis_bits, lex, u8);
    1765   add_scope_streams.do_block(lex, scope1);
    1766   parse_CtCDPI.do_block(ctCDPI_Callouts, lex, scope1, check_streams);
    1767   parse_tags.do_block(lex, scope1, ctCDPI_Callouts, tag_Callouts);
    1768   parse_refs.do_block(lex, scope1, ctCDPI_Callouts, ref_Callouts);
    1769   validate_xml_names.do_block(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, xml_names, check_streams);
    1770   do_check_streams.do_block(ctCDPI_Callouts, tag_Callouts, lex, u8, scope1, ref_Callouts, xml_names, check_streams);
    1771   form_Length_Group_Bitstreams.do_block(tag_Callouts);
    1772   compute_Hash_Value_Bitstream.do_block(hash_data, basis_bits);
    1773 
    1774           tracker.StoreNewlines(lex.LF);
    1775                   postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
    1776           matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
    1777           tracker.AdvanceBlock();
    1778       }
    1779       matcher.StreamScan(chars_avail);
    1780       matcher.Advance_buffer();
    1781       PERF_SEC_END(parser_timer, chars_avail);
    1782 
    1783           memmove(src_buffer, &src_buffer[SEGMENT_SIZE], LOOKAHEAD_SIZE); // CopyBack Trailing Padding
    1784 
    1785           // memmove(src_base, &src_buf[SEGMENT_SIZE-COPYBACK_SIZE], COPYBACK_SIZE); // Symbol Table CopyBack
    1786           chars_read = fread(&src_buffer[LOOKAHEAD_SIZE], 1, SEGMENT_SIZE, infile) + LOOKAHEAD_SIZE;
    1787           chars_avail = chars_read;
    1788           if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
    1789           buffer_pos += chars_avail;
    1790           buffer_base = buffer_pos;
    1791     }
    1792 
    1793     //////////////////////////////////////////////////////////////////////////////////////////
    1794     // Final Partial Segment
    1795     //////////////////////////////////////////////////////////////////////////////////////////
    1796     PERF_SEC_START(parser_timer);
    1797 
    1798     block_pos = 0;
    1799     int remaining = chars_avail;
    1800 
    1801     /* Full Blocks */
    1802     while (remaining >= BLOCK_SIZE) {
    1803           block_base = block_pos;
    1804                   s2p_do_block((BytePack *) &src_buffer[block_pos], basis_bits);
    1805          
    1806   classify_bytes_Validate_utf8.do_block(basis_bits, lex, u8);
    1807   add_scope_streams.do_block(lex, scope1);
    1808   parse_CtCDPI.do_block(ctCDPI_Callouts, lex, scope1, check_streams);
    1809   parse_tags.do_block(lex, scope1, ctCDPI_Callouts, tag_Callouts);
    1810   parse_refs.do_block(lex, scope1, ctCDPI_Callouts, ref_Callouts);
    1811   validate_xml_names.do_block(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, xml_names, check_streams);
    1812   do_check_streams.do_block(ctCDPI_Callouts, tag_Callouts, lex, u8, scope1, ref_Callouts, xml_names, check_streams);
    1813   form_Length_Group_Bitstreams.do_block(tag_Callouts);
    1814   compute_Hash_Value_Bitstream.do_block(hash_data, basis_bits);
    1815           tracker.StoreNewlines(lex.LF);
    1816                   postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
    1817                   matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
    1818           tracker.AdvanceBlock();
    1819           block_pos += BLOCK_SIZE;
    1820           remaining -= BLOCK_SIZE;
    1821     }
    1822     block_base = block_pos;
    1823 
    1824     /* Partial Block or Any Carry */
    1825     if (remaining > 0 || parse_refs.carryQ.CarryTest(0, 6) || parse_tags.carryQ.CarryTest(0, 19) || classify_bytes_Validate_utf8.carryQ.CarryTest(0, 12) || parse_CtCDPI.carryQ.CarryTest(0, 19) || validation_Vec_2.carryQ.CarryTest(0, 1) || form_Length_Group_Bitstreams.carryQ.CarryTest(0, 16) || add_scope_streams.carryQ.CarryTest(0, 2) || validate_xml_names.carryQ.CarryTest(0, 9)) {
    1826           EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
    1827                   s2p_do_final_block((BytePack *) &src_buffer[block_pos], basis_bits, EOF_mask);
    1828          
    1829   classify_bytes_Validate_utf8.do_final_block(basis_bits, lex, u8, EOF_mask);
    1830   add_scope_streams.do_final_block(lex, scope1, EOF_mask);
    1831   parse_CtCDPI.do_final_block(ctCDPI_Callouts, lex, scope1, check_streams, EOF_mask);
    1832   parse_tags.do_final_block(lex, scope1, ctCDPI_Callouts, tag_Callouts, EOF_mask);
    1833   parse_refs.do_final_block(lex, scope1, ctCDPI_Callouts, ref_Callouts, EOF_mask);
    1834   validate_xml_names.do_final_block(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, xml_names, check_streams, EOF_mask);
    1835   do_check_streams.do_final_block(ctCDPI_Callouts, tag_Callouts, lex, u8, scope1, ref_Callouts, xml_names, check_streams, EOF_mask);
    1836   form_Length_Group_Bitstreams.do_final_block(tag_Callouts, EOF_mask);
    1837   compute_Hash_Value_Bitstream.do_final_block(hash_data, basis_bits, EOF_mask);
    1838           tracker.StoreNewlines(lex.LF);
    1839                   postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
    1840           matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
    1841           tracker.AdvanceBlock();
    1842 
    1843     }
    1844 
    1845     buffer_pos += chars_avail;
    1846     buffer_base = buffer_pos;
    1847 
    1848     matcher.StreamScan(chars_avail);
    1849     matcher.Advance_buffer();
    1850 
    1851     if (matcher.depth != 0) {
    1852       fprintf(stderr, "tag matching error (depth %i) at position %i\n", matcher.depth, buffer_base);
    1853       exit(-1);
    1854     }
    1855         PERF_SEC_END(parser_timer, chars_avail);
     2242static inline void s2p_do_final_block(BytePack U8[], Basis_bits & basis_bits, BitBlock EOF_mask) {
     2243  s2p_do_block(U8, basis_bits);
     2244  basis_bits.bit_0 = simd_and(basis_bits.bit_0, EOF_mask);
     2245  basis_bits.bit_1 = simd_and(basis_bits.bit_1, EOF_mask);
     2246  basis_bits.bit_2 = simd_and(basis_bits.bit_2, EOF_mask);
     2247  basis_bits.bit_3 = simd_and(basis_bits.bit_3, EOF_mask);
     2248  basis_bits.bit_4 = simd_and(basis_bits.bit_4, EOF_mask);
     2249  basis_bits.bit_5 = simd_and(basis_bits.bit_5, EOF_mask);
     2250  basis_bits.bit_6 = simd_and(basis_bits.bit_6, EOF_mask);
     2251  basis_bits.bit_7 = simd_and(basis_bits.bit_7, EOF_mask);
    18562252}
    1857 
    1858 static inline int bitblock_has_bit(BitBlock block){
    1859         if (count_forward_zeroes(block) == sizeof(block)*8){
    1860                 return 0;
    1861         }else{
    1862                 return 1;
    1863         }
    1864 }
    1865 
    1866 static inline int ScanBackwardPos(BitBlock * block, int pos, int previous_block_last_elem_start)
    1867 {
    1868     BitBlock s = block[0];
    1869     BitBlock temp = simd_and( s, simd_not(simd<128>::sll(simd<1>::constant<1>(), convert(pos))) );
    1870 
    1871     if (bitblock_has_bit(temp))
    1872     {
    1873         // sizeof (BitBlock)*8 - cbzl( s & ~(~0 << pos)) - 1;
    1874         return BLOCK_SIZE - count_reverse_zeroes (temp) - 1;
    1875     }
    1876     else
    1877     {
    1878         //handle boundary case
    1879         return previous_block_last_elem_start - 1;
    1880     }
    1881 }
    1882 
    1883 static inline int compute_hash_value (int lgth, int start)
    1884 {
    1885     unsigned int offset_bit = start + 128;
    1886     uint64_t stream = *((uint64_t*)(((uint32_t*)hashvalues)+(offset_bit>>5)));
    1887     return stream >> (offset_bit & 0x1F) & ~(~0 << lgth);
    1888 }
    1889 
    1890 static inline char ElemEnd_grouping(int pos) {
    1891     int end = block_base + pos;
    1892     int start = ScanBackwardPos (&elem_starts, pos,previous_block_last_elem_start) + block_base;
    1893     int lgth = end - start;
    1894     int hashvalue = compute_hash_value(lgth, start - block_base);
    1895     char gid = 0;
    1896 //    if (lgth >16){
    1897 //      cout << "length = " << lgth << endl;
    1898 //      cout << "hashvalue = " << hashvalue << endl;
    1899 //    }
    1900 
    1901 //    if (lgth < 32)
    1902 //    {
    1903 //      gid = pbgs_symbol_table.Lookup_or_Insert_Name_32(source + start, hashvalue, lgth);
    1904 //    }
    1905 //    else
    1906 
    1907         switch (lgth)
    1908     {
    1909     case 1:
    1910         gid = pbgs_symbol_table.Lookup_or_Insert_Name<1>(source + start, hashvalue);
    1911         break;
    1912     case 2:
    1913         gid = pbgs_symbol_table.Lookup_or_Insert_Name<2>(source + start, hashvalue);
    1914         break;
    1915     case 3:
    1916         gid = pbgs_symbol_table.Lookup_or_Insert_Name<3>(source + start, hashvalue);
    1917         break;
    1918     case 4:
    1919         gid = pbgs_symbol_table.Lookup_or_Insert_Name<4>(source + start, hashvalue);
    1920         break;
    1921     case 5:
    1922         gid = pbgs_symbol_table.Lookup_or_Insert_Name<5>(source + start, hashvalue);
    1923         break;
    1924     case 6:
    1925         gid = pbgs_symbol_table.Lookup_or_Insert_Name<6>(source + start, hashvalue);
    1926         break;
    1927     case 7:
    1928         gid = pbgs_symbol_table.Lookup_or_Insert_Name<7>(source + start, hashvalue);
    1929         break;
    1930     case 8:
    1931         gid = pbgs_symbol_table.Lookup_or_Insert_Name<8>(source + start, hashvalue);
    1932         break;
    1933     case 9:
    1934         gid = pbgs_symbol_table.Lookup_or_Insert_Name<9>(source + start, hashvalue);
    1935         break;
    1936     case 10:
    1937         gid = pbgs_symbol_table.Lookup_or_Insert_Name<10>(source + start, hashvalue);
    1938         break;
    1939     case 11:
    1940         gid = pbgs_symbol_table.Lookup_or_Insert_Name<11>(source + start, hashvalue);
    1941         break;
    1942     case 12:
    1943         gid = pbgs_symbol_table.Lookup_or_Insert_Name<12>(source + start, hashvalue);
    1944         break;
    1945     case 13:
    1946         gid = pbgs_symbol_table.Lookup_or_Insert_Name<13>(source + start, hashvalue);
    1947         break;
    1948     case 14:
    1949         gid = pbgs_symbol_table.Lookup_or_Insert_Name<14>(source + start, hashvalue);
    1950         break;
    1951     case 15:
    1952         gid = pbgs_symbol_table.Lookup_or_Insert_Name<15>(source + start, hashvalue);
    1953         break;
    1954     case 16:
    1955         gid = pbgs_symbol_table.Lookup_or_Insert_Name<16>(source + start, hashvalue);
    1956         break;
    1957     default:
    1958         gid = pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
    1959         break;
    1960     }
    1961 /*
    1962     {
    1963         gid = (char)pbgs_symbol_table.Lookup_or_Insert_Name(source + start, hashvalue, lgth);
    1964     }
    19652253*/
    1966     gids.push_back(gid);
    1967 #if DEBUG
    1968     char* symbol = new char[lgth+1];
    1969     strncpy ( symbol, source + start, lgth );
    1970     symbol[lgth] ='\0';
    1971     printf ("%s | start: %i[%i] | end: %i[%i] | lgth: %i | hashvalue: %i | gid: %i | symbol: %s\n", __FUNCTION__, start, start - block_base, end, end - block_base, lgth, hashvalue, gid, symbol);
    1972 #endif
    1973         char* symbol = new char[lgth+1];
    1974     strncpy ( symbol, source + start, lgth );
    1975     symbol[lgth] ='\0';
    1976         fprintf(ofile,"%i\t%s\n",gid,symbol);
    1977     return gid;
    1978 }
    1979 
    1980 static inline int StreamScanLengthGrouping(Tag_Callouts & tag_Callouts, int blk_count) {
    1981     int blk;
    1982     int block_pos = 0;
    1983     ScanBlock *stream, *stream2;
    1984     stream = (ScanBlock *) &tag_Callouts.ElemName_ends;
    1985     stream2 = (ScanBlock *) &tag_Callouts.Tag_closing;
    1986     for (blk = 0; blk < blk_count; blk++) {
    1987 //      if (counter >= 600){
    1988 //              print_buffer(val_buf,"buffer");
    1989 //              cout << val_buf_curser << endl;
    1990 //              cout << "flag = " << content_start_flag << endl;
    1991 //      }
    1992         //cout << "flag = " << content_start_flag << endl;
    1993         ScanBlock s = stream[blk];
    1994         ScanBlock s2 = stream2[blk];
    1995         if (content_start_flag == 1){           //content_start is in the incomming blocks
    1996                 //cout << "curser=" << val_buf_curser;
    1997                 //cout << "block_pos=" << block_pos;
    1998 //              val_double_list(tag_Callouts,-2,block_pos+block_base,blk);
    1999                 //print_buffer(val_buf,"buffer");
    2000         }else if (content_start_flag == -1){            //content_start already occured in the former block
    2001                 //cout << "curser=" << val_buf_curser;
    2002                 //cout << "block_pos=" << block_pos;
    2003 //              val_double_list(tag_Callouts,-1,block_pos+block_base,blk);
    2004                 //print_buffer(val_buf,"buffer");
    2005         }
    2006         while(s | s2) {
    2007 //              cout << "end_pos:"<<s<<endl;
    2008 //              cout << "end_tag_marks:"<<s2<<endl;
    2009             int end_pos = __builtin_ctzll(s) + block_pos;
    2010             int end_tag = __builtin_ctzll(s2) + block_pos;
    2011 //          cout << "count_forward_zeroes(s):" << count_forward_zeroes(s)<<endl;
    2012 //          cout << "count_forward_zeroes(s2):" << count_forward_zeroes(s2)<<endl;
    2013 //              cout << (end_tag < end_pos && s2 != 0)<< "||" <<(s == 0 && s2 != 0)  << endl;
    2014             if ((end_tag < end_pos && s2 != 0) | (s == 0 && s2 != 0)){
    2015 //          cout << "closing tag" << endl;
    2016 //              cout << "endtag:" << end_tag << " " << end_pos<<endl;
    2017 //              cout << "1"<< endl;
    2018 //              cout << elem_vec_stack.size()<<endl;
    2019 //              cout << "2" << endl;
    2020 //              cout << elem_vec_stack.top()<<endl;
    2021                         if (!(*elem_vec_stack.top()).empty()){
    2022                                 if ((*elem_vec_stack.top()).back() != 0 ){
    2023                                         (*elem_vec_stack.top()).push_back(0);   
    2024                                 }
    2025                         }
    2026                         elem_vec_stack.pop();
    2027 //              cout<<"poped\n";
    2028                         s2 = s2 & (s2 - 1);
    2029                         content_start_flag = 0;
    2030             }else{
    2031 //              cout << " end_pos == " << end_pos<<endl;
    2032                 char gid = ElemEnd_grouping(end_pos);
    2033                 // if the gid == 12, the element content is of doublelist type, should do memcpy
    2034                 if (gid == 13){
    2035                         //cout << "gid = 12" << endl; 
    2036                         //cout << "curser=" << val_buf_curser<<endl;
    2037                         //cout << "block_pos=" << block_pos<<endl;
    2038 //                      val_double_list(tag_Callouts,end_pos,block_pos+block_base,blk);
    2039                         //print_buffer(val_buf,"buffer");
    2040                 }
    2041 //              cout << gid << endl;
    2042                 s = s & (s-1);  // clear rightmost bit.
    2043                         if (gid != 0){
    2044 //                              cout << "gid " << gid<< endl;
    2045                                 if(elem_vec_map.find(gid) != elem_vec_map.end()){
    2046 //                                      cout << "found vector" << gid << endl;
    2047                                         (*elem_vec_stack.top()).push_back(gid);
    2048 //                                      cout << "top vector size: " << (*elem_vec_map[gid]).size() << endl;
    2049                                         elem_vec_stack.push(elem_vec_map[gid]);
    2050 //                                      cout << "pushed in" << endl;
    2051 //                                      cout << "found " << gid << endl;
    2052                                 }else{
    2053 //                                      cout << "size :" << elem_vec_stack.size()<<endl;
    2054 //                                      cout << endl;
    2055 //                                      cout << elem_vec_stack.top()<<endl;
    2056                                
    2057 //                                      cout << "size before " <<(*elem_vec_stack.top()).size() << endl;
    2058                                         (*elem_vec_stack.top()).push_back(gid);
    2059 //                                      cout << "size after " <<(*elem_vec_stack.top()).size() << endl;
    2060                                
    2061                                         vector<char> *tmp = new vector<char>;
    2062                                         elem_vec_map[gid] = tmp;
    2063                                         elem_vec_stack.push(tmp);
    2064                                         //cout << "size after :" << elem_vec_stack.size()<<endl;
    2065                                 }
    2066                         }
    2067 
    2068             }
    2069         }
    2070         block_pos += 8 * sizeof(ScanBlock);
    2071     }
    2072     return 0;
    2073 }
    2074 
    2075 static inline void print_vecs(){
    2076         int i = 1;
    2077         int lst[256] = {0};
    2078         cout << elem_vec_map.size() << " vectors" << endl;
    2079         for (int i = 0;i < gids.size(); i++){
    2080                 if (lst[gids[i]] == 0){
    2081                         cout << "gid: " << gids[i] << endl;
    2082                         lst[gids[i]] = 1;
    2083                         vector<char> *tmp = elem_vec_map[gids[i]];
    2084                         cout << "vector size:"<< (*tmp).size()<<endl;
    2085                         if ((*tmp).size()!=0){
    2086                                 cout << "elements: ";
    2087                                 for (int j = 0; j < (*tmp).size();j++){
    2088                                         cout << (int)(*tmp)[j] << " ";
    2089                                 }
    2090                                 cout << endl;
    2091                         }
    2092                         cout << endl;
    2093                 }
    2094         }
    2095 /*
    2096         for(int i = 0;i < gids.size();i++){
    2097                 cout << " i : " << i << endl;
    2098                 vector<short int> *tmp = elem_vec_map[gids[i]];
    2099                 cout << "found" << endl;
    2100                 cout << "map size:" << elem_vec_map.size()<< endl;
    2101                 for (map<short int, vector<short int>* >::iterator pos = elem_vec_map.begin(); pos != elem_vec_map.end(); pos++){
    2102                         cout << "vector size" << (pos->second)->size() << endl;
    2103                 }
    2104                 cout << "size:" << tmp->size()<< endl;
    2105                
    2106                 for (int j = 0;j < tmp->size();j++){
    2107                         cout <<tmp[j]<<" ";
    2108                 }
    2109 
    2110         }
    2111 */
    2112 }
    2113 
    2114 
    2115 static inline void print_elem_vec(vector<char>* vec, int size){
    2116         cout << "vector: ";
    2117         for (int i = 0; i<size ; i++){
    2118                 cout << (int)(*vec)[i] <<" ";
    2119         }
    2120         cout << endl;
    2121 }
    2122 
    2123 static inline void validation_do_block(Lex & lex, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail){
    2124         elem_starts = tag_Callouts.ElemName_starts_no_ns;
    2125 //      print_register("starts",tag_Callouts.ElemName_starts_no_ns);
    2126 //      print_register("ends",tag_Callouts.ElemName_ends);
    2127 //      print_register("cont start",tag_Callouts.ElemContent_starts);
    2128         hashvalues[1] = hash_data.Hash_value;
    2129         StreamScanLengthGrouping(tag_Callouts,sizeof(BitBlock)/sizeof(ScanBlock));
    2130 //      StreamScanLengthGrouping((ScanBlock *) &tag_Callouts.ElemName_ends,(ScanBlock *) &tag_Callouts.Tag_closing, sizeof(BitBlock)/sizeof(ScanBlock));
    2131        
    2132         previous_block_last_elem_start = - count_reverse_zeroes (elem_starts);
    2133 //      cout << previous_block_last_elem_start << endl;
    2134 }
    2135 
    21362254static inline void print_basis_bits(Basis_bits & basis_bits){
    21372255        print_register("bit0:",basis_bits.bit_0);
     
    21452263}
    21462264
    2147 static inline void validate_city_gml(){
    2148         vector<char>* vector_11 = elem_vec_map[2];
    2149         cout << "vector_11 size " << (*vector_11).size() << endl;
    2150         print_elem_vec(vector_11, (*vector_11).size());
    2151         int lgth = (*vector_11).size();
     2265static inline void print_elem_vec(vector<char>* vec, int size){
     2266        cout << "vector: ";
     2267        for (int i = 0; i<size ; i++){
     2268                cout << (int)(*vec)[i] <<" ";
     2269        }
     2270        cout << endl;
     2271}
     2272
     2273
     2274static inline void validate(){
     2275        struct Elem elem;
     2276        Validate_Elem_Vec validate_Elem_Vec;
     2277        cout << "vector size " << gid_vec.size() << endl;
     2278        int lgth = gid_vec.size();
    21522279        int remaining = 0;
    2153         Basis_bits basis_bits;
     2280        Basis_bits temp_basis_bits;
    21542281        if (lgth != 0){
    21552282                int i = 0;
    21562283                int num_block = lgth /(BLOCK_SIZE *8 / VECTOR_WIDTH);
    21572284                remaining = lgth %(BLOCK_SIZE *8 / VECTOR_WIDTH);
    2158                 cout << "num_block " << num_block << " remaining " << remaining << endl;
    2159                 for (i = 0;i < num_block;){
     2285                for (i = 0;i < num_block; i++){
    21602286//                      s2p_do_block((BytePack*)(vector_11[i]),temp_basis_bits);
    2161                         s2p_do_block(((BytePack*)(&(*vector_11)[i])), basis_bits);
    2162                         Citygml citygml;
    2163                         validation_Vec_2.do_block(citygml, basis_bits);
    2164                         print_elem_vec(vector_11,lgth);
    2165                         print_basis_bits(basis_bits);
     2287                        s2p_do_block((BytePack*)(&(gid_vec[i])), temp_basis_bits);
     2288                        print_elem_vec(&(gid_vec),lgth);
     2289                        print_basis_bits(temp_basis_bits);
    21662290//                      (BytePack*)(iter->second)
    2167                         i += (BLOCK_SIZE *8 / VECTOR_WIDTH);
     2291                        validate_Elem_Vec.do_block(elem, temp_basis_bits);
    21682292                        cout << i ;
    21692293                }
    2170                 cout << i << endl;
     2294                cout << endl;
    21712295                if (remaining !=0){
    2172                         EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
    2173                         print_register("eof", EOF_mask);
    2174                         s2p_do_final_block(((BytePack*)(&(*vector_11)[i])), basis_bits,EOF_mask);
    2175                         validation_Vec_2.do_final_block(citygml,basis_bits,EOF_mask);
    2176                         print_elem_vec((vector_11)+i,lgth);
    2177                         print_basis_bits(basis_bits);
     2296                        BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(),convert(BLOCK_SIZE-remaining));
     2297                        s2p_do_final_block((BytePack*)(&(gid_vec[i])), temp_basis_bits,EOF_mask);
     2298                        print_elem_vec(&(gid_vec),lgth);
     2299                        print_basis_bits(temp_basis_bits);
     2300                        validate_Elem_Vec.do_final_block(elem, temp_basis_bits, EOF_mask);
    21782301                }
    21792302        }
    2180 //      print_bit_block("error",error);
    2181         cout << "validation vec_11 done" << endl;
     2303/*
     2304        vector<int> test;
     2305        for(int i = 1;i <12; i++){
     2306                test.push_back(i);
     2307        }
     2308//      int test[] = {1,2,3,4,5,6,7,8,9,10,11};
     2309        Basis_bits temp_basis_bits;
     2310        BitBlock EOF_mask = sisd_srl(simd_const_1(1),sisd_from_int(BLOCK_SIZE-44));
     2311        s2p_do_final_block((BytePack*)(&(test[0])), temp_basis_bits,EOF_mask);
     2312        print_basis_bits(temp_basis_bits);
     2313*/
     2314/*
     2315        for (map<char, vector<char>* > ::const_iterator iter = elem_vec_map.begin(); iter!= elem_vec_map.end(); ++iter){
     2316                int i = 0;
     2317                int lgth = (*(iter->second)).size();
     2318                int remaining;
     2319                if (lgth != 0){
     2320                        cout <<"vector key "<< (int)iter->first << endl;
     2321//                      cout <<"vector ptr" << iter->second<<endl;
     2322               
     2323                        cout <<"vector size: " << lgth << endl;
     2324                        int num_block = lgth /(BLOCK_SIZE *8 / VECTOR_WIDTH);
     2325                        remaining = lgth %(BLOCK_SIZE *8 / VECTOR_WIDTH);
     2326                        for (i = 0;i < num_block; i++){
     2327                                Basis_bits temp_basis_bits;
     2328                                s2p_do_block(((BytePack*)(&(*(iter->second))[i])), temp_basis_bits);
     2329                                print_elem_vec(iter->second,lgth);
     2330                                print_basis_bits(temp_basis_bits);
     2331//                              (BytePack*)(iter->second)
     2332                                cout << i ;
     2333                        }
     2334                        cout << endl;
     2335                        if (remaining !=0){
     2336                                BitBlock EOF_mask = sisd_srl(simd_const_1(1),sisd_from_int(BLOCK_SIZE-remaining));
     2337                                Basis_bits temp_basis_bits;
     2338                                s2p_do_final_block(((BytePack*)(&(*(iter->second))[i])), temp_basis_bits,EOF_mask);
     2339                                print_elem_vec((iter->second)+i,lgth);
     2340                                print_basis_bits(temp_basis_bits);
     2341                        }
     2342                }                       
     2343        }
     2344*/
    21822345}
    21832346
    2184 
    2185 
    2186 
    2187 static inline void print_streams(char * s1,BitBlock & block,char * name){
    2188         int lgth = 2;
    2189         int step = 128/lgth;
    2190         char tmp[step+1];
    2191         char test = s1[0];
     2347static inline void vectoring(Tag_Callouts & tag_Callouts, Hash_data & hash_data,char *source,int block_base){
     2348
     2349        elem_starts = tag_Callouts.ElemName_starts;
     2350    hashvalues[1] = hash_data.Hash_value;
     2351   
     2352        StreamScanLengthGrouping(source,block_base,(ScanBlock *) &tag_Callouts.ElemName_ends,(ScanBlock *) &tag_Callouts.Tag_closing, sizeof(BitBlock)/sizeof(ScanBlock));
    21922353       
    2193         for (int i = 0;i<128;i=i+step){
    2194                 printf("%20s = ","Source");
    2195                 for (int j = 0;j<step;j++){
    2196                         if (s1[i+j]!='\n' && s1[i+j]!='\r'){
    2197                                 cout << s1[i+j];
    2198                         }else{
    2199                                 cout << " ";
    2200                         }
     2354        // Store the last starting position in case we hit boundary case
     2355    previous_block_last_elem_start = - count_reverse_zeroes(elem_starts);
     2356
     2357    //copy current hash value data as previous one.
     2358    memmove (&hashvalues[0], &hashvalues[1], 16);
     2359}
     2360/*
     2361static inline void postprocess_do_block(Lex & lex, CtCDPI_Callouts & ctCDPI_Callouts, Ref_Callouts & ref_Callouts, Check_streams & check_streams, Tag_Callouts & tag_Callouts, Hash_data & hash_data, int chars_avail){
     2362
     2363
     2364    //NOTE: We are NOT handling the case for an extremely long symbol name (length > 1541).
     2365    //      TagMatcher will crash if we feed a long symbol name.
     2366    //      Sample file: test/long_sym_name.xml
     2367
     2368    tracker.StoreNewlines(lex.LF);
     2369    elem_starts = tag_Callouts.ElemName_starts;
     2370    hashvalues[1] = hash_data.Hash_value;
     2371
     2372        StreamScanLengthGrouping((ScanBlock *) &tag_Callouts.ElemName_ends,(ScanBlock *) &tag_Callouts.Tag_closing, sizeof(BitBlock)/sizeof(ScanBlock));
     2373       
     2374/*
     2375    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_1) )
     2376    {
     2377        StreamScanLengthGrouping<1>((ScanBlock *) &tag_Callouts.ElemName_ends_1,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2378    }
     2379
     2380    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_2) )
     2381    {
     2382        StreamScanLengthGrouping<2>((ScanBlock *) &tag_Callouts.ElemName_ends_2,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2383    }
     2384
     2385    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_3) )
     2386    {
     2387        StreamScanLengthGrouping<3>((ScanBlock *) &tag_Callouts.ElemName_ends_3,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2388    }
     2389
     2390    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_4) )
     2391    {
     2392        StreamScanLengthGrouping<4>((ScanBlock *) &tag_Callouts.ElemName_ends_4,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2393    }
     2394
     2395    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_5) )
     2396    {
     2397        StreamScanLengthGrouping<5>((ScanBlock *) &tag_Callouts.ElemName_ends_5,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2398    }
     2399
     2400    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_6) )
     2401    {
     2402        StreamScanLengthGrouping<6>((ScanBlock *) &tag_Callouts.ElemName_ends_6,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2403    }
     2404
     2405    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_7) )
     2406    {
     2407        StreamScanLengthGrouping<7>((ScanBlock *) &tag_Callouts.ElemName_ends_7,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2408    }
     2409
     2410    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_8) )
     2411    {
     2412        StreamScanLengthGrouping<8>((ScanBlock *) &tag_Callouts.ElemName_ends_8,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2413    }
     2414
     2415    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_9) )
     2416    {
     2417        StreamScanLengthGrouping<9>((ScanBlock *) &tag_Callouts.ElemName_ends_9,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2418    }
     2419
     2420    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_10) )
     2421    {
     2422        StreamScanLengthGrouping<10>((ScanBlock *) &tag_Callouts.ElemName_ends_10,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2423    }
     2424
     2425    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_11) )
     2426    {
     2427        StreamScanLengthGrouping<11>((ScanBlock *) &tag_Callouts.ElemName_ends_11,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2428    }
     2429
     2430    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_12) )
     2431    {
     2432        StreamScanLengthGrouping<12>((ScanBlock *) &tag_Callouts.ElemName_ends_12,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2433    }
     2434
     2435    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_13) )
     2436    {
     2437        StreamScanLengthGrouping<13>((ScanBlock *) &tag_Callouts.ElemName_ends_13,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2438    }
     2439
     2440    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_14) )
     2441    {
     2442        StreamScanLengthGrouping<14>((ScanBlock *) &tag_Callouts.ElemName_ends_14,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2443    }
     2444
     2445    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_15) )
     2446    {
     2447        StreamScanLengthGrouping<15>((ScanBlock *) &tag_Callouts.ElemName_ends_15,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2448    }
     2449
     2450    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_16) )
     2451    {
     2452        StreamScanLengthGrouping<16>((ScanBlock *) &tag_Callouts.ElemName_ends_16,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2453    }
     2454
     2455    if ( bitblock_has_bit(tag_Callouts.ElemName_ends_17_and_longer) )
     2456    {
     2457        StreamScanLengthGrouping<17>((ScanBlock *) &tag_Callouts.ElemName_ends_17_and_longer,(ScanBlock *) &tag_Callouts.EndTag_marks, sizeof(BitBlock)/sizeof(ScanBlock));
     2458    }
     2459
     2460*/
     2461
     2462
     2463/*
     2464    // Store the last starting position in case we hit boundary case
     2465    previous_block_last_elem_start = - count_reverse_zeroes(elem_starts);
     2466
     2467    //copy current hash value data as previous one.
     2468    memmove (&hashvalues[0], &hashvalues[1], 16);
     2469
     2470    if (bitblock::any(simd_or(check_streams.non_ascii_name_starts, check_streams.non_ascii_names))) {
     2471      StreamScan((ScanBlock *) &check_streams.non_ascii_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), NameStrt_check);
     2472      StreamScan((ScanBlock *) &check_streams.non_ascii_names, sizeof(BitBlock)/sizeof(ScanBlock), Name_check);
     2473    }
     2474
     2475    if (bitblock::any(ctCDPI_Callouts.PI_name_starts)){
     2476      StreamScan((ScanBlock *) &ctCDPI_Callouts.PI_name_starts, sizeof(BitBlock)/sizeof(ScanBlock), PIName_check);
     2477    }
     2478
     2479    if (bitblock::any(ctCDPI_Callouts.CD_starts)){
     2480      StreamScan((ScanBlock *) &ctCDPI_Callouts.CD_starts, sizeof(BitBlock)/sizeof(ScanBlock), CD_check);
     2481    }
     2482
     2483    if (bitblock::any(ref_Callouts.GenRef_starts)){
     2484      StreamScan((ScanBlock *) &ref_Callouts.GenRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), GenRef_check);
     2485    }
     2486
     2487    if (bitblock::any(ref_Callouts.DecRef_starts)){
     2488      StreamScan((ScanBlock *) &ref_Callouts.DecRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), DecRef_check);
     2489    }
     2490
     2491    if (bitblock::any(ref_Callouts.HexRef_starts)){
     2492      StreamScan((ScanBlock *) &ref_Callouts.HexRef_starts, sizeof(BitBlock)/sizeof(ScanBlock), HexRef_check);
     2493    }
     2494
     2495    if (bitblock::any(check_streams.att_refs)){
     2496      StreamScan((ScanBlock *) &check_streams.att_refs, sizeof(BitBlock)/sizeof(ScanBlock), AttRef_check);
     2497    }
     2498
     2499//    error_tracker.If_Error_Report_First();
     2500/*
     2501    matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
     2502    tracker.AdvanceBlock();
     2503    */
     2504    /*
     2505}
     2506*/
     2507
     2508static inline void print_GIDS()
     2509{
     2510    int span_count = gids.size();
     2511    for(int i=0;i<span_count;i++) {
     2512             cout << gids[i] << " ";
     2513    }
     2514    cout << endl;
     2515}
     2516
     2517static inline int test(int)
     2518{
     2519    return 0;
     2520}
     2521
     2522void do_process(FILE *infile, FILE *outfile) {
     2523
     2524        ///////////////
     2525        //gid file
     2526        ///////////////
     2527        gid_writein = fopen("gid.out","w");
     2528  struct U8 u8;
     2529
     2530  struct Lex lex;
     2531
     2532  struct Scope1 scope1;
     2533
     2534  struct Marker marker;
     2535
     2536  struct CtCDPI_Callouts ctCDPI_Callouts;
     2537
     2538  struct Ref_Callouts ref_Callouts;
     2539
     2540  struct Hash_data hash_data;
     2541
     2542  struct Tag_Callouts tag_Callouts;
     2543
     2544  struct Basis_bits basis_bits;
     2545
     2546  struct Check_streams check_streams;
     2547
     2548  struct Xml_names xml_names;
     2549
     2550  struct Elem elem;
     2551
     2552
     2553
     2554        #ifndef STL_ALIGNED_VECTOR
     2555                TagMatcher<SEGMENT_SIZE,LOOKAHEAD_SIZE> matcher;
     2556        #endif
     2557        ///////////////////////////////////////////////////
     2558        //preload element_IDs into symbol table from schema
     2559        ///////////////////////////////////////////////////
     2560        FILE *element_ID_in = fopen("../test/element_ID","r");
     2561        char symble[50];
     2562        int index;
     2563        if(element_ID_in == NULL){
     2564                printf("Error opening element_ID file.\n");
     2565                exit(0);
     2566        }
     2567        hashvalues[1] = hash_data.Hash_value;
     2568        while(fscanf(element_ID_in,"%s %d\n",&symble[0],&index)==2){
     2569                int lgth = strlen(symble);
     2570                int hashvalue = compute_hash_value(lgth,1);
     2571                cout << symble <<" "<< lgth<<" "<<hashvalue << endl;
     2572                int gid = 0;
     2573                switch (lgth)
     2574        {
     2575                        case 1:
     2576                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<1>(symble, hashvalue);
     2577                        break;
     2578                        case 2:
     2579                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<2>(symble, hashvalue);
     2580                        break;
     2581                        case 3:
     2582                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<3>(symble, hashvalue);
     2583                        break;
     2584                        case 4:
     2585                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<4>(symble, hashvalue);
     2586                        break;
     2587                        case 5:
     2588                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<5>(symble, hashvalue);
     2589                        break;
     2590                        case 6:
     2591                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<6>(symble, hashvalue);
     2592                        break;
     2593                        case 7:
     2594                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<7>(symble, hashvalue);
     2595                        break;
     2596                        case 8:
     2597                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<8>(symble, hashvalue);
     2598                        break;
     2599                        case 9:
     2600                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<9>(symble, hashvalue);
     2601                        break;
     2602                        case 10:
     2603                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<10>(symble, hashvalue);
     2604                        break;
     2605                        case 11:
     2606                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<11>(symble, hashvalue);
     2607                        break;
     2608                        case 12:
     2609                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<12>(symble, hashvalue);
     2610                        break;
     2611                        case 13:
     2612                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<13>(symble, hashvalue);
     2613                        break;
     2614                        case 14:
     2615                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<14>(symble, hashvalue);
     2616                        break;
     2617                        case 15:
     2618                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<15>(symble, hashvalue);
     2619                        break;
     2620                        case 16:
     2621                        gid = pbgs_symbol_table.Lookup_or_Insert_Name<16>(symble, hashvalue);
     2622                        break;
     2623                        default:
     2624                        gid = pbgs_symbol_table.Lookup_or_Insert_Name(symble, hashvalue, lgth);
     2625                        break;         
    22012626                }
    2202                 cout << endl;
    2203                 printf("%20s = ",name);
    2204                 for (int j = 0;j<step;j++){
    2205                         BitBlock temp = simd_and(bitblock::srl(block,convert(i+j)),convert(1));
    2206                         if (bitblock::any(temp)){
    2207                                 cout << 1;
    2208                         }else{
    2209                                 cout << "_";
    2210                         }
     2627                cout << "gid=" << gid << endl;
     2628        }       
     2629        fclose(element_ID_in);
     2630        return;
     2631
     2632        int buffer_base = 0;
     2633        int block_base = 0;
     2634        int buffer_pos = 0;
     2635        int block_pos = 0;
     2636        int errpos = 0;
     2637        int chars_avail = 0;
     2638        int check_pos = 0;
     2639        int chars_read = 0;
     2640 
     2641        //////////////////////////////////////////////////////////////////////////////////////////
     2642    // Buffer Management
     2643    //////////////////////////////////////////////////////////////////////////////////////////
     2644                //      BitBlock buf[(BUFFER_SIZE)/sizeof(BitBlock)];
     2645                //      uint8_t * src_buffer = (uint8_t *)buf + COPYBACK_SIZE;
     2646
     2647                #ifdef STL_ALIGNED_VECTOR
     2648                        std::vector<uint8_t, AAllocator<uint8_t> > src_buffer;
     2649                        src_buffer.reserve(BUFFER_SIZE);
     2650                #else
     2651                        uint8_t * COPYBACK;
     2652                        uint8_t * src_buffer;
     2653                        ALLOC_STATIC_ALIGNED_BYTE_BUFFER_WITH_COPYBACK(COPYBACK, src_buffer);
     2654                #endif
     2655  /*
     2656  BytePack buf[(BUFFER_SIZE+BLOCK_SIZE+LOOKAHEAD_SIZE*2)/sizeof(SIMD_type)];
     2657
     2658  char * srcbuf = ((char *) buf) + LOOKAHEAD_SIZE;
     2659  buffer_base = buf_pos;
     2660  source = srcbuf;
     2661  */
     2662  chars_read = fread((void *)&src_buffer[0], 1, SEGMENT_SIZE, infile);
     2663  chars_avail = chars_read;
     2664  if (chars_avail > BUFFER_SIZE) chars_avail = SEGMENT_SIZE;
     2665
     2666  if(chars_read<4){
     2667    fprintf(stderr,"File is too short. Not well formed.\n");
     2668    exit(-1);
     2669  }
     2670
     2671  Entity_Info * e = new Entity_Info;
     2672  e->AnalyzeSignature((unsigned char *)&src_buffer[0]);
     2673
     2674  if (e->code_unit_base == ASCII) {
     2675
     2676    XML_Decl_Parser<ASCII> decl_parser((unsigned char *)&src_buffer[0]);
     2677
     2678    decl_parser.ReadXMLInfo(*e);
     2679
     2680    if (e->code_unit_size != SingleByte || (e->has_encoding_decl && (!at_UTF_8(e->encoding)))){
     2681        fprintf(stderr,"Sorry, this xmlwf demo only works for UTF-8.\n");
     2682        exit(-1);
     2683    }
     2684  }
     2685  else {
     2686    fprintf(stderr,"Sorry, this xmlwf demo does not process EBCDIC.\n");
     2687        exit(-1);
     2688  }
     2689
     2690  if (e->content_start != 0) {
     2691        memmove(&src_buffer[0], &src_buffer[e->content_start], chars_read - e->content_start);
     2692        buffer_pos = e->content_start;
     2693//      buffer_base = buffer_pos;
     2694        if ((chars_avail-e->content_start) < SEGMENT_SIZE) {
     2695                        chars_read = chars_avail - e->content_start + fread(&src_buffer[chars_avail-e->content_start], 1, e->content_start, infile);
     2696                        chars_avail = chars_read;
    22112697                }
    2212                 cout << endl;
    2213                 cout << endl;
     2698                if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
     2699  }
     2700
     2701        //////////////////////////////////////////////////////////////////////////////////////////
     2702        // Read OVERLAP bytes to support post processing validation lookahead.
     2703        //////////////////////////////////////////////////////////////////////////////////////////
     2704        chars_read = chars_avail + fread(&src_buffer[chars_avail], 1, LOOKAHEAD_SIZE, infile);
     2705        chars_avail = chars_read;
     2706        if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
     2707
     2708  Parse_refs parse_refs;
     2709  Validate_Elem_Vec validate_Elem_Vec;
     2710  Parse_tags parse_tags;
     2711  Classify_bytes_Validate_utf8 classify_bytes_Validate_utf8;
     2712  Parse_CtCDPI parse_CtCDPI;
     2713  Do_check_streams do_check_streams;
     2714  Form_Length_Group_Bitstreams form_Length_Group_Bitstreams;
     2715  Validate_xml_names validate_xml_names;
     2716  Compute_Hash_Value_Bitstream compute_Hash_Value_Bitstream;
     2717  Add_scope_streams add_scope_streams;
     2718
     2719
     2720        #ifndef STL_ALIGNED_VECTOR
     2721        matcher.setSrc((char *)&src_buffer[0]);
     2722        #endif
     2723
     2724/* Full Buffers */
     2725    int block_segment_num = 0;
     2726    while (chars_avail >= SEGMENT_SIZE) {
     2727      PERF_SEC_START(parser_timer);
     2728      for (int blk = 0; blk < SEGMENT_BLOCKS; blk++) {
     2729        cout << "loops" << ends;
     2730                block_base = blk*BLOCK_SIZE;
     2731        s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits);
     2732     
     2733               
     2734  classify_bytes_Validate_utf8.do_block(basis_bits, lex, u8);
     2735  add_scope_streams.do_block(lex, scope1);
     2736  parse_CtCDPI.do_block(lex, marker, ctCDPI_Callouts, check_streams);
     2737  parse_tags.do_block(lex, marker, tag_Callouts);
     2738  parse_refs.do_block(lex, marker, ref_Callouts);
     2739  validate_xml_names.do_block(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, check_streams);
     2740  do_check_streams.do_block(marker, tag_Callouts, check_streams);
     2741  form_Length_Group_Bitstreams.do_block(tag_Callouts);
     2742  compute_Hash_Value_Bitstream.do_block(hash_data, basis_bits);
     2743         
     2744                tracker.StoreNewlines(lex.LF);
     2745                cout << "post start" << endl;
     2746//              postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, tag_Callouts, hash_data, chars_avail);
     2747                postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)&src_buffer[0], buffer_base, block_base, chars_avail, tracker);
     2748                vectoring(tag_Callouts,hash_data,(char *)&src_buffer[0],block_base);
     2749                cout << "post done" << endl;
     2750                #ifndef STL_ALIGNED_VECTOR
     2751                matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
     2752                #endif
     2753                tracker.AdvanceBlock();
     2754                cout << "loop" << endl;
     2755      }
     2756      #ifndef STL_ALIGNED_VECTOR
     2757      matcher.StreamScan(chars_avail);
     2758      matcher.Advance_buffer();
     2759      #endif
     2760      cout << "11111" << endl;
     2761      PERF_SEC_END(parser_timer, chars_avail);
     2762          cout << "here" << endl;
     2763      int bytes_left = chars_read - chars_avail;
     2764      //////////////////
     2765      //element name buffer, for scanning element name backwards
     2766      /////////////////
     2767      memmove(&element_name_buffer[0],&src_buffer[SEGMENT_SIZE-ELEMENT_BUFFER_SIZE],ELEMENT_BUFFER_SIZE);
     2768      cross_buffer_flag = 1;
     2769     
     2770      memmove(&src_buffer[0], &src_buffer[SEGMENT_SIZE], LOOKAHEAD_SIZE);
     2771      chars_read = fread(&src_buffer[LOOKAHEAD_SIZE],1,SEGMENT_SIZE, infile) + LOOKAHEAD_SIZE;
     2772      chars_avail = chars_read;
     2773      cout << "here" << endl;
     2774      if (chars_avail > SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
     2775      buffer_pos += chars_avail;
     2776      buffer_base = buffer_pos;
     2777      block_segment_num++;
     2778
     2779  }
     2780       
     2781/* Final Partial Buffer */
     2782    PERF_SEC_START(parser_timer);
     2783
     2784    block_pos = 0;
     2785    int remaining = chars_avail;
     2786
     2787
     2788
     2789/* Full Blocks */
     2790    while (remaining >= BLOCK_SIZE) {
     2791          block_base = block_pos;
     2792          s2p_do_block((BytePack *) &src_buffer[block_pos], basis_bits);
     2793         
     2794         
     2795  classify_bytes_Validate_utf8.do_block(basis_bits, lex, u8);
     2796  add_scope_streams.do_block(lex, scope1);
     2797  parse_CtCDPI.do_block(lex, marker, ctCDPI_Callouts, check_streams);
     2798  parse_tags.do_block(lex, marker, tag_Callouts);
     2799  parse_refs.do_block(lex, marker, ref_Callouts);
     2800  validate_xml_names.do_block(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, check_streams);
     2801  do_check_streams.do_block(marker, tag_Callouts, check_streams);
     2802  form_Length_Group_Bitstreams.do_block(tag_Callouts);
     2803  compute_Hash_Value_Bitstream.do_block(hash_data, basis_bits);
     2804         
     2805          tracker.StoreNewlines(lex.LF);
     2806          cout << "post start1" << endl;
     2807         
     2808//        postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, tag_Callouts, hash_data, chars_avail);
     2809          postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)&src_buffer[0], buffer_base, block_base, chars_avail, tracker);
     2810          vectoring(tag_Callouts,hash_data,(char *)&src_buffer[0],block_base);
     2811          cout << "post end2" << endl;
     2812          #ifndef STL_ALIGNED_VECTOR
     2813          matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
     2814          #endif
     2815          tracker.AdvanceBlock();
     2816          block_pos += BLOCK_SIZE;
     2817          remaining -= BLOCK_SIZE;
     2818    }
     2819    block_base = block_pos;
     2820        #ifdef USE_ANY_CARRY_TEST
     2821    if (remaining > 0 || parse_refs.carryQ.CarryTest(0, 6) || 0 || parse_tags.carryQ.CarryTest(0, 13) || classify_bytes_Validate_utf8.carryQ.CarryTest(0, 12) || parse_CtCDPI.carryQ.CarryTest(0, 19) || 0 || form_Length_Group_Bitstreams.carryQ.CarryTest(0, 16) || validate_xml_names.carryQ.CarryTest(0, 9) || 0 || add_scope_streams.carryQ.CarryTest(0, 2)) {
     2822        #endif
     2823          EOF_mask = bitblock::srl(simd<1>::constant<1>(),convert(BLOCK_SIZE-remaining));
     2824          s2p_do_final_block((BytePack *) &src_buffer[block_pos], basis_bits, EOF_mask);
     2825         
     2826  classify_bytes_Validate_utf8.do_final_block(basis_bits, lex, u8, EOF_mask);
     2827  add_scope_streams.do_final_block(lex, scope1, EOF_mask);
     2828  parse_CtCDPI.do_final_block(lex, marker, ctCDPI_Callouts, check_streams, EOF_mask);
     2829  parse_tags.do_final_block(lex, marker, tag_Callouts, EOF_mask);
     2830  parse_refs.do_final_block(lex, marker, ref_Callouts, EOF_mask);
     2831  validate_xml_names.do_final_block(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, check_streams, EOF_mask);
     2832  do_check_streams.do_final_block(marker, tag_Callouts, check_streams, EOF_mask);
     2833  form_Length_Group_Bitstreams.do_final_block(tag_Callouts, EOF_mask);
     2834  compute_Hash_Value_Bitstream.do_final_block(hash_data, basis_bits, EOF_mask);
     2835          cout << "post start3" << endl;
     2836//        postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, tag_Callouts, hash_data, chars_avail);
     2837          postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)&src_buffer[0], buffer_base, block_base, chars_avail, tracker);
     2838          vectoring(tag_Callouts,hash_data,(char *)&src_buffer[0],block_base);
     2839          cout << "post end3" << endl;
     2840          #ifndef STL_ALIGNED_VECTOR
     2841      matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
     2842          #endif
     2843          tracker.AdvanceBlock();
     2844        #ifdef USE_ANY_CARRY_TEST
     2845    }
     2846    #endif
     2847    buffer_pos += chars_avail;
     2848    buffer_base = buffer_pos;
     2849        #ifndef STL_ALIGNED_VECTOR
     2850    matcher.StreamScan(chars_avail);
     2851    matcher.Advance_buffer();
     2852//    tracker.Advance_buffer();
     2853
     2854   
     2855    if (matcher.depth != 0) {
     2856      fprintf(stderr, "tag matching error (depth %i) at position %i\n", matcher.depth, buffer_base);
     2857      exit(-1);
     2858     
     2859     
     2860    }
     2861    #endif
     2862    PERF_SEC_END(parser_timer, chars_avail);
     2863//  print_GIDS();
     2864//    pbgs_symbol_table.Print_Symbol_Table_Distribution();
     2865}
     2866
     2867
     2868
     2869int
     2870main(int argc, char * argv[]) {
     2871        char * infilename, * outfilename;
     2872        FILE *infile, *outfile;
     2873        struct stat fileinfo;
     2874
     2875        if (argc < 2) {
     2876                printf("Usage: %s <filename> [<outputfile>]\n", argv[0]);
     2877                exit(-1);
    22142878        }
    2215 }
    2216 
    2217 static inline void print_buffer(char * block, char * name){
    2218         int step = 128/2;
    2219         char *ptr = (char*)block;
    2220         for (int i = 0;i<128;i=i+step){
    2221                 printf("%20s = ",name);
    2222                 for (int j = 0;j<step;j++){
    2223                         if (ptr[i+j]!='\n' && ptr[i+j]!='\r'){
    2224                                 cout << ptr[i+j];
    2225                         }else{
    2226                                 cout << " ";
    2227                         }
    2228                 }
    2229                 cout << endl;
    2230                
     2879
     2880        infilename = argv[1];
     2881        stat(infilename, &fileinfo);
     2882        infile = fopen(infilename, "rb");
     2883        if (!infile) {
     2884                fprintf(stderr, "Error: cannot open %s for input.\n", infilename);
     2885                exit(-1);
    22312886        }
    2232         cout << endl;
    2233 }
    2234 
    2235 static inline void val_double_list(Tag_Callouts & tag_Callouts,int elem_end,int block_pos,int blk){
    2236         ScanBlock *content_start = (ScanBlock *) &tag_Callouts.ElemContent_starts;
    2237         ScanBlock start_blk =content_start[blk];
    2238         ScanBlock *content_end = (ScanBlock *) &tag_Callouts.ElemContent_ends;
    2239         ScanBlock end_blk = content_end[blk];
    2240         int sb_size = sizeof(ScanBlock)*8;
    2241         //print_streams(&srcbuf[block_base],tag_Callouts.ElemContent_starts,"elem_content_starts");
    2242         //print_streams(&srcbuf[block_base],tag_Callouts.ElemContent_ends,"elem_content_ends");
    2243         //cout << "block num = " << blk << endl;
    2244         //cout << "elem_end="<<elem_end<<endl<<endl;
    2245         //cout << "flag = " << content_start_flag << endl;
    2246         if (elem_end == -1){//content_start_pos has occured
    2247                 if (end_blk == 0){
    2248                         //print_buffer(val_buf,"buffer");
    2249                         if (sb_size < val_buf_avail){
    2250                                 memcpy(&val_buf[val_buf_curser],source+block_pos,sb_size);
    2251                                 val_buf_avail -= sb_size;
    2252                                 val_buf_curser += sb_size;
    2253                         }else{
    2254                                 //cout << "buf_avail = " << val_buf_avail << endl;
    2255                                 memcpy(&val_buf[val_buf_curser],source+block_pos,val_buf_avail);
    2256                                 int char_left = sb_size - val_buf_avail;
    2257                                 val_dl_block();
    2258                                 memcpy(val_buf,source+block_pos+val_buf_avail, char_left);
    2259                                 val_buf_avail = sizeof(BitBlock)*8 - char_left;
    2260                                 val_buf_curser = char_left;
    2261                         }
    2262                 }else{
    2263                         //cout << "HHHHHHHHHH" << endl;
    2264                         int end_pos = __builtin_ctzll(end_blk);
    2265                         //cout << "end_pos = "<<end_pos << endl;
    2266                         //cout <<"buf_avail = "<<val_buf_avail<<endl;
    2267                         if (end_pos > val_buf_avail){
    2268                                 memcpy(&val_buf[val_buf_curser],source+block_pos,val_buf_avail);
    2269                                 int char_left = end_pos - val_buf_avail;
    2270                                 val_dl_block();
    2271                                 if (char_left != 0){
    2272                                         memcpy(val_buf,source+block_pos+val_buf_avail, char_left);
    2273                                 }
    2274                                 val_buf_avail = sizeof(BitBlock)*8 - char_left;
    2275                                 val_buf_curser = char_left;
    2276                         }else{
    2277                                 memcpy(&val_buf[val_buf_curser],source+block_pos,end_pos);
    2278                                 //cout <<"source "<< source[block_pos]<<source[block_pos+1]<<source[block_pos+2]<<endl;
    2279                                 val_buf_avail = val_buf_avail - end_pos;
    2280                                 val_buf_curser = val_buf_curser + end_pos;
    2281                         }
    2282                         content_start_flag = 0;
    2283                         //print_buffer(val_buf,"buffer");
    2284                         //cout << "s = " << sizeof(val_buf) << endl;
    2285                         //cout << "curser = " << val_buf_curser << endl;
    2286                 }
    2287         }else if(elem_end == -2){
    2288                 if (start_blk != 0){
    2289                         int start_pos = __builtin_ctzll(start_blk);
    2290                         int end_pos = 0;
    2291                         if (end_blk == 0){
    2292                                 end_pos = sb_size;
    2293                         }else{
    2294                                 end_pos = __builtin_ctzll(end_blk);
    2295                         //cout << "s_p:e_p" << start_pos << " " << end_pos << endl;
    2296                                 while(end_pos < start_pos){
    2297                                         end_blk = end_blk & (end_blk - 1);
    2298                                         end_pos = __builtin_ctzll(end_blk);
    2299                                 }
    2300                         }
    2301                         //cout << "s_p:e_p" << start_pos << " " << end_pos << endl;
    2302                         int char_left = end_pos - start_pos;
    2303                         //cout << "buf_avail " << val_buf_avail<< endl;
    2304                         //cout << "char left " << char_left << endl;
    2305                         if (val_buf_avail > char_left){
    2306                                 memcpy(&val_buf[val_buf_curser],source+block_pos+start_pos,char_left);
    2307                                 val_buf_avail = val_buf_avail - char_left;
    2308                                 val_buf_curser = val_buf_curser + char_left;
    2309                         }else{
    2310                                 memcpy(&val_buf[val_buf_curser],source+block_pos+start_pos,val_buf_avail);
    2311                                 val_dl_block();
    2312                                 char_left -= val_buf_avail;
    2313                                 val_buf_avail = sb_size - char_left + val_buf_avail;
    2314                                 if (char_left != 0){
    2315                                         memcpy(val_buf,source+block_pos+start_pos+ val_buf_avail,char_left);
    2316                                 }
    2317                                 val_buf_curser = char_left;
    2318                         }
    2319                         if (end_pos != sb_size){
    2320                                 content_start_flag = 0;
    2321                         }else{
    2322                                 content_start_flag = -1;
    2323                         }                       
    2324                 }else{
    2325                         content_start_flag = 1;
    2326                         return;
    2327                 }
    2328                 //print_buffer(val_buf,"buffer");
    2329         }else{
    2330                 //cout << "buf_avail " << val_buf_avail<< endl;
    2331                 if (start_blk == 0){
    2332                         content_start_flag = 1;
    2333                         return;
    2334                 }else{
    2335                         //cout << "#curser=" << val_buf_curser<<endl;
    2336                         //cout << "#block_pos=" << block_pos<<endl;
    2337                         int start_pos = __builtin_ctzll(start_blk);
    2338                         while(start_pos <= elem_end){                           //?
    2339                                 start_blk = start_blk & (start_blk -1);
    2340                                 start_pos = __builtin_ctzll(start_blk);
    2341                         }
    2342                         if (start_pos == sb_size){
    2343                                 content_start_flag = -1;
    2344                                 return;
    2345                         }
    2346                         int end_pos = 0;
    2347                         if (end_blk == 0){
    2348                                 end_pos = sb_size;
    2349                         }else{
    2350                                 end_pos = __builtin_ctzll(end_blk);
    2351                                 while(end_pos < start_pos){
    2352                                         end_blk = end_blk & (end_blk - 1);
    2353                                         end_pos = __builtin_ctzll(end_blk);
    2354                                 }
    2355                         }
    2356                         int char_left = end_pos - start_pos;
    2357                         if (val_buf_avail > char_left){
    2358                                 memcpy(&val_buf[val_buf_curser],source+block_pos+start_pos,char_left);
    2359                                 val_buf_avail = val_buf_avail - char_left;
    2360                                 val_buf_curser = val_buf_curser + char_left;
    2361                         }else{
    2362                                 memcpy(&val_buf[val_buf_curser],source+block_pos+start_pos,val_buf_avail);
    2363                                 val_dl_block();
    2364                                 char_left -= val_buf_avail;
    2365                                 val_buf_avail = sb_size - char_left + val_buf_avail;
    2366                                 val_buf_curser = 0;
    2367                                 if(char_left != 0){
    2368                                         memcpy(val_buf,source+block_pos+start_pos+ val_buf_avail,char_left);
    2369                                 }
    2370                                 val_buf_curser = char_left;
    2371                         }
    2372                         if (end_pos == sb_size){
    2373                                 content_start_flag = 1;
    2374                         }
    2375                         //cout << "start" << start_pos << endl;
    2376                         //print_buffer(val_buf,"buffer");
     2887
     2888        if (argc < 3) outfile = stdout;
     2889        else {
     2890                outfilename = argv[2];
     2891                outfile = fopen(outfilename, "wb");
     2892                if (!outfile) {
     2893                        fprintf(stderr, "Error: cannot open %s for writing.\n", outfilename);
     2894                        exit(-1);
    23772895                }
    23782896        }
     2897
     2898//      PERF_SEC_BIND(1);
     2899
     2900        PERF_SEC_INIT(parser_timer);
     2901
     2902        do_process(infile, outfile);
     2903       
     2904#if DEBUG
     2905print_gid_vec();
     2906#endif
     2907        //cout << "validate start" << endl;
     2908        validate();
     2909
     2910        PERF_SEC_DUMP(parser_timer);
     2911
     2912        PERF_SEC_DESTROY(parser_timer);
     2913
     2914        fclose(infile);
     2915        fclose(outfile);
     2916
     2917        printf ("Done procressing\n");
     2918        return(0);
    23792919}
    2380 
    2381 static inline void val_dl_block(){
    2382         print_buffer(val_buf,"buffer");
    2383         //cout << "vvvvvvvvvvvvvvv"<< endl;
    2384 }
    2385 
    2386 
Note: See TracChangeset for help on using the changeset viewer.