Changeset 1442 for proto/SymbolTable


Ignore:
Timestamp:
Sep 14, 2011, 5:43:40 PM (8 years ago)
Author:
vla24
Message:

SymbolTable?: Fixed integration with IDISA, it does not work well for pbgs_log and pbgs_id_adv. Changed bitutil.* to pablo.* for compiler generated files

Location:
proto/SymbolTable
Files:
6 added
5 deleted
11 edited
1 moved

Legend:

Unmodified
Added
Removed
  • proto/SymbolTable/Makefile

    r1232 r1442  
    33PABLO_SRCFILE=../parabix2/parabix2_pablo.py
    44PABLO_COMPILER=../Compiler/pablomain.py
    5 PABLO_SYMTAB_HASH=parabix2_symtab_hash.py
     5PABLO_SYMTAB_ID=parabix2_symtab_id.py
    66PABLO_SYMTAB_PBS=parabix2_symtab_pbs.py
    77PABLO_SYMTAB_PBS_LOG=parabix2_symtab_pbs_log.py
     
    2828        python $(PABLO_COMPILER) $(PABLO_SRCFILE) -t $(HASH_SYMBOLTABLE_TEMPLATE) -o $(OUTFILE)
    2929
    30 symtab_id:      $(PABLO_SYMTAB_HASH) # Paralel bitstream based group sorting
    31         python $(PABLO_COMPILER) $(PABLO_FLAGS) $(PABLO_SYMTAB_HASH) -t $(SYMBOLTABLE_IDENTITY_TEMPLATE) -o $(OUTFILE)
     30symtab_id:      $(PABLO_SYMTAB_ID) # Paralel bitstream based group sorting
     31        python $(PABLO_COMPILER) $(PABLO_FLAGS) $(PABLO_SYMTAB_ID) -t $(SYMBOLTABLE_IDENTITY_TEMPLATE) -o $(OUTFILE)
    3232
    3333symtab_pbgs_id: $(PABLO_SYMTAB_PBS) # Paralel bitstream based group sorting
  • proto/SymbolTable/automate-build.sh

    r1426 r1442  
    1 make symtab_ls
     1make symtab_pbgs_log
    22cd src
    33make all
    4 ./xmlwf ../test/test.xml
     4./xmlwf ../test/test_files/soap.xml
    55cd ..
    66
  • proto/SymbolTable/automate-build_idisa.sh

    r1428 r1442  
    22cd src
    33make idisa
    4 ./xmlwf ../test/soap.xml
     4./xmlwf ../../../trunk/QA/TD/roads-2.gml
    55cd ..
    66
  • proto/SymbolTable/parabix2_symtab_id.py

    r1441 r1442  
    1414
    1515#import bitutil
     16
     17class Basis_bits():
     18        bit_0 = 0
     19        bit_1 = 0
     20        bit_2 = 0
     21        bit_3 = 0
     22        bit_4 = 0
     23        bit_5 = 0
     24        bit_6 = 0
     25        bit_7 = 0
     26
    1627
    1728class u8 ():
     
    7586        WS = 0
    7687
    77 class Scope1 ():
    78         RefStart = 0
    79         LAngle = 0
    80         Hyphen = 0
    81         QMark = 0
    82         RBracket = 0
     88class Marker ():
     89        LAngle_scope = 0
     90        Ref_opener = 0
     91        CD_closer = 0
    8392
    8493class CtCDPI_Callouts():
    85         CD_end = 0
    8694        Ct_starts = 0
    8795        Ct_ends = 0
     
    92100        PI_name_ends = 0
    93101        PI_ends = 0
    94         CtCDPI_mask = 0
    95102
    96103class Ref_Callouts():
     
    116123        EndTag_marks = 0
    117124
    118 class Basis_bits():
    119         bit_0 = 0
    120         bit_1 = 0
    121         bit_2 = 0
    122         bit_3 = 0
    123         bit_4 = 0
    124         bit_5 = 0
    125         bit_6 = 0
    126         bit_7 = 0
    127 
    128125class Check_streams():
    129126        misc_mask = 0
     
    133130        name_follows = 0
    134131        att_refs = 0
    135 
    136 class Xml_names():
    137         namespace_error = 0
    138132
    139133def Classify_bytes_Validate_utf8(basis_bits, lex, u8):
     
    231225        lex.Hex = (temp62 | temp65);
    232226        lex_error = x00_x1F &~ lex.WS
    233         if lex_error & EOF_mask:
     227        if pablo.inFile(lex_error):
    234228                error_tracker.NoteError("Error: illegal character", lex_error)
    235229
     
    255249                u8.badprefix = (temp68 | temp71);
    256250                u8_error = u8.badprefix
    257                 u8.scope22 = bitutil.Advance(u8.prefix2)
     251                u8.scope22 = pablo.Advance(u8.prefix2)
    258252                u8anyscope = u8.scope22
    259253                if u8.prefix3 | u8.prefix4:
     
    271265                        u8.xBF = (temp73 & temp23);
    272266                        u8.xBE = (temp73 & temp15);
    273                         u8.xE0_scope = bitutil.Advance(xE0);
    274                         u8.xED_scope = bitutil.Advance(xED);
    275                         u8.xF0_scope = bitutil.Advance(xF0);
    276                         u8.xF4_scope = bitutil.Advance(xF4);
    277                         u8.xEF_scope = bitutil.Advance(xEF);
    278                         u8.scope32 = bitutil.Advance(u8.prefix3)
    279                         u8.scope33 = bitutil.Advance(u8.scope32)
    280                         u8.scope42 = bitutil.Advance(u8.prefix4)
    281                         u8.scope43 = bitutil.Advance(u8.scope42)
    282                         u8.scope44 = bitutil.Advance(u8.scope43)
     267#
     268#
     269#                       scope3_32 = pablo.Advance32(u8.prefix3)
     270#                       scope4_32 = pablo.Advance32(u8.prefix4)
     271#                       u8.scope32 = interpose32(u8.prefix3, scope3_32, 1)
     272#                       u8.scope33 = interpose32(u8.prefix3, scope3_32, 2)
     273#                       u8.scope42 = interpose32(u8.prefix4, scope4_32, 1)
     274#                       u8.scope43 = interpose32(u8.prefix4, scope4_32, 2)
     275#                       u8.scope44 = interpose32(u8.prefix4, scope4_32, 3)
     276#
     277                        u8.scope32 = pablo.Advance(u8.prefix3)
     278                        u8.scope33 = pablo.Advance(u8.scope32)
     279                        u8.scope42 = pablo.Advance(u8.prefix4)
     280                        u8.scope43 = pablo.Advance(u8.scope42)
     281                        u8.scope44 = pablo.Advance(u8.scope43)
     282#
     283#                       u8.xE0_scope = pablo.Advance(xE0);
     284#                       u8.xED_scope = pablo.Advance(xED);
     285#                       u8.xF0_scope = pablo.Advance(xF0);
     286#                       u8.xF4_scope = pablo.Advance(xF4);
     287                        E0_F0_scope = pablo.Advance(xE0 | xF0)
     288                        ED_F4_scope = pablo.Advance(xED | xF4)
     289                        u8.xE0_scope = u8.scope32 & E0_F0_scope
     290                        u8.xED_scope = u8.scope32 & ED_F4_scope
     291                        u8.xF0_scope = u8.scope42 & E0_F0_scope
     292                        u8.xF4_scope = u8.scope42 & ED_F4_scope
     293                        u8.xEF_scope = pablo.Advance(xEF);
    283294
    284295                        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
     
    292303                        u8_error |= u8error1 | u8error2 | u8error3 | u8error4
    293304
    294                         EF_BF_pending = bitutil.Advance(u8.xEF_scope & u8.xBF)
     305                        EF_BF_pending = pablo.Advance(u8.xEF_scope & u8.xBF)
    295306
    296307                        u8_FFFE_FFFF = (EF_BF_pending & (u8.xBE | u8.xBF))
     
    301312
    302313
    303 def Add_scope_streams(lex, scope1):
    304         #scope1.LAngle = bitutil.Advance(lex.LAngle)
    305         #scope1.Hyphen = bitutil.Advance(lex.Hyphen)
    306         #scope1.QMark = bitutil.Advance(lex.QMark)
    307         v = lex.LAngle | lex.Hyphen
    308         w = lex.Hyphen | lex.QMark
    309         v1 = bitutil.Advance(v)
    310         w1 = bitutil.Advance(w)
    311         scope1.LAngle = v1 &~ w1
    312         scope1.Hyphen = v1 & w1
    313         scope1.QMark = w1 &~ v1
    314         scope1.RefStart = 0 # default
    315 
    316 def Parse_CtCDPI(ctCDPI_Callouts, lex, scope1, check_streams):
    317         ctCDPI_Callouts.CD_end = 0
     314def Parse_CtCDPI(lex, marker, ctCDPI_Callouts, check_streams):
    318315        ctCDPI_Callouts.Ct_starts = 0
    319316        ctCDPI_Callouts.Ct_ends = 0
     
    324321        ctCDPI_Callouts.PI_name_ends = 0
    325322        ctCDPI_Callouts.PI_ends = 0
    326         ctCDPI_Callouts.CtCDPI_mask = 0
    327         ctCDPI_error = 0
    328323        CtCDPI_starts = 0
    329         Ct_errors = 0
    330 
     324        CtCDPI_ends = 0
     325        ctCDPI_mask = 0
     326
     327        v = lex.LAngle | lex.Hyphen
     328        w = lex.Hyphen | lex.QMark
     329        v1 = pablo.Advance(v)
     330        w1 = pablo.Advance(w)
     331
     332        LAngle_scope = v1 &~ w1  #pablo.Advance(lex.LAngle)
     333        PI_opener = LAngle_scope & lex.QMark
     334        CtCD_opener= LAngle_scope & lex.Exclam
     335        CtCDPI_opener = PI_opener | CtCD_opener
     336
     337        #DoubleHyphen = 0
     338        CD_closer = 0
     339        #PI_closer = 0
     340
     341        #if lex.Hyphen: DoubleHyphen = pablo.Advance(lex.Hyphen) & lex.Hyphen
     342        DoubleHyphen = v1 & w1 & lex.Hyphen
    331343        if lex.RBracket:
    332                 scope1.RBracket = bitutil.Advance(lex.RBracket)
    333                 ctCDPI_Callouts.CD_end = bitutil.Advance(scope1.RBracket & lex.RBracket) & lex.RAngle
    334         PI_start = scope1.LAngle & lex.QMark
    335         CtCD_start = scope1.LAngle & lex.Exclam
    336         CtCDPI_start = PI_start | CtCD_start
    337 
    338         DoubleHyphen = scope1.Hyphen & lex.Hyphen
    339         PI_end = scope1.QMark & lex.RAngle
    340 
     344                DoubleRBracket = pablo.Advance(lex.RBracket) & lex.RBracket
     345                CD_closer = pablo.Advance(DoubleRBracket) & lex.RAngle
     346        #if lex.QMark: PI_closer = pablo.Advance(lex.QMark) & lex.RAngle
     347        PI_closer = w1 & ~v1 & lex.RAngle
    341348
    342349        #
    343350        # Initiate the scan
    344         CtCDPI_Cursor = bitutil.ScanToFirst(CtCDPI_start)
     351        CtCDPI_Cursor = pablo.ScanToFirst(CtCDPI_opener)
    345352        while CtCDPI_Cursor:
    346353                CtCDPI_starts |= CtCDPI_Cursor
    347                 PI_Cursor = CtCDPI_Cursor & PI_start
    348                 CD_Ct_Cursor = bitutil.Advance(CtCDPI_Cursor & ~PI_Cursor)
     354                PI_Cursor = CtCDPI_Cursor & PI_opener
     355                CD_Ct_Cursor = pablo.Advance(CtCDPI_Cursor & ~PI_Cursor)
    349356                CD_Cursor = CD_Ct_Cursor & lex.LBracket
    350357                Ct_Cursor = CD_Ct_Cursor & lex.Hyphen
    351                 ctCDPI_Callouts.PI_starts |= PI_Cursor
    352                 ctCDPI_Callouts.CD_starts |= CD_Cursor
    353                 ctCDPI_Callouts.Ct_starts |= Ct_Cursor
    354                 Ct_Cursor = bitutil.Advance(Ct_Cursor)
    355                 Ct_errors |= Ct_Cursor & ~ lex.Hyphen
    356                 # Advance twice past <!--, so that we don't treat <!---
    357                 # as being a terminated comment.
    358                 Ct_Cursor = bitutil.Advance(bitutil.Advance(Ct_Cursor))
    359                 PI_Cursor = bitutil.Advance(PI_Cursor)
    360                 ctCDPI_Callouts.PI_name_starts |= PI_Cursor
    361                 PI_name_end = bitutil.ScanThru(PI_Cursor, lex.NameScan)
    362                 ctCDPI_Callouts.PI_name_ends |= PI_name_end
    363                 PI_Cursor = bitutil.ScanTo(PI_name_end, PI_end)
    364                 CD_Cursor = bitutil.ScanTo(CD_Cursor, ctCDPI_Callouts.CD_end)
    365                 Ct_Cursor = bitutil.Advance(bitutil.ScanTo(Ct_Cursor, DoubleHyphen))
    366                 ctCDPI_Callouts.PI_ends |= PI_Cursor
    367                 ctCDPI_Callouts.CD_ends |= CD_Cursor
    368                 ctCDPI_Callouts.Ct_ends |= Ct_Cursor
     358                # PI processing
     359                if PI_Cursor:
     360                        ctCDPI_Callouts.PI_starts |= PI_Cursor
     361                        PI_Cursor = pablo.Advance(PI_Cursor)
     362                        ctCDPI_Callouts.PI_name_starts |= PI_Cursor
     363                        PI_name_end = pablo.ScanThru(PI_Cursor, lex.NameScan)
     364                        PI_error = PI_Cursor & PI_name_end
     365                        PI_error |= pablo.Advance(PI_name_end & ~ lex.WS) & ~ PI_closer
     366                        if PI_error:
     367                                error_tracker.NoteError("Error in PI syntax", PI_name_end)
     368                        ctCDPI_Callouts.PI_name_ends |= PI_name_end
     369                        PI_Cursor = pablo.ScanTo(PI_name_end, PI_closer)
     370                        ctCDPI_Callouts.PI_ends |= PI_Cursor
     371                        CtCDPI_ends |= PI_Cursor
     372
     373                # CDATA section processing
     374                if CD_Cursor:
     375                        ctCDPI_Callouts.CD_starts |= CD_Cursor
     376                        CD_Cursor = pablo.ScanTo(CD_Cursor, CD_closer)
     377                        ctCDPI_Callouts.CD_ends |= CD_Cursor
     378                        CtCDPI_ends |= CD_Cursor
     379
     380                # Comment processing
     381                if Ct_Cursor:
     382                        ctCDPI_Callouts.Ct_starts |= Ct_Cursor
     383                        Ct_Cursor = pablo.Advance(Ct_Cursor)
     384                        Ct_error = Ct_Cursor & ~ lex.Hyphen
     385                        # Advance twice past <!--, so that we don't treat <!---
     386                        # as being a terminated comment.
     387                        Ct_Cursor = pablo.Advance(pablo.Advance(Ct_Cursor))
     388                        Ct_Cursor = pablo.Advance(pablo.ScanTo(Ct_Cursor, DoubleHyphen))
     389                        Ct_error |= Ct_Cursor & ~ lex.RAngle
     390                        ctCDPI_Callouts.Ct_ends |= Ct_Cursor
     391                        CtCDPI_ends |= Ct_Cursor
     392                        if Ct_error:
     393                                error_tracker.NoteError("Error in comment syntax", Ct_error)
     394
     395                # Common processing
    369396                CtCDPI_Cursor = PI_Cursor | CD_Cursor | Ct_Cursor
    370                 CtCDPI_Cursor = bitutil.ScanTo(CtCDPI_Cursor, CtCDPI_start)
    371 
    372                 ctCDPI_Callouts.CtCDPI_mask = bitutil.Advance(ctCDPI_Callouts.CD_ends | ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - CtCDPI_starts
    373                 #ctCDPI_Callouts.error = Ct_ends & ~lex.RAngle | Ct_starts & ~ lex.Hyphen
    374                 ctCDPI_error = Ct_errors | ctCDPI_Callouts.Ct_ends & ~lex.RAngle
    375                 ctCDPI_error |= bitutil.Advance(ctCDPI_Callouts.PI_name_ends & ~ lex.WS) & ~ PI_end
    376                 ctCDPI_error |= ctCDPI_Callouts.PI_name_starts & ctCDPI_Callouts.PI_name_ends
     397                CtCDPI_Cursor = pablo.ScanTo(CtCDPI_Cursor, CtCDPI_opener)
     398                ctCDPI_mask |= (CtCDPI_ends - CtCDPI_starts) | CtCDPI_ends
    377399                # If any of the Comment, CDATA or PI markups are unterminated, it is an error.
    378                 ctCDPI_error |= ctCDPI_Callouts.CtCDPI_mask &~ EOF_mask
    379 
    380         if ctCDPI_error:
    381                 error_tracker.NoteError("Error in comment, CDATA or processing instruction syntax", ctCDPI_error)
    382 
    383         check_streams.misc_mask = (lex.WS | lex.LAngle | (bitutil.Advance(ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - (ctCDPI_Callouts.Ct_starts | ctCDPI_Callouts.PI_starts)) | CtCDPI_starts) & EOF_mask
    384 
    385 def Parse_tags(lex, scope1, ctCDPI_Callouts, tag_Callouts):
    386 
     400                ctCDPI_error = pablo.atEOF(ctCDPI_mask)
     401
     402                if ctCDPI_error:
     403                        error_tracker.NoteError("Error in comment, CDATA or processing instruction syntax", ctCDPI_error)
     404
     405        check_streams.misc_mask = (lex.WS | lex.LAngle | (pablo.Advance(ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - (ctCDPI_Callouts.Ct_starts | ctCDPI_Callouts.PI_starts)) | CtCDPI_starts) & EOF_mask
     406
     407
     408        # Identify the remaining significant markers for XML processing.
     409        marker.LAngle_scope = LAngle_scope &~ ctCDPI_mask
     410        marker.Ref_opener = lex.RefStart &~ ctCDPI_mask
     411        marker.CD_closer = CD_closer &~ ctCDPI_mask
     412
     413def Parse_tags(lex, marker, tag_Callouts):
    387414
    388415        # Delimiters for scans.
     
    393420        # Start the parallel parsing by inspecting the character
    394421        # after the opening "<" of a tag.
    395         LAngleFollow = scope1.LAngle &~ ctCDPI_Callouts.CtCDPI_mask
    396         tag_Callouts.ElemName_starts = LAngleFollow & ~lex.Slash
    397         tag_Callouts.EndTag_marks = LAngleFollow & lex.Slash
     422        tag_Callouts.ElemName_starts = marker.LAngle_scope & ~lex.Slash
     423        tag_Callouts.EndTag_marks = marker.LAngle_scope & lex.Slash
    398424
    399425        # Start Tag/Empty Element Tag Parsing
    400426
    401427        # Advance all cursors by scanning through the tag name.
    402         tag_Callouts.ElemName_ends = bitutil.ScanThru(tag_Callouts.ElemName_starts, lex.NameScan)
     428        tag_Callouts.ElemName_ends = pablo.ScanThru(tag_Callouts.ElemName_starts, lex.NameScan)
    403429        # Must have at least one name character for a legal start tag.
    404430        # Mark any occurrences of null names as errors.
     
    408434        tag_Callouts.AttName_starts = 0
    409435        tag_Callouts.AttName_ends = 0
    410         EqToCheck = 0
    411436        tag_Callouts.AttVal_starts = 0
    412         AttValEnds = 0
    413437        tag_Callouts.AttVal_ends = 0
    414438
    415439        # After the element name, there may or may not be an attlist.
    416         AfterWS = bitutil.ScanThru(tag_Callouts.ElemName_ends, lex.WS)
    417         AttListEnd = AfterWS & AttListDelim
    418         AttNameStart = AfterWS & ~AttListDelim
    419         # At least one WS character is required between ElemNames and AttNames.
    420         ParseError |= tag_Callouts.ElemName_ends & AttNameStart
    421 
    422         #
    423         # The following loop iterates through attributes within a start tag.
    424         # Because all start tags are processed in parallel, the number of
    425         # iterations is the maximum number of attributes found in any one
    426         # start tag, plus one.
    427         while AttNameStart:
    428                 tag_Callouts.AttName_starts |= AttNameStart
    429                 AttNameFollow = bitutil.ScanThru(AttNameStart, lex.NameScan)
    430                 tag_Callouts.AttName_ends |= AttNameFollow
    431                 # Scan through WS to the expected '=' delimiter.
    432                 # EqExpected = bitutil.ScanThru(AttNameFollow, lex.WS)
    433                 # But use if test to optimize.
    434                 if AttNameFollow & lex.WS:
    435                         EqExpected = bitutil.ScanThru(AttNameFollow, lex.WS)
    436                 else: EqExpected = AttNameFollow
    437                 EqToCheck |= EqExpected
    438                 AttValPos = bitutil.ScanThru(EqExpected, EqExpected | lex.WS)
    439                 tag_Callouts.AttVal_starts |= AttValPos
    440                 DQuoteAttVal = AttValPos & lex.DQuote
    441                 SQuoteAttVal = AttValPos & lex.SQuote
    442                 DQuoteAttEnd = bitutil.ScanTo(DQuoteAttVal, DQuoteDelim &~ DQuoteAttVal)
    443                 SQuoteAttEnd = bitutil.ScanTo(SQuoteAttVal, SQuoteDelim &~ SQuoteAttVal)
    444                 AttValEnd = DQuoteAttEnd | SQuoteAttEnd
    445                 AttValEnds |= AttValEnd
    446                 AttValFollow = bitutil.Advance(AttValEnd)
    447                 tag_Callouts.AttVal_ends |= AttValFollow
    448                 #  AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
    449                 if AttValFollow & lex.WS:
    450                         AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
    451                 else: AfterWS = AttValFollow
    452                 AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
    453                 AttListEnd |= AfterWS & AttListDelim
     440        if tag_Callouts.ElemName_ends & lex.WS:
     441                AfterWS = pablo.ScanThru(tag_Callouts.ElemName_ends, lex.WS)
     442                AttListEnd = AfterWS & AttListDelim
    454443                AttNameStart = AfterWS & ~AttListDelim
    455 
    456         # No more attribute values to process when AttNameStart == 0.
     444                #
     445                # The following loop iterates through attributes within a start tag.
     446                # Because all start tags are processed in parallel, the number of
     447                # iterations is the maximum number of attributes found in any one
     448                # start tag, plus one.
     449                while AttNameStart:
     450                        ParseError |= AttNameStart &~ lex.NameScan
     451                        tag_Callouts.AttName_starts |= AttNameStart
     452                        AttNameFollow = pablo.ScanThru(AttNameStart, lex.NameScan)
     453                        tag_Callouts.AttName_ends |= AttNameFollow
     454                        # Scan through WS to the expected '=' delimiter.
     455                        # EqExpected = pablo.ScanThru(AttNameFollow, lex.WS)
     456                        # But use if test to optimize.
     457                        if AttNameFollow & lex.WS:
     458                                EqExpected = pablo.ScanThru(AttNameFollow, lex.WS)
     459                        else: EqExpected = AttNameFollow
     460                        ParseError |= EqExpected &~ lex.Equals
     461                        AttValPos = pablo.ScanThru(EqExpected, EqExpected | lex.WS)
     462                        tag_Callouts.AttVal_starts |= AttValPos
     463                        DQuoteAttVal = AttValPos & lex.DQuote
     464                        SQuoteAttVal = AttValPos & lex.SQuote
     465                        DQuoteAttEnd = pablo.ScanTo(DQuoteAttVal, DQuoteDelim &~ DQuoteAttVal)
     466                        SQuoteAttEnd = pablo.ScanTo(SQuoteAttVal, SQuoteDelim &~ SQuoteAttVal)
     467                        AttValEnd = DQuoteAttEnd | SQuoteAttEnd
     468                        ParseError |= (AttValPos | AttValEnd) &~ (lex.DQuote | lex.SQuote)
     469                        AttValFollow = pablo.Advance(AttValEnd)
     470                        tag_Callouts.AttVal_ends |= AttValFollow
     471                        #  AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
     472                        if AttValFollow & lex.WS:
     473                                AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
     474                                AttListEnd |= AfterWS & AttListDelim
     475                                AttNameStart = AfterWS & ~AttListDelim
     476                        else:
     477                                AttListEnd |= AttValFollow & AttListDelim
     478                                AttNameStart = 0
     479                                ParseError |= AttValFollow & ~AttListDelim
     480        else:
     481                # No WS character after ElemName; must be at the end
     482                AttListEnd = tag_Callouts.ElemName_ends & AttListDelim
     483                ParseError |= tag_Callouts.ElemName_ends & ~AttListDelim
     484
    457485        STagEnds = AttListEnd & lex.RAngle
    458486        # Mark any "/" characters found as the ends of empty element tags.
    459         tag_Callouts.EmptyTag_marks = bitutil.Advance(AttListEnd & lex.Slash)
    460 
    461         # Check for errors.
    462         ParseError |= tag_Callouts.AttVal_ends & tag_Callouts.AttName_starts # No intervening WS.
    463         ParseError |= tag_Callouts.AttName_starts & tag_Callouts.AttName_ends # Null AttName
    464         ParseError |= EqToCheck & ~lex.Equals # = not found where expected.
    465         ParseError |= tag_Callouts.AttVal_starts & ~ (lex.DQuote | lex.SQuote)
    466         ParseError |= AttValEnds & ~ (lex.DQuote | lex.SQuote)
     487        tag_Callouts.EmptyTag_marks = pablo.Advance(AttListEnd & lex.Slash)
     488
    467489        ParseError |= tag_Callouts.EmptyTag_marks & ~lex.RAngle
    468490
    469491        # End Tag Parsing
    470492
    471         EndTagEnds = bitutil.ScanThru(tag_Callouts.EndTag_marks, tag_Callouts.EndTag_marks | lex.NameScan)
     493        EndTagEnds = pablo.ScanThru(tag_Callouts.EndTag_marks, tag_Callouts.EndTag_marks | lex.NameScan)
    472494        if EndTagEnds & lex.WS:
    473                 EndTagEnds = bitutil.ScanThru(EndTagEnds, lex.WS)
     495                EndTagEnds = pablo.ScanThru(EndTagEnds, lex.WS)
    474496        ParseError |= EndTagEnds & ~lex.RAngle
    475497        if ParseError:
     
    480502        tag_Callouts.AttVal_spans = tag_Callouts.AttVal_ends - tag_Callouts.AttVal_starts
    481503
    482 def Parse_refs(lex, scope1, ctCDPI_Callouts, ref_Callouts):
     504def Parse_refs(lex, marker, ref_Callouts):
    483505        ref_Callouts.GenRef_starts = 0
    484506        ref_Callouts.GenRef_ends = 0
     
    489511        ref_error = 0
    490512
    491         Ref1 = lex.RefStart &~ ctCDPI_Callouts.CtCDPI_mask
    492513        # All remaining "&" must be reference start characters; parse them.
    493         if Ref1:
    494                 scope1.RefStart = bitutil.Advance(Ref1)
    495                 NumRef2 = scope1.RefStart & lex.Hash
    496                 ref_Callouts.GenRef_starts = scope1.RefStart &~ lex.Hash
    497                 NumRef3 = bitutil.Advance(NumRef2)
     514        if marker.Ref_opener:
     515                Ref_scope = pablo.Advance(marker.Ref_opener)
     516                NumRef2 = Ref_scope & lex.Hash
     517                ref_Callouts.GenRef_starts = Ref_scope &~ lex.Hash
     518                NumRef3 = pablo.Advance(NumRef2)
    498519                HexRef3 = NumRef3 & lex.x
    499520                ref_Callouts.DecRef_starts = NumRef3 &~ lex.x
    500                 ref_Callouts.HexRef_starts = bitutil.Advance(HexRef3)
    501                 ref_Callouts.GenRef_ends = bitutil.ScanThru(ref_Callouts.GenRef_starts, lex.NameScan)
    502                 ref_Callouts.DecRef_ends = bitutil.ScanThru(ref_Callouts.DecRef_starts, lex.Digit)
    503                 ref_Callouts.HexRef_ends = bitutil.ScanThru(ref_Callouts.HexRef_starts, lex.Hex)
     521                ref_Callouts.HexRef_starts = pablo.Advance(HexRef3)
     522                ref_Callouts.GenRef_ends = pablo.ScanThru(ref_Callouts.GenRef_starts, lex.NameScan)
     523                ref_Callouts.DecRef_ends = pablo.ScanThru(ref_Callouts.DecRef_starts, lex.Digit)
     524                ref_Callouts.HexRef_ends = pablo.ScanThru(ref_Callouts.HexRef_starts, lex.Hex)
    504525                # Error checks
    505526                # At least one digit required for DecRef, one hex digit for HexRef.
     
    515536
    516537
    517 def Validate_xml_names(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, xml_names, check_streams):
     538def Validate_xml_names(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, check_streams):
    518539        PI_names = ctCDPI_Callouts.PI_name_ends - ctCDPI_Callouts.PI_name_starts
    519540        GenRefs = ref_Callouts.GenRef_ends - ref_Callouts.GenRef_starts
     
    523544        ncname_stream = PI_names | GenRefs
    524545        name_stream = qname_stream | ncname_stream
    525         name_start = name_stream &~ bitutil.Advance(name_stream)
    526         name_cursor = name_stream & ~bitutil.Advance(name_stream)
     546        name_start = name_stream &~ pablo.Advance(name_stream)
     547        name_cursor = name_stream & ~pablo.Advance(name_stream)
    527548        void_prefix_err = name_cursor & lex.Colon
    528         namespace_sep = bitutil.ScanThru(name_cursor, lex.NameScan &~ lex.Colon) & lex.Colon
    529         local_part_start = bitutil.Advance(namespace_sep)
     549        namespace_sep = pablo.ScanThru(name_cursor, lex.NameScan &~ lex.Colon) & lex.Colon
     550        local_part_start = pablo.Advance(namespace_sep)
    530551        local_part_err = local_part_start &~ lex.NameScan
    531         colon2_err = bitutil.ScanThru(local_part_start, lex.NameScan &~ lex.Colon) & lex.Colon
     552        colon2_err = pablo.ScanThru(local_part_start, lex.NameScan &~ lex.Colon) & lex.Colon
    532553        ncname_err = ncname_stream & lex.Colon
    533554        namespace_error = void_prefix_err | local_part_err | colon2_err | ncname_err
     
    539560        check_streams.non_ascii_names = (name_stream &~ name_start) & ~lex.ASCII_name_char & ~u8.suffix
    540561
    541 def Do_check_streams(ctCDPI_Callouts, tag_Callouts, lex, u8, scope1, ref_Callouts, xml_names, check_streams):
    542     # Ensure that no occurrence of ]]> occurs outside of markup.
    543     CD_end_error = ctCDPI_Callouts.CD_end & ~(ctCDPI_Callouts.CtCDPI_mask | tag_Callouts.AttVal_spans)
    544 
    545     # Consolidate and check for errors
    546     if CD_end_error:
     562def Do_check_streams(marker, tag_Callouts, check_streams):
     563        CD_end_error = marker.CD_closer & ~tag_Callouts.AttVal_spans
     564        if CD_end_error:
    547565                error_tracker.NoteError("Error: ]]> in text", CD_end_error)
    548 
    549 
    550     check_streams.tag_marks = tag_Callouts.EmptyTag_marks | tag_Callouts.ElemName_starts | tag_Callouts.EndTag_marks | tag_Callouts.AttName_starts
    551     check_streams.name_follows = tag_Callouts.ElemName_ends | tag_Callouts.AttName_ends
    552     check_streams.att_refs = tag_Callouts.AttVal_spans & scope1.RefStart
     566        check_streams.tag_marks = tag_Callouts.EmptyTag_marks | tag_Callouts.ElemName_starts | tag_Callouts.EndTag_marks | tag_Callouts.AttName_starts
     567        check_streams.name_follows = tag_Callouts.ElemName_ends | tag_Callouts.AttName_ends
     568        check_streams.att_refs = tag_Callouts.AttVal_spans & marker.Ref_opener
    553569
    554570def Compute_Hash_Value_Bitstream(hash_data, basis_bits):
     
    564580        # Validate UTF-8 multibyte sequences and determine the UTF-8 scope streams
    565581        # Validate_utf8(basis_bits, u8)
    566                                
     582
    567583        Classify_bytes_Validate_utf8(basis_bits, lex, u8)
    568584
    569         Add_scope_streams(lex, scope1)
    570    
    571585        # Parse all comments, CDATA sections and processing instructions.
    572         Parse_CtCDPI(ctCDPI_Callouts, lex, scope1, check_streams)
    573                
     586        Parse_CtCDPI(lex, marker, ctCDPI_Callouts, check_streams)
     587
    574588        # All remaining '<' must be tag start characters; parse tags.
    575         Parse_tags(lex, scope1, ctCDPI_Callouts, tag_Callouts)
     589        Parse_tags(lex, marker, tag_Callouts)
    576590
    577591        # All remaining '&' must be reference start characters; parse them.
    578         Parse_refs(lex, scope1, ctCDPI_Callouts, ref_Callouts)
    579        
     592        Parse_refs(lex, marker, ref_Callouts)
     593
    580594        # Validate XML namespaces and generate bit streams to post validate non-ascii range XML names
    581         Validate_xml_names(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, xml_names, check_streams)
    582    
    583    
    584         Do_check_streams(ctCDPI_Callouts, tag_Callouts, lex, u8, scope1, ref_Callouts, xml_names, check_streams)
     595        Validate_xml_names(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, check_streams)
     596
     597        Do_check_streams(marker, tag_Callouts, check_streams)
    585598
    586599        Compute_Hash_Value_Bitstream(hash_data, basis_bits);
  • proto/SymbolTable/parabix2_symtab_pbs.py

    r1232 r1442  
    272272                u8.badprefix = (temp68 | temp71);
    273273                u8_error = u8.badprefix
    274                 u8.scope22 = bitutil.Advance(u8.prefix2)
     274                u8.scope22 = pablo.Advance(u8.prefix2)
    275275                u8anyscope = u8.scope22
    276276                if u8.prefix3 | u8.prefix4:
     
    288288                        u8.xBF = (temp73 & temp23);
    289289                        u8.xBE = (temp73 & temp15);
    290                         u8.xE0_scope = bitutil.Advance(xE0);
    291                         u8.xED_scope = bitutil.Advance(xED);
    292                         u8.xF0_scope = bitutil.Advance(xF0);
    293                         u8.xF4_scope = bitutil.Advance(xF4);
    294                         u8.xEF_scope = bitutil.Advance(xEF);
    295                         u8.scope32 = bitutil.Advance(u8.prefix3)
    296                         u8.scope33 = bitutil.Advance(u8.scope32)
    297                         u8.scope42 = bitutil.Advance(u8.prefix4)
    298                         u8.scope43 = bitutil.Advance(u8.scope42)
    299                         u8.scope44 = bitutil.Advance(u8.scope43)
     290                        u8.xE0_scope = pablo.Advance(xE0);
     291                        u8.xED_scope = pablo.Advance(xED);
     292                        u8.xF0_scope = pablo.Advance(xF0);
     293                        u8.xF4_scope = pablo.Advance(xF4);
     294                        u8.xEF_scope = pablo.Advance(xEF);
     295                        u8.scope32 = pablo.Advance(u8.prefix3)
     296                        u8.scope33 = pablo.Advance(u8.scope32)
     297                        u8.scope42 = pablo.Advance(u8.prefix4)
     298                        u8.scope43 = pablo.Advance(u8.scope42)
     299                        u8.scope44 = pablo.Advance(u8.scope43)
    300300
    301301                        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
     
    309309                        u8_error |= u8error1 | u8error2 | u8error3 | u8error4
    310310
    311                         EF_BF_pending = bitutil.Advance(u8.xEF_scope & u8.xBF)
     311                        EF_BF_pending = pablo.Advance(u8.xEF_scope & u8.xBF)
    312312
    313313                        u8_FFFE_FFFF = (EF_BF_pending & (u8.xBE | u8.xBF))
     
    319319
    320320def Add_scope_streams(lex, scope1):
    321         #scope1.LAngle = bitutil.Advance(lex.LAngle)
    322         #scope1.Hyphen = bitutil.Advance(lex.Hyphen)
    323         #scope1.QMark = bitutil.Advance(lex.QMark)
     321        #scope1.LAngle = pablo.Advance(lex.LAngle)
     322        #scope1.Hyphen = pablo.Advance(lex.Hyphen)
     323        #scope1.QMark = pablo.Advance(lex.QMark)
    324324        v = lex.LAngle | lex.Hyphen
    325325        w = lex.Hyphen | lex.QMark
    326         v1 = bitutil.Advance(v)
    327         w1 = bitutil.Advance(w)
     326        v1 = pablo.Advance(v)
     327        w1 = pablo.Advance(w)
    328328        scope1.LAngle = v1 &~ w1
    329329        scope1.Hyphen = v1 & w1
     
    347347
    348348        if lex.RBracket:
    349                 scope1.RBracket = bitutil.Advance(lex.RBracket)
    350                 ctCDPI_Callouts.CD_end = bitutil.Advance(scope1.RBracket & lex.RBracket) & lex.RAngle
     349                scope1.RBracket = pablo.Advance(lex.RBracket)
     350                ctCDPI_Callouts.CD_end = pablo.Advance(scope1.RBracket & lex.RBracket) & lex.RAngle
    351351        PI_start = scope1.LAngle & lex.QMark
    352352        CtCD_start = scope1.LAngle & lex.Exclam
     
    359359        #
    360360        # Initiate the scan
    361         CtCDPI_Cursor = bitutil.ScanToFirst(CtCDPI_start)
     361        CtCDPI_Cursor = pablo.ScanToFirst(CtCDPI_start)
    362362        while CtCDPI_Cursor:
    363363                CtCDPI_starts |= CtCDPI_Cursor
    364364                PI_Cursor = CtCDPI_Cursor & PI_start
    365                 CD_Ct_Cursor = bitutil.Advance(CtCDPI_Cursor & ~PI_Cursor)
     365                CD_Ct_Cursor = pablo.Advance(CtCDPI_Cursor & ~PI_Cursor)
    366366                CD_Cursor = CD_Ct_Cursor & lex.LBracket
    367367                Ct_Cursor = CD_Ct_Cursor & lex.Hyphen
     
    369369                ctCDPI_Callouts.CD_starts |= CD_Cursor
    370370                ctCDPI_Callouts.Ct_starts |= Ct_Cursor
    371                 Ct_Cursor = bitutil.Advance(Ct_Cursor)
     371                Ct_Cursor = pablo.Advance(Ct_Cursor)
    372372                Ct_errors |= Ct_Cursor & ~ lex.Hyphen
    373373                # Advance twice past <!--, so that we don't treat <!---
    374374                # as being a terminated comment.
    375                 Ct_Cursor = bitutil.Advance(bitutil.Advance(Ct_Cursor))
    376                 PI_Cursor = bitutil.Advance(PI_Cursor)
     375                Ct_Cursor = pablo.Advance(pablo.Advance(Ct_Cursor))
     376                PI_Cursor = pablo.Advance(PI_Cursor)
    377377                ctCDPI_Callouts.PI_name_starts |= PI_Cursor
    378                 PI_name_end = bitutil.ScanThru(PI_Cursor, lex.NameScan)
     378                PI_name_end = pablo.ScanThru(PI_Cursor, lex.NameScan)
    379379                ctCDPI_Callouts.PI_name_ends |= PI_name_end
    380                 PI_Cursor = bitutil.ScanTo(PI_name_end, PI_end)
    381                 CD_Cursor = bitutil.ScanTo(CD_Cursor, ctCDPI_Callouts.CD_end)
    382                 Ct_Cursor = bitutil.Advance(bitutil.ScanTo(Ct_Cursor, DoubleHyphen))
     380                PI_Cursor = pablo.ScanTo(PI_name_end, PI_end)
     381                CD_Cursor = pablo.ScanTo(CD_Cursor, ctCDPI_Callouts.CD_end)
     382                Ct_Cursor = pablo.Advance(pablo.ScanTo(Ct_Cursor, DoubleHyphen))
    383383                ctCDPI_Callouts.PI_ends |= PI_Cursor
    384384                ctCDPI_Callouts.CD_ends |= CD_Cursor
    385385                ctCDPI_Callouts.Ct_ends |= Ct_Cursor
    386386                CtCDPI_Cursor = PI_Cursor | CD_Cursor | Ct_Cursor
    387                 CtCDPI_Cursor = bitutil.ScanTo(CtCDPI_Cursor, CtCDPI_start)
    388 
    389                 ctCDPI_Callouts.CtCDPI_mask = bitutil.Advance(ctCDPI_Callouts.CD_ends | ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - CtCDPI_starts
     387                CtCDPI_Cursor = pablo.ScanTo(CtCDPI_Cursor, CtCDPI_start)
     388
     389                ctCDPI_Callouts.CtCDPI_mask = pablo.Advance(ctCDPI_Callouts.CD_ends | ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - CtCDPI_starts
    390390                #ctCDPI_Callouts.error = Ct_ends & ~lex.RAngle | Ct_starts & ~ lex.Hyphen
    391391                ctCDPI_error = Ct_errors | ctCDPI_Callouts.Ct_ends & ~lex.RAngle
    392                 ctCDPI_error |= bitutil.Advance(ctCDPI_Callouts.PI_name_ends & ~ lex.WS) & ~ PI_end
     392                ctCDPI_error |= pablo.Advance(ctCDPI_Callouts.PI_name_ends & ~ lex.WS) & ~ PI_end
    393393                ctCDPI_error |= ctCDPI_Callouts.PI_name_starts & ctCDPI_Callouts.PI_name_ends
    394394                # If any of the Comment, CDATA or PI markups are unterminated, it is an error.
     
    398398                error_tracker.NoteError("Error in comment, CDATA or processing instruction syntax", ctCDPI_error)
    399399
    400         check_streams.misc_mask = (lex.WS | lex.LAngle | (bitutil.Advance(ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - (ctCDPI_Callouts.Ct_starts | ctCDPI_Callouts.PI_starts)) | CtCDPI_starts) & EOF_mask
     400        check_streams.misc_mask = (lex.WS | lex.LAngle | (pablo.Advance(ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - (ctCDPI_Callouts.Ct_starts | ctCDPI_Callouts.PI_starts)) | CtCDPI_starts) & EOF_mask
    401401
    402402def Parse_tags(lex, scope1, ctCDPI_Callouts, tag_Callouts):
     
    417417
    418418        # Advance all cursors by scanning through the tag name.
    419         tag_Callouts.ElemName_ends = bitutil.ScanThru(tag_Callouts.ElemName_starts, lex.NameScan)
     419        tag_Callouts.ElemName_ends = pablo.ScanThru(tag_Callouts.ElemName_starts, lex.NameScan)
    420420        # Must have at least one name character for a legal start tag.
    421421        # Mark any occurrences of null names as errors.
     
    431431
    432432        # After the element name, there may or may not be an attlist.
    433         AfterWS = bitutil.ScanThru(tag_Callouts.ElemName_ends, lex.WS)
     433        AfterWS = pablo.ScanThru(tag_Callouts.ElemName_ends, lex.WS)
    434434        AttListEnd = AfterWS & AttListDelim
    435435        AttNameStart = AfterWS & ~AttListDelim
     
    444444        while AttNameStart:
    445445                tag_Callouts.AttName_starts |= AttNameStart
    446                 AttNameFollow = bitutil.ScanThru(AttNameStart, lex.NameScan)
     446                AttNameFollow = pablo.ScanThru(AttNameStart, lex.NameScan)
    447447                tag_Callouts.AttName_ends |= AttNameFollow
    448448                # Scan through WS to the expected '=' delimiter.
    449                 # EqExpected = bitutil.ScanThru(AttNameFollow, lex.WS)
     449                # EqExpected = pablo.ScanThru(AttNameFollow, lex.WS)
    450450                # But use if test to optimize.
    451451                if AttNameFollow & lex.WS:
    452                         EqExpected = bitutil.ScanThru(AttNameFollow, lex.WS)
     452                        EqExpected = pablo.ScanThru(AttNameFollow, lex.WS)
    453453                else: EqExpected = AttNameFollow
    454454                EqToCheck |= EqExpected
    455                 AttValPos = bitutil.ScanThru(EqExpected, EqExpected | lex.WS)
     455                AttValPos = pablo.ScanThru(EqExpected, EqExpected | lex.WS)
    456456                tag_Callouts.AttVal_starts |= AttValPos
    457457                DQuoteAttVal = AttValPos & lex.DQuote
    458458                SQuoteAttVal = AttValPos & lex.SQuote
    459                 DQuoteAttEnd = bitutil.ScanTo(DQuoteAttVal, DQuoteDelim &~ DQuoteAttVal)
    460                 SQuoteAttEnd = bitutil.ScanTo(SQuoteAttVal, SQuoteDelim &~ SQuoteAttVal)
     459                DQuoteAttEnd = pablo.ScanTo(DQuoteAttVal, DQuoteDelim &~ DQuoteAttVal)
     460                SQuoteAttEnd = pablo.ScanTo(SQuoteAttVal, SQuoteDelim &~ SQuoteAttVal)
    461461                AttValEnd = DQuoteAttEnd | SQuoteAttEnd
    462462                AttValEnds |= AttValEnd
    463                 AttValFollow = bitutil.Advance(AttValEnd)
     463                AttValFollow = pablo.Advance(AttValEnd)
    464464                tag_Callouts.AttVal_ends |= AttValFollow
    465                 #  AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
     465                #  AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
    466466                if AttValFollow & lex.WS:
    467                         AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
     467                        AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
    468468                else: AfterWS = AttValFollow
    469                 AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
     469                AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
    470470                AttListEnd |= AfterWS & AttListDelim
    471471                AttNameStart = AfterWS & ~AttListDelim
     
    474474        STagEnds = AttListEnd & lex.RAngle
    475475        # Mark any "/" characters found as the ends of empty element tags.
    476         tag_Callouts.EmptyTag_marks = bitutil.Advance(AttListEnd & lex.Slash)
     476        tag_Callouts.EmptyTag_marks = pablo.Advance(AttListEnd & lex.Slash)
    477477
    478478        # Check for errors.
     
    486486        # End Tag Parsing
    487487
    488         EndTagEnds = bitutil.ScanThru(tag_Callouts.EndTag_marks, tag_Callouts.EndTag_marks | lex.NameScan)
     488        EndTagEnds = pablo.ScanThru(tag_Callouts.EndTag_marks, tag_Callouts.EndTag_marks | lex.NameScan)
    489489        if EndTagEnds & lex.WS:
    490                 EndTagEnds = bitutil.ScanThru(EndTagEnds, lex.WS)
     490                EndTagEnds = pablo.ScanThru(EndTagEnds, lex.WS)
    491491        ParseError |= EndTagEnds & ~lex.RAngle
    492492        if ParseError:
     
    509509        # All remaining "&" must be reference start characters; parse them.
    510510        if Ref1:
    511                 scope1.RefStart = bitutil.Advance(Ref1)
     511                scope1.RefStart = pablo.Advance(Ref1)
    512512                NumRef2 = scope1.RefStart & lex.Hash
    513513                ref_Callouts.GenRef_starts = scope1.RefStart &~ lex.Hash
    514                 NumRef3 = bitutil.Advance(NumRef2)
     514                NumRef3 = pablo.Advance(NumRef2)
    515515                HexRef3 = NumRef3 & lex.x
    516516                ref_Callouts.DecRef_starts = NumRef3 &~ lex.x
    517                 ref_Callouts.HexRef_starts = bitutil.Advance(HexRef3)
    518                 ref_Callouts.GenRef_ends = bitutil.ScanThru(ref_Callouts.GenRef_starts, lex.NameScan)
    519                 ref_Callouts.DecRef_ends = bitutil.ScanThru(ref_Callouts.DecRef_starts, lex.Digit)
    520                 ref_Callouts.HexRef_ends = bitutil.ScanThru(ref_Callouts.HexRef_starts, lex.Hex)
     517                ref_Callouts.HexRef_starts = pablo.Advance(HexRef3)
     518                ref_Callouts.GenRef_ends = pablo.ScanThru(ref_Callouts.GenRef_starts, lex.NameScan)
     519                ref_Callouts.DecRef_ends = pablo.ScanThru(ref_Callouts.DecRef_starts, lex.Digit)
     520                ref_Callouts.HexRef_ends = pablo.ScanThru(ref_Callouts.HexRef_starts, lex.Hex)
    521521                # Error checks
    522522                # At least one digit required for DecRef, one hex digit for HexRef.
     
    540540        ncname_stream = PI_names | GenRefs
    541541        name_stream = qname_stream | ncname_stream
    542         name_start = name_stream &~ bitutil.Advance(name_stream)
    543         name_cursor = name_stream & ~bitutil.Advance(name_stream)
     542        name_start = name_stream &~ pablo.Advance(name_stream)
     543        name_cursor = name_stream & ~pablo.Advance(name_stream)
    544544        void_prefix_err = name_cursor & lex.Colon
    545         namespace_sep = bitutil.ScanThru(name_cursor, lex.NameScan &~ lex.Colon) & lex.Colon
    546         local_part_start = bitutil.Advance(namespace_sep)
     545        namespace_sep = pablo.ScanThru(name_cursor, lex.NameScan &~ lex.Colon) & lex.Colon
     546        local_part_start = pablo.Advance(namespace_sep)
    547547        local_part_err = local_part_start &~ lex.NameScan
    548         colon2_err = bitutil.ScanThru(local_part_start, lex.NameScan &~ lex.Colon) & lex.Colon
     548        colon2_err = pablo.ScanThru(local_part_start, lex.NameScan &~ lex.Colon) & lex.Colon
    549549        ncname_err = ncname_stream & lex.Colon
    550550        namespace_error = void_prefix_err | local_part_err | colon2_err | ncname_err
     
    577577
    578578    # Group symbols of length 1
    579     temp = bitutil.Advance(temp)
     579    temp = pablo.Advance(temp)
    580580    tag_Callouts.ElemName_ends_1 = temp & remaining_ends
    581581    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_1
    582582
    583583    # Group symbols of length 2
    584     temp = bitutil.Advance(temp)
     584    temp = pablo.Advance(temp)
    585585    tag_Callouts.ElemName_ends_2 = temp & remaining_ends
    586586    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_2
    587587
    588588    # Group symbols of length 3
    589     temp = bitutil.Advance(temp)
     589    temp = pablo.Advance(temp)
    590590    tag_Callouts.ElemName_ends_3 = temp & remaining_ends
    591591    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_3
    592592
    593593    # Group symbols of length 4
    594     temp = bitutil.Advance(temp)
     594    temp = pablo.Advance(temp)
    595595    tag_Callouts.ElemName_ends_4 = temp & remaining_ends
    596596    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_4
    597597
    598598    # Group symbols of length 5
    599     temp = bitutil.Advance(temp)
     599    temp = pablo.Advance(temp)
    600600    tag_Callouts.ElemName_ends_5 = temp & remaining_ends
    601601    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_5
    602602
    603603    # Group symbols of length 6
    604     temp = bitutil.Advance(temp)
     604    temp = pablo.Advance(temp)
    605605    tag_Callouts.ElemName_ends_6 = temp & remaining_ends
    606606    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_6
    607607
    608608    # Group symbols of length 7
    609     temp = bitutil.Advance(temp)
     609    temp = pablo.Advance(temp)
    610610    tag_Callouts.ElemName_ends_7 = temp & remaining_ends
    611611    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_7
    612612
    613613    # Group symbols of length 8
    614     temp = bitutil.Advance(temp)
     614    temp = pablo.Advance(temp)
    615615    tag_Callouts.ElemName_ends_8 = temp & remaining_ends
    616616    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_8
    617617
    618618    # Group symbols of length 9
    619     temp = bitutil.Advance(temp)
     619    temp = pablo.Advance(temp)
    620620    tag_Callouts.ElemName_ends_9 = temp & remaining_ends
    621621    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_9
    622622
    623623    # Group symbols of length 10
    624     temp = bitutil.Advance(temp)
     624    temp = pablo.Advance(temp)
    625625    tag_Callouts.ElemName_ends_10 = temp & remaining_ends
    626626    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_10
    627627
    628628    # Group symbols of length 11
    629     temp = bitutil.Advance(temp)
     629    temp = pablo.Advance(temp)
    630630    tag_Callouts.ElemName_ends_11 = temp & remaining_ends
    631631    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_11
    632632
    633633    # Group symbols of length 12
    634     temp = bitutil.Advance(temp)
     634    temp = pablo.Advance(temp)
    635635    tag_Callouts.ElemName_ends_12 = temp & remaining_ends
    636636    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_12
    637637
    638638    # Group symbols of length 13
    639     temp = bitutil.Advance(temp)
     639    temp = pablo.Advance(temp)
    640640    tag_Callouts.ElemName_ends_13 = temp & remaining_ends
    641641    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_13
    642642
    643643    # Group symbols of length 14
    644     temp = bitutil.Advance(temp)
     644    temp = pablo.Advance(temp)
    645645    tag_Callouts.ElemName_ends_14 = temp & remaining_ends
    646646    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_14
    647647
    648648    # Group symbols of length 15
    649     temp = bitutil.Advance(temp)
     649    temp = pablo.Advance(temp)
    650650    tag_Callouts.ElemName_ends_15 = temp & remaining_ends
    651651    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_15
    652652
    653653    # Group symbols of length 16
    654     temp = bitutil.Advance(temp)
     654    temp = pablo.Advance(temp)
    655655    tag_Callouts.ElemName_ends_16 = temp & remaining_ends
    656656    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_16
  • proto/SymbolTable/parabix2_symtab_pbs_div.py

    r1387 r1442  
    1414
    1515#import bitutil
     16
     17class Basis_bits():
     18        bit_0 = 0
     19        bit_1 = 0
     20        bit_2 = 0
     21        bit_3 = 0
     22        bit_4 = 0
     23        bit_5 = 0
     24        bit_6 = 0
     25        bit_7 = 0
     26
    1627
    1728class u8 ():
     
    7586        WS = 0
    7687
    77 class Scope1 ():
    78         RefStart = 0
    79         LAngle = 0
    80         Hyphen = 0
    81         QMark = 0
    82         RBracket = 0
     88class Marker ():
     89        LAngle_scope = 0
     90        Ref_opener = 0
     91        CD_closer = 0
    8392
    8493class CtCDPI_Callouts():
    85         CD_end = 0
    8694        Ct_starts = 0
    8795        Ct_ends = 0
     
    92100        PI_name_ends = 0
    93101        PI_ends = 0
    94         CtCDPI_mask = 0
    95102
    96103class Ref_Callouts():
     
    125132        EndTag_marks = 0
    126133
    127 class Basis_bits():
    128         bit_0 = 0
    129         bit_1 = 0
    130         bit_2 = 0
    131         bit_3 = 0
    132         bit_4 = 0
    133         bit_5 = 0
    134         bit_6 = 0
    135         bit_7 = 0
    136 
    137134class Check_streams():
    138135        misc_mask = 0
     
    142139        name_follows = 0
    143140        att_refs = 0
    144 
    145 class Xml_names():
    146         namespace_error = 0
    147141
    148142def Classify_bytes_Validate_utf8(basis_bits, lex, u8):
     
    240234        lex.Hex = (temp62 | temp65);
    241235        lex_error = x00_x1F &~ lex.WS
    242         if lex_error & EOF_mask:
     236        if pablo.inFile(lex_error):
    243237                error_tracker.NoteError("Error: illegal character", lex_error)
    244238
     
    264258                u8.badprefix = (temp68 | temp71);
    265259                u8_error = u8.badprefix
    266                 u8.scope22 = bitutil.Advance(u8.prefix2)
     260                u8.scope22 = pablo.Advance(u8.prefix2)
    267261                u8anyscope = u8.scope22
    268262                if u8.prefix3 | u8.prefix4:
     
    280274                        u8.xBF = (temp73 & temp23);
    281275                        u8.xBE = (temp73 & temp15);
    282                         u8.xE0_scope = bitutil.Advance(xE0);
    283                         u8.xED_scope = bitutil.Advance(xED);
    284                         u8.xF0_scope = bitutil.Advance(xF0);
    285                         u8.xF4_scope = bitutil.Advance(xF4);
    286                         u8.xEF_scope = bitutil.Advance(xEF);
    287                         u8.scope32 = bitutil.Advance(u8.prefix3)
    288                         u8.scope33 = bitutil.Advance(u8.scope32)
    289                         u8.scope42 = bitutil.Advance(u8.prefix4)
    290                         u8.scope43 = bitutil.Advance(u8.scope42)
    291                         u8.scope44 = bitutil.Advance(u8.scope43)
     276#
     277#
     278#                       scope3_32 = pablo.Advance32(u8.prefix3)
     279#                       scope4_32 = pablo.Advance32(u8.prefix4)
     280#                       u8.scope32 = interpose32(u8.prefix3, scope3_32, 1)
     281#                       u8.scope33 = interpose32(u8.prefix3, scope3_32, 2)
     282#                       u8.scope42 = interpose32(u8.prefix4, scope4_32, 1)
     283#                       u8.scope43 = interpose32(u8.prefix4, scope4_32, 2)
     284#                       u8.scope44 = interpose32(u8.prefix4, scope4_32, 3)
     285#
     286                        u8.scope32 = pablo.Advance(u8.prefix3)
     287                        u8.scope33 = pablo.Advance(u8.scope32)
     288                        u8.scope42 = pablo.Advance(u8.prefix4)
     289                        u8.scope43 = pablo.Advance(u8.scope42)
     290                        u8.scope44 = pablo.Advance(u8.scope43)
     291#
     292#                       u8.xE0_scope = pablo.Advance(xE0);
     293#                       u8.xED_scope = pablo.Advance(xED);
     294#                       u8.xF0_scope = pablo.Advance(xF0);
     295#                       u8.xF4_scope = pablo.Advance(xF4);
     296                        E0_F0_scope = pablo.Advance(xE0 | xF0)
     297                        ED_F4_scope = pablo.Advance(xED | xF4)
     298                        u8.xE0_scope = u8.scope32 & E0_F0_scope
     299                        u8.xED_scope = u8.scope32 & ED_F4_scope
     300                        u8.xF0_scope = u8.scope42 & E0_F0_scope
     301                        u8.xF4_scope = u8.scope42 & ED_F4_scope
     302                        u8.xEF_scope = pablo.Advance(xEF);
    292303
    293304                        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
     
    301312                        u8_error |= u8error1 | u8error2 | u8error3 | u8error4
    302313
    303                         EF_BF_pending = bitutil.Advance(u8.xEF_scope & u8.xBF)
     314                        EF_BF_pending = pablo.Advance(u8.xEF_scope & u8.xBF)
    304315
    305316                        u8_FFFE_FFFF = (EF_BF_pending & (u8.xBE | u8.xBF))
     
    310321
    311322
    312 def Add_scope_streams(lex, scope1):
    313         #scope1.LAngle = bitutil.Advance(lex.LAngle)
    314         #scope1.Hyphen = bitutil.Advance(lex.Hyphen)
    315         #scope1.QMark = bitutil.Advance(lex.QMark)
    316         v = lex.LAngle | lex.Hyphen
    317         w = lex.Hyphen | lex.QMark
    318         v1 = bitutil.Advance(v)
    319         w1 = bitutil.Advance(w)
    320         scope1.LAngle = v1 &~ w1
    321         scope1.Hyphen = v1 & w1
    322         scope1.QMark = w1 &~ v1
    323         scope1.RefStart = 0 # default
    324 
    325 def Parse_CtCDPI(ctCDPI_Callouts, lex, scope1, check_streams):
    326         ctCDPI_Callouts.CD_end = 0
     323def Parse_CtCDPI(lex, marker, ctCDPI_Callouts, check_streams):
    327324        ctCDPI_Callouts.Ct_starts = 0
    328325        ctCDPI_Callouts.Ct_ends = 0
     
    333330        ctCDPI_Callouts.PI_name_ends = 0
    334331        ctCDPI_Callouts.PI_ends = 0
    335         ctCDPI_Callouts.CtCDPI_mask = 0
    336         ctCDPI_error = 0
    337332        CtCDPI_starts = 0
    338         Ct_errors = 0
    339 
     333        CtCDPI_ends = 0
     334        ctCDPI_mask = 0
     335
     336        v = lex.LAngle | lex.Hyphen
     337        w = lex.Hyphen | lex.QMark
     338        v1 = pablo.Advance(v)
     339        w1 = pablo.Advance(w)
     340
     341        LAngle_scope = v1 &~ w1  #pablo.Advance(lex.LAngle)
     342        PI_opener = LAngle_scope & lex.QMark
     343        CtCD_opener= LAngle_scope & lex.Exclam
     344        CtCDPI_opener = PI_opener | CtCD_opener
     345
     346        #DoubleHyphen = 0
     347        CD_closer = 0
     348        #PI_closer = 0
     349
     350        #if lex.Hyphen: DoubleHyphen = pablo.Advance(lex.Hyphen) & lex.Hyphen
     351        DoubleHyphen = v1 & w1 & lex.Hyphen
    340352        if lex.RBracket:
    341                 scope1.RBracket = bitutil.Advance(lex.RBracket)
    342                 ctCDPI_Callouts.CD_end = bitutil.Advance(scope1.RBracket & lex.RBracket) & lex.RAngle
    343         PI_start = scope1.LAngle & lex.QMark
    344         CtCD_start = scope1.LAngle & lex.Exclam
    345         CtCDPI_start = PI_start | CtCD_start
    346 
    347         DoubleHyphen = scope1.Hyphen & lex.Hyphen
    348         PI_end = scope1.QMark & lex.RAngle
    349 
     353                DoubleRBracket = pablo.Advance(lex.RBracket) & lex.RBracket
     354                CD_closer = pablo.Advance(DoubleRBracket) & lex.RAngle
     355        #if lex.QMark: PI_closer = pablo.Advance(lex.QMark) & lex.RAngle
     356        PI_closer = w1 & ~v1 & lex.RAngle
    350357
    351358        #
    352359        # Initiate the scan
    353         CtCDPI_Cursor = bitutil.ScanToFirst(CtCDPI_start)
     360        CtCDPI_Cursor = pablo.ScanToFirst(CtCDPI_opener)
    354361        while CtCDPI_Cursor:
    355362                CtCDPI_starts |= CtCDPI_Cursor
    356                 PI_Cursor = CtCDPI_Cursor & PI_start
    357                 CD_Ct_Cursor = bitutil.Advance(CtCDPI_Cursor & ~PI_Cursor)
     363                PI_Cursor = CtCDPI_Cursor & PI_opener
     364                CD_Ct_Cursor = pablo.Advance(CtCDPI_Cursor & ~PI_Cursor)
    358365                CD_Cursor = CD_Ct_Cursor & lex.LBracket
    359366                Ct_Cursor = CD_Ct_Cursor & lex.Hyphen
    360                 ctCDPI_Callouts.PI_starts |= PI_Cursor
    361                 ctCDPI_Callouts.CD_starts |= CD_Cursor
    362                 ctCDPI_Callouts.Ct_starts |= Ct_Cursor
    363                 Ct_Cursor = bitutil.Advance(Ct_Cursor)
    364                 Ct_errors |= Ct_Cursor & ~ lex.Hyphen
    365                 # Advance twice past <!--, so that we don't treat <!---
    366                 # as being a terminated comment.
    367                 Ct_Cursor = bitutil.Advance(bitutil.Advance(Ct_Cursor))
    368                 PI_Cursor = bitutil.Advance(PI_Cursor)
    369                 ctCDPI_Callouts.PI_name_starts |= PI_Cursor
    370                 PI_name_end = bitutil.ScanThru(PI_Cursor, lex.NameScan)
    371                 ctCDPI_Callouts.PI_name_ends |= PI_name_end
    372                 PI_Cursor = bitutil.ScanTo(PI_name_end, PI_end)
    373                 CD_Cursor = bitutil.ScanTo(CD_Cursor, ctCDPI_Callouts.CD_end)
    374                 Ct_Cursor = bitutil.Advance(bitutil.ScanTo(Ct_Cursor, DoubleHyphen))
    375                 ctCDPI_Callouts.PI_ends |= PI_Cursor
    376                 ctCDPI_Callouts.CD_ends |= CD_Cursor
    377                 ctCDPI_Callouts.Ct_ends |= Ct_Cursor
     367                # PI processing
     368                if PI_Cursor:
     369                        ctCDPI_Callouts.PI_starts |= PI_Cursor
     370                        PI_Cursor = pablo.Advance(PI_Cursor)
     371                        ctCDPI_Callouts.PI_name_starts |= PI_Cursor
     372                        PI_name_end = pablo.ScanThru(PI_Cursor, lex.NameScan)
     373                        PI_error = PI_Cursor & PI_name_end
     374                        PI_error |= pablo.Advance(PI_name_end & ~ lex.WS) & ~ PI_closer
     375                        if PI_error:
     376                                error_tracker.NoteError("Error in PI syntax", PI_name_end)
     377                        ctCDPI_Callouts.PI_name_ends |= PI_name_end
     378                        PI_Cursor = pablo.ScanTo(PI_name_end, PI_closer)
     379                        ctCDPI_Callouts.PI_ends |= PI_Cursor
     380                        CtCDPI_ends |= PI_Cursor
     381
     382                # CDATA section processing
     383                if CD_Cursor:
     384                        ctCDPI_Callouts.CD_starts |= CD_Cursor
     385                        CD_Cursor = pablo.ScanTo(CD_Cursor, CD_closer)
     386                        ctCDPI_Callouts.CD_ends |= CD_Cursor
     387                        CtCDPI_ends |= CD_Cursor
     388
     389                # Comment processing
     390                if Ct_Cursor:
     391                        ctCDPI_Callouts.Ct_starts |= Ct_Cursor
     392                        Ct_Cursor = pablo.Advance(Ct_Cursor)
     393                        Ct_error = Ct_Cursor & ~ lex.Hyphen
     394                        # Advance twice past <!--, so that we don't treat <!---
     395                        # as being a terminated comment.
     396                        Ct_Cursor = pablo.Advance(pablo.Advance(Ct_Cursor))
     397                        Ct_Cursor = pablo.Advance(pablo.ScanTo(Ct_Cursor, DoubleHyphen))
     398                        Ct_error |= Ct_Cursor & ~ lex.RAngle
     399                        ctCDPI_Callouts.Ct_ends |= Ct_Cursor
     400                        CtCDPI_ends |= Ct_Cursor
     401                        if Ct_error:
     402                                error_tracker.NoteError("Error in comment syntax", Ct_error)
     403
     404                # Common processing
    378405                CtCDPI_Cursor = PI_Cursor | CD_Cursor | Ct_Cursor
    379                 CtCDPI_Cursor = bitutil.ScanTo(CtCDPI_Cursor, CtCDPI_start)
    380 
    381                 ctCDPI_Callouts.CtCDPI_mask = bitutil.Advance(ctCDPI_Callouts.CD_ends | ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - CtCDPI_starts
    382                 #ctCDPI_Callouts.error = Ct_ends & ~lex.RAngle | Ct_starts & ~ lex.Hyphen
    383                 ctCDPI_error = Ct_errors | ctCDPI_Callouts.Ct_ends & ~lex.RAngle
    384                 ctCDPI_error |= bitutil.Advance(ctCDPI_Callouts.PI_name_ends & ~ lex.WS) & ~ PI_end
    385                 ctCDPI_error |= ctCDPI_Callouts.PI_name_starts & ctCDPI_Callouts.PI_name_ends
     406                CtCDPI_Cursor = pablo.ScanTo(CtCDPI_Cursor, CtCDPI_opener)
     407                ctCDPI_mask |= (CtCDPI_ends - CtCDPI_starts) | CtCDPI_ends
    386408                # If any of the Comment, CDATA or PI markups are unterminated, it is an error.
    387                 ctCDPI_error |= ctCDPI_Callouts.CtCDPI_mask &~ EOF_mask
    388 
    389         if ctCDPI_error:
    390                 error_tracker.NoteError("Error in comment, CDATA or processing instruction syntax", ctCDPI_error)
    391 
    392         check_streams.misc_mask = (lex.WS | lex.LAngle | (bitutil.Advance(ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - (ctCDPI_Callouts.Ct_starts | ctCDPI_Callouts.PI_starts)) | CtCDPI_starts) & EOF_mask
    393 
    394 def Parse_tags(lex, scope1, ctCDPI_Callouts, tag_Callouts):
    395 
     409                ctCDPI_error = pablo.atEOF(ctCDPI_mask)
     410
     411                if ctCDPI_error:
     412                        error_tracker.NoteError("Error in comment, CDATA or processing instruction syntax", ctCDPI_error)
     413
     414        check_streams.misc_mask = (lex.WS | lex.LAngle | (pablo.Advance(ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - (ctCDPI_Callouts.Ct_starts | ctCDPI_Callouts.PI_starts)) | CtCDPI_starts) & EOF_mask
     415
     416
     417        # Identify the remaining significant markers for XML processing.
     418        marker.LAngle_scope = LAngle_scope &~ ctCDPI_mask
     419        marker.Ref_opener = lex.RefStart &~ ctCDPI_mask
     420        marker.CD_closer = CD_closer &~ ctCDPI_mask
     421
     422def Parse_tags(lex, marker, tag_Callouts):
    396423
    397424        # Delimiters for scans.
     
    402429        # Start the parallel parsing by inspecting the character
    403430        # after the opening "<" of a tag.
    404         LAngleFollow = scope1.LAngle &~ ctCDPI_Callouts.CtCDPI_mask
    405         tag_Callouts.ElemName_starts = LAngleFollow & ~lex.Slash
    406         tag_Callouts.EndTag_marks = LAngleFollow & lex.Slash
     431        tag_Callouts.ElemName_starts = marker.LAngle_scope & ~lex.Slash
     432        tag_Callouts.EndTag_marks = marker.LAngle_scope & lex.Slash
    407433
    408434        # Start Tag/Empty Element Tag Parsing
    409435
    410436        # Advance all cursors by scanning through the tag name.
    411         tag_Callouts.ElemName_ends = bitutil.ScanThru(tag_Callouts.ElemName_starts, lex.NameScan)
     437        tag_Callouts.ElemName_ends = pablo.ScanThru(tag_Callouts.ElemName_starts, lex.NameScan)
    412438        # Must have at least one name character for a legal start tag.
    413439        # Mark any occurrences of null names as errors.
     
    417443        tag_Callouts.AttName_starts = 0
    418444        tag_Callouts.AttName_ends = 0
    419         EqToCheck = 0
    420445        tag_Callouts.AttVal_starts = 0
    421         AttValEnds = 0
    422446        tag_Callouts.AttVal_ends = 0
    423447
    424448        # After the element name, there may or may not be an attlist.
    425         AfterWS = bitutil.ScanThru(tag_Callouts.ElemName_ends, lex.WS)
    426         AttListEnd = AfterWS & AttListDelim
    427         AttNameStart = AfterWS & ~AttListDelim
    428         # At least one WS character is required between ElemNames and AttNames.
    429         ParseError |= tag_Callouts.ElemName_ends & AttNameStart
    430 
    431         #
    432         # The following loop iterates through attributes within a start tag.
    433         # Because all start tags are processed in parallel, the number of
    434         # iterations is the maximum number of attributes found in any one
    435         # start tag, plus one.
    436         while AttNameStart:
    437                 tag_Callouts.AttName_starts |= AttNameStart
    438                 AttNameFollow = bitutil.ScanThru(AttNameStart, lex.NameScan)
    439                 tag_Callouts.AttName_ends |= AttNameFollow
    440                 # Scan through WS to the expected '=' delimiter.
    441                 # EqExpected = bitutil.ScanThru(AttNameFollow, lex.WS)
    442                 # But use if test to optimize.
    443                 if AttNameFollow & lex.WS:
    444                         EqExpected = bitutil.ScanThru(AttNameFollow, lex.WS)
    445                 else: EqExpected = AttNameFollow
    446                 EqToCheck |= EqExpected
    447                 AttValPos = bitutil.ScanThru(EqExpected, EqExpected | lex.WS)
    448                 tag_Callouts.AttVal_starts |= AttValPos
    449                 DQuoteAttVal = AttValPos & lex.DQuote
    450                 SQuoteAttVal = AttValPos & lex.SQuote
    451                 DQuoteAttEnd = bitutil.ScanTo(DQuoteAttVal, DQuoteDelim &~ DQuoteAttVal)
    452                 SQuoteAttEnd = bitutil.ScanTo(SQuoteAttVal, SQuoteDelim &~ SQuoteAttVal)
    453                 AttValEnd = DQuoteAttEnd | SQuoteAttEnd
    454                 AttValEnds |= AttValEnd
    455                 AttValFollow = bitutil.Advance(AttValEnd)
    456                 tag_Callouts.AttVal_ends |= AttValFollow
    457                 #  AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
    458                 if AttValFollow & lex.WS:
    459                         AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
    460                 else: AfterWS = AttValFollow
    461                 AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
    462                 AttListEnd |= AfterWS & AttListDelim
     449        if tag_Callouts.ElemName_ends & lex.WS:
     450                AfterWS = pablo.ScanThru(tag_Callouts.ElemName_ends, lex.WS)
     451                AttListEnd = AfterWS & AttListDelim
    463452                AttNameStart = AfterWS & ~AttListDelim
    464 
    465         # No more attribute values to process when AttNameStart == 0.
     453                #
     454                # The following loop iterates through attributes within a start tag.
     455                # Because all start tags are processed in parallel, the number of
     456                # iterations is the maximum number of attributes found in any one
     457                # start tag, plus one.
     458                while AttNameStart:
     459                        ParseError |= AttNameStart &~ lex.NameScan
     460                        tag_Callouts.AttName_starts |= AttNameStart
     461                        AttNameFollow = pablo.ScanThru(AttNameStart, lex.NameScan)
     462                        tag_Callouts.AttName_ends |= AttNameFollow
     463                        # Scan through WS to the expected '=' delimiter.
     464                        # EqExpected = pablo.ScanThru(AttNameFollow, lex.WS)
     465                        # But use if test to optimize.
     466                        if AttNameFollow & lex.WS:
     467                                EqExpected = pablo.ScanThru(AttNameFollow, lex.WS)
     468                        else: EqExpected = AttNameFollow
     469                        ParseError |= EqExpected &~ lex.Equals
     470                        AttValPos = pablo.ScanThru(EqExpected, EqExpected | lex.WS)
     471                        tag_Callouts.AttVal_starts |= AttValPos
     472                        DQuoteAttVal = AttValPos & lex.DQuote
     473                        SQuoteAttVal = AttValPos & lex.SQuote
     474                        DQuoteAttEnd = pablo.ScanTo(DQuoteAttVal, DQuoteDelim &~ DQuoteAttVal)
     475                        SQuoteAttEnd = pablo.ScanTo(SQuoteAttVal, SQuoteDelim &~ SQuoteAttVal)
     476                        AttValEnd = DQuoteAttEnd | SQuoteAttEnd
     477                        ParseError |= (AttValPos | AttValEnd) &~ (lex.DQuote | lex.SQuote)
     478                        AttValFollow = pablo.Advance(AttValEnd)
     479                        tag_Callouts.AttVal_ends |= AttValFollow
     480                        #  AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
     481                        if AttValFollow & lex.WS:
     482                                AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
     483                                AttListEnd |= AfterWS & AttListDelim
     484                                AttNameStart = AfterWS & ~AttListDelim
     485                        else:
     486                                AttListEnd |= AttValFollow & AttListDelim
     487                                AttNameStart = 0
     488                                ParseError |= AttValFollow & ~AttListDelim
     489        else:
     490                # No WS character after ElemName; must be at the end
     491                AttListEnd = tag_Callouts.ElemName_ends & AttListDelim
     492                ParseError |= tag_Callouts.ElemName_ends & ~AttListDelim
     493
    466494        STagEnds = AttListEnd & lex.RAngle
    467495        # Mark any "/" characters found as the ends of empty element tags.
    468         tag_Callouts.EmptyTag_marks = bitutil.Advance(AttListEnd & lex.Slash)
    469 
    470         # Check for errors.
    471         ParseError |= tag_Callouts.AttVal_ends & tag_Callouts.AttName_starts # No intervening WS.
    472         ParseError |= tag_Callouts.AttName_starts & tag_Callouts.AttName_ends # Null AttName
    473         ParseError |= EqToCheck & ~lex.Equals # = not found where expected.
    474         ParseError |= tag_Callouts.AttVal_starts & ~ (lex.DQuote | lex.SQuote)
    475         ParseError |= AttValEnds & ~ (lex.DQuote | lex.SQuote)
     496        tag_Callouts.EmptyTag_marks = pablo.Advance(AttListEnd & lex.Slash)
     497
    476498        ParseError |= tag_Callouts.EmptyTag_marks & ~lex.RAngle
    477499
    478500        # End Tag Parsing
    479501
    480         EndTagEnds = bitutil.ScanThru(tag_Callouts.EndTag_marks, tag_Callouts.EndTag_marks | lex.NameScan)
     502        EndTagEnds = pablo.ScanThru(tag_Callouts.EndTag_marks, tag_Callouts.EndTag_marks | lex.NameScan)
    481503        if EndTagEnds & lex.WS:
    482                 EndTagEnds = bitutil.ScanThru(EndTagEnds, lex.WS)
     504                EndTagEnds = pablo.ScanThru(EndTagEnds, lex.WS)
    483505        ParseError |= EndTagEnds & ~lex.RAngle
    484506        if ParseError:
     
    489511        tag_Callouts.AttVal_spans = tag_Callouts.AttVal_ends - tag_Callouts.AttVal_starts
    490512
    491 def Parse_refs(lex, scope1, ctCDPI_Callouts, ref_Callouts):
     513def Parse_refs(lex, marker, ref_Callouts):
    492514        ref_Callouts.GenRef_starts = 0
    493515        ref_Callouts.GenRef_ends = 0
     
    498520        ref_error = 0
    499521
    500         Ref1 = lex.RefStart &~ ctCDPI_Callouts.CtCDPI_mask
    501522        # All remaining "&" must be reference start characters; parse them.
    502         if Ref1:
    503                 scope1.RefStart = bitutil.Advance(Ref1)
    504                 NumRef2 = scope1.RefStart & lex.Hash
    505                 ref_Callouts.GenRef_starts = scope1.RefStart &~ lex.Hash
    506                 NumRef3 = bitutil.Advance(NumRef2)
     523        if marker.Ref_opener:
     524                Ref_scope = pablo.Advance(marker.Ref_opener)
     525                NumRef2 = Ref_scope & lex.Hash
     526                ref_Callouts.GenRef_starts = Ref_scope &~ lex.Hash
     527                NumRef3 = pablo.Advance(NumRef2)
    507528                HexRef3 = NumRef3 & lex.x
    508529                ref_Callouts.DecRef_starts = NumRef3 &~ lex.x
    509                 ref_Callouts.HexRef_starts = bitutil.Advance(HexRef3)
    510                 ref_Callouts.GenRef_ends = bitutil.ScanThru(ref_Callouts.GenRef_starts, lex.NameScan)
    511                 ref_Callouts.DecRef_ends = bitutil.ScanThru(ref_Callouts.DecRef_starts, lex.Digit)
    512                 ref_Callouts.HexRef_ends = bitutil.ScanThru(ref_Callouts.HexRef_starts, lex.Hex)
     530                ref_Callouts.HexRef_starts = pablo.Advance(HexRef3)
     531                ref_Callouts.GenRef_ends = pablo.ScanThru(ref_Callouts.GenRef_starts, lex.NameScan)
     532                ref_Callouts.DecRef_ends = pablo.ScanThru(ref_Callouts.DecRef_starts, lex.Digit)
     533                ref_Callouts.HexRef_ends = pablo.ScanThru(ref_Callouts.HexRef_starts, lex.Hex)
    513534                # Error checks
    514535                # At least one digit required for DecRef, one hex digit for HexRef.
     
    524545
    525546
    526 def Validate_xml_names(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, xml_names, check_streams):
     547def Validate_xml_names(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, check_streams):
    527548        PI_names = ctCDPI_Callouts.PI_name_ends - ctCDPI_Callouts.PI_name_starts
    528549        GenRefs = ref_Callouts.GenRef_ends - ref_Callouts.GenRef_starts
     
    532553        ncname_stream = PI_names | GenRefs
    533554        name_stream = qname_stream | ncname_stream
    534         name_start = name_stream &~ bitutil.Advance(name_stream)
    535         name_cursor = name_stream & ~bitutil.Advance(name_stream)
     555        name_start = name_stream &~ pablo.Advance(name_stream)
     556        name_cursor = name_stream & ~pablo.Advance(name_stream)
    536557        void_prefix_err = name_cursor & lex.Colon
    537         namespace_sep = bitutil.ScanThru(name_cursor, lex.NameScan &~ lex.Colon) & lex.Colon
    538         local_part_start = bitutil.Advance(namespace_sep)
     558        namespace_sep = pablo.ScanThru(name_cursor, lex.NameScan &~ lex.Colon) & lex.Colon
     559        local_part_start = pablo.Advance(namespace_sep)
    539560        local_part_err = local_part_start &~ lex.NameScan
    540         colon2_err = bitutil.ScanThru(local_part_start, lex.NameScan &~ lex.Colon) & lex.Colon
     561        colon2_err = pablo.ScanThru(local_part_start, lex.NameScan &~ lex.Colon) & lex.Colon
    541562        ncname_err = ncname_stream & lex.Colon
    542563        namespace_error = void_prefix_err | local_part_err | colon2_err | ncname_err
     
    548569        check_streams.non_ascii_names = (name_stream &~ name_start) & ~lex.ASCII_name_char & ~u8.suffix
    549570
    550 def Do_check_streams(ctCDPI_Callouts, tag_Callouts, lex, u8, scope1, ref_Callouts, xml_names, check_streams):
    551     # Ensure that no occurrence of ]]> occurs outside of markup.
    552     CD_end_error = ctCDPI_Callouts.CD_end & ~(ctCDPI_Callouts.CtCDPI_mask | tag_Callouts.AttVal_spans)
    553 
    554     # Consolidate and check for errors
    555     if CD_end_error:
     571def Do_check_streams(marker, tag_Callouts, check_streams):
     572        CD_end_error = marker.CD_closer & ~tag_Callouts.AttVal_spans
     573        if CD_end_error:
    556574                error_tracker.NoteError("Error: ]]> in text", CD_end_error)
    557 
    558 
    559     check_streams.tag_marks = tag_Callouts.EmptyTag_marks | tag_Callouts.ElemName_starts | tag_Callouts.EndTag_marks | tag_Callouts.AttName_starts
    560     check_streams.name_follows = tag_Callouts.ElemName_ends | tag_Callouts.AttName_ends
    561     check_streams.att_refs = tag_Callouts.AttVal_spans & scope1.RefStart
     575        check_streams.tag_marks = tag_Callouts.EmptyTag_marks | tag_Callouts.ElemName_starts | tag_Callouts.EndTag_marks | tag_Callouts.AttName_starts
     576        check_streams.name_follows = tag_Callouts.ElemName_ends | tag_Callouts.AttName_ends
     577        check_streams.att_refs = tag_Callouts.AttVal_spans & marker.Ref_opener
    562578
    563579def Form_Length_Group_Bitstreams(tag_Callouts):
     
    566582    ends = tag_Callouts.ElemName_ends
    567583
    568     temp = ends | bitutil.Advance(ends)
     584    temp = ends | pablo.Advance(ends)
    569585
    570586    # Group symbols of length 1 and 2
    571     start_2 = bitutil.Advance(bitutil.Advance(starts))
     587    start_2 = pablo.Advance(pablo.Advance(starts))
    572588    tag_Callouts.ElemName_ends_1_to_2 =  start_2 & temp
    573589
    574590    # Group symbols of length 3 and 4
    575     start_4 = bitutil.Advance(bitutil.Advance(start_2 & ~temp))
     591    start_4 = pablo.Advance(pablo.Advance(start_2 & ~temp))
    576592    tag_Callouts.ElemName_ends_3_to_4 =  start_4 & temp
    577593
    578594    # Group symbols of length 5 and 6
    579     start_6 = bitutil.Advance(bitutil.Advance(start_4 & ~temp))
     595    start_6 = pablo.Advance(pablo.Advance(start_4 & ~temp))
    580596    tag_Callouts.ElemName_ends_5_to_6 =  start_6 & temp
    581597
    582598    # Group symbols of length 7 and 8
    583     start_8 = bitutil.Advance(bitutil.Advance(start_6 & ~temp))
     599    start_8 = pablo.Advance(pablo.Advance(start_6 & ~temp))
    584600    tag_Callouts.ElemName_ends_7_to_8 =  start_8 & temp
    585601
    586602    # Group symbols of length 9 and 10
    587     start_10 = bitutil.Advance(bitutil.Advance(start_8 & ~temp))
     603    start_10 = pablo.Advance(pablo.Advance(start_8 & ~temp))
    588604    tag_Callouts.ElemName_ends_9_to_10 =  start_10 & temp
    589605
    590606    # Group symbols of length 11 and 12
    591     start_12 = bitutil.Advance(bitutil.Advance(start_10 & ~temp))
     607    start_12 = pablo.Advance(pablo.Advance(start_10 & ~temp))
    592608    tag_Callouts.ElemName_ends_11_to_12 =  start_12 & temp
    593609
    594610    # Group symbols of length 13 and 14
    595     start_14 = bitutil.Advance(bitutil.Advance(start_12 & ~temp))
     611    start_14 = pablo.Advance(pablo.Advance(start_12 & ~temp))
    596612    tag_Callouts.ElemName_ends_13_to_14 =  start_14 & temp
    597613
    598614    # Group symbols of length 15 and 16
    599     start_16 = bitutil.Advance(bitutil.Advance(start_14 & ~temp))
     615    start_16 = pablo.Advance(pablo.Advance(start_14 & ~temp))
    600616    tag_Callouts.ElemName_ends_15_to_16 =  start_16 & temp
    601617
     
    619635        Classify_bytes_Validate_utf8(basis_bits, lex, u8)
    620636
    621         Add_scope_streams(lex, scope1)
    622 
    623637        # Parse all comments, CDATA sections and processing instructions.
    624         Parse_CtCDPI(ctCDPI_Callouts, lex, scope1, check_streams)
     638        Parse_CtCDPI(lex, marker, ctCDPI_Callouts, check_streams)
    625639
    626640        # All remaining '<' must be tag start characters; parse tags.
    627         Parse_tags(lex, scope1, ctCDPI_Callouts, tag_Callouts)
     641        Parse_tags(lex, marker, tag_Callouts)
    628642
    629643        # All remaining '&' must be reference start characters; parse them.
    630         Parse_refs(lex, scope1, ctCDPI_Callouts, ref_Callouts)
     644        Parse_refs(lex, marker, ref_Callouts)
    631645
    632646        # Validate XML namespaces and generate bit streams to post validate non-ascii range XML names
    633         Validate_xml_names(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, xml_names, check_streams)
    634 
    635 
    636         Do_check_streams(ctCDPI_Callouts, tag_Callouts, lex, u8, scope1, ref_Callouts, xml_names, check_streams)
     647        Validate_xml_names(ctCDPI_Callouts, ref_Callouts, tag_Callouts, lex, u8, check_streams)
     648
     649        Do_check_streams(marker, tag_Callouts, check_streams)
    637650
    638651        # These methods are needed to do Paralel Bitstream Based Length Sorting
  • proto/SymbolTable/parabix2_symtab_pbs_log.py

    r1232 r1442  
    261261                u8.badprefix = (temp68 | temp71);
    262262                u8_error = u8.badprefix
    263                 u8.scope22 = bitutil.Advance(u8.prefix2)
     263                u8.scope22 = pablo.Advance(u8.prefix2)
    264264                u8anyscope = u8.scope22
    265265                if u8.prefix3 | u8.prefix4:
     
    277277                        u8.xBF = (temp73 & temp23);
    278278                        u8.xBE = (temp73 & temp15);
    279                         u8.xE0_scope = bitutil.Advance(xE0);
    280                         u8.xED_scope = bitutil.Advance(xED);
    281                         u8.xF0_scope = bitutil.Advance(xF0);
    282                         u8.xF4_scope = bitutil.Advance(xF4);
    283                         u8.xEF_scope = bitutil.Advance(xEF);
    284                         u8.scope32 = bitutil.Advance(u8.prefix3)
    285                         u8.scope33 = bitutil.Advance(u8.scope32)
    286                         u8.scope42 = bitutil.Advance(u8.prefix4)
    287                         u8.scope43 = bitutil.Advance(u8.scope42)
    288                         u8.scope44 = bitutil.Advance(u8.scope43)
     279                        u8.xE0_scope = pablo.Advance(xE0);
     280                        u8.xED_scope = pablo.Advance(xED);
     281                        u8.xF0_scope = pablo.Advance(xF0);
     282                        u8.xF4_scope = pablo.Advance(xF4);
     283                        u8.xEF_scope = pablo.Advance(xEF);
     284                        u8.scope32 = pablo.Advance(u8.prefix3)
     285                        u8.scope33 = pablo.Advance(u8.scope32)
     286                        u8.scope42 = pablo.Advance(u8.prefix4)
     287                        u8.scope43 = pablo.Advance(u8.scope42)
     288                        u8.scope44 = pablo.Advance(u8.scope43)
    289289
    290290                        u8lastscope = u8.scope22 | u8.scope33 | u8.scope44
     
    298298                        u8_error |= u8error1 | u8error2 | u8error3 | u8error4
    299299
    300                         EF_BF_pending = bitutil.Advance(u8.xEF_scope & u8.xBF)
     300                        EF_BF_pending = pablo.Advance(u8.xEF_scope & u8.xBF)
    301301
    302302                        u8_FFFE_FFFF = (EF_BF_pending & (u8.xBE | u8.xBF))
     
    308308
    309309def Add_scope_streams(lex, scope1):
    310         #scope1.LAngle = bitutil.Advance(lex.LAngle)
    311         #scope1.Hyphen = bitutil.Advance(lex.Hyphen)
    312         #scope1.QMark = bitutil.Advance(lex.QMark)
     310        #scope1.LAngle = pablo.Advance(lex.LAngle)
     311        #scope1.Hyphen = pablo.Advance(lex.Hyphen)
     312        #scope1.QMark = pablo.Advance(lex.QMark)
    313313        v = lex.LAngle | lex.Hyphen
    314314        w = lex.Hyphen | lex.QMark
    315         v1 = bitutil.Advance(v)
    316         w1 = bitutil.Advance(w)
     315        v1 = pablo.Advance(v)
     316        w1 = pablo.Advance(w)
    317317        scope1.LAngle = v1 &~ w1
    318318        scope1.Hyphen = v1 & w1
     
    336336
    337337        if lex.RBracket:
    338                 scope1.RBracket = bitutil.Advance(lex.RBracket)
    339                 ctCDPI_Callouts.CD_end = bitutil.Advance(scope1.RBracket & lex.RBracket) & lex.RAngle
     338                scope1.RBracket = pablo.Advance(lex.RBracket)
     339                ctCDPI_Callouts.CD_end = pablo.Advance(scope1.RBracket & lex.RBracket) & lex.RAngle
    340340        PI_start = scope1.LAngle & lex.QMark
    341341        CtCD_start = scope1.LAngle & lex.Exclam
     
    348348        #
    349349        # Initiate the scan
    350         CtCDPI_Cursor = bitutil.ScanToFirst(CtCDPI_start)
     350        CtCDPI_Cursor = pablo.ScanToFirst(CtCDPI_start)
    351351        while CtCDPI_Cursor:
    352352                CtCDPI_starts |= CtCDPI_Cursor
    353353                PI_Cursor = CtCDPI_Cursor & PI_start
    354                 CD_Ct_Cursor = bitutil.Advance(CtCDPI_Cursor & ~PI_Cursor)
     354                CD_Ct_Cursor = pablo.Advance(CtCDPI_Cursor & ~PI_Cursor)
    355355                CD_Cursor = CD_Ct_Cursor & lex.LBracket
    356356                Ct_Cursor = CD_Ct_Cursor & lex.Hyphen
     
    358358                ctCDPI_Callouts.CD_starts |= CD_Cursor
    359359                ctCDPI_Callouts.Ct_starts |= Ct_Cursor
    360                 Ct_Cursor = bitutil.Advance(Ct_Cursor)
     360                Ct_Cursor = pablo.Advance(Ct_Cursor)
    361361                Ct_errors |= Ct_Cursor & ~ lex.Hyphen
    362362                # Advance twice past <!--, so that we don't treat <!---
    363363                # as being a terminated comment.
    364                 Ct_Cursor = bitutil.Advance(bitutil.Advance(Ct_Cursor))
    365                 PI_Cursor = bitutil.Advance(PI_Cursor)
     364                Ct_Cursor = pablo.Advance(pablo.Advance(Ct_Cursor))
     365                PI_Cursor = pablo.Advance(PI_Cursor)
    366366                ctCDPI_Callouts.PI_name_starts |= PI_Cursor
    367                 PI_name_end = bitutil.ScanThru(PI_Cursor, lex.NameScan)
     367                PI_name_end = pablo.ScanThru(PI_Cursor, lex.NameScan)
    368368                ctCDPI_Callouts.PI_name_ends |= PI_name_end
    369                 PI_Cursor = bitutil.ScanTo(PI_name_end, PI_end)
    370                 CD_Cursor = bitutil.ScanTo(CD_Cursor, ctCDPI_Callouts.CD_end)
    371                 Ct_Cursor = bitutil.Advance(bitutil.ScanTo(Ct_Cursor, DoubleHyphen))
     369                PI_Cursor = pablo.ScanTo(PI_name_end, PI_end)
     370                CD_Cursor = pablo.ScanTo(CD_Cursor, ctCDPI_Callouts.CD_end)
     371                Ct_Cursor = pablo.Advance(pablo.ScanTo(Ct_Cursor, DoubleHyphen))
    372372                ctCDPI_Callouts.PI_ends |= PI_Cursor
    373373                ctCDPI_Callouts.CD_ends |= CD_Cursor
    374374                ctCDPI_Callouts.Ct_ends |= Ct_Cursor
    375375                CtCDPI_Cursor = PI_Cursor | CD_Cursor | Ct_Cursor
    376                 CtCDPI_Cursor = bitutil.ScanTo(CtCDPI_Cursor, CtCDPI_start)
    377 
    378                 ctCDPI_Callouts.CtCDPI_mask = bitutil.Advance(ctCDPI_Callouts.CD_ends | ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - CtCDPI_starts
     376                CtCDPI_Cursor = pablo.ScanTo(CtCDPI_Cursor, CtCDPI_start)
     377
     378                ctCDPI_Callouts.CtCDPI_mask = pablo.Advance(ctCDPI_Callouts.CD_ends | ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - CtCDPI_starts
    379379                #ctCDPI_Callouts.error = Ct_ends & ~lex.RAngle | Ct_starts & ~ lex.Hyphen
    380380                ctCDPI_error = Ct_errors | ctCDPI_Callouts.Ct_ends & ~lex.RAngle
    381                 ctCDPI_error |= bitutil.Advance(ctCDPI_Callouts.PI_name_ends & ~ lex.WS) & ~ PI_end
     381                ctCDPI_error |= pablo.Advance(ctCDPI_Callouts.PI_name_ends & ~ lex.WS) & ~ PI_end
    382382                ctCDPI_error |= ctCDPI_Callouts.PI_name_starts & ctCDPI_Callouts.PI_name_ends
    383383                # If any of the Comment, CDATA or PI markups are unterminated, it is an error.
     
    387387                error_tracker.NoteError("Error in comment, CDATA or processing instruction syntax", ctCDPI_error)
    388388
    389         check_streams.misc_mask = (lex.WS | lex.LAngle | (bitutil.Advance(ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - (ctCDPI_Callouts.Ct_starts | ctCDPI_Callouts.PI_starts)) | CtCDPI_starts) & EOF_mask
     389        check_streams.misc_mask = (lex.WS | lex.LAngle | (pablo.Advance(ctCDPI_Callouts.Ct_ends | ctCDPI_Callouts.PI_ends) - (ctCDPI_Callouts.Ct_starts | ctCDPI_Callouts.PI_starts)) | CtCDPI_starts) & EOF_mask
    390390
    391391def Parse_tags(lex, scope1, ctCDPI_Callouts, tag_Callouts):
     
    406406
    407407        # Advance all cursors by scanning through the tag name.
    408         tag_Callouts.ElemName_ends = bitutil.ScanThru(tag_Callouts.ElemName_starts, lex.NameScan)
     408        tag_Callouts.ElemName_ends = pablo.ScanThru(tag_Callouts.ElemName_starts, lex.NameScan)
    409409        # Must have at least one name character for a legal start tag.
    410410        # Mark any occurrences of null names as errors.
     
    420420
    421421        # After the element name, there may or may not be an attlist.
    422         AfterWS = bitutil.ScanThru(tag_Callouts.ElemName_ends, lex.WS)
     422        AfterWS = pablo.ScanThru(tag_Callouts.ElemName_ends, lex.WS)
    423423        AttListEnd = AfterWS & AttListDelim
    424424        AttNameStart = AfterWS & ~AttListDelim
     
    433433        while AttNameStart:
    434434                tag_Callouts.AttName_starts |= AttNameStart
    435                 AttNameFollow = bitutil.ScanThru(AttNameStart, lex.NameScan)
     435                AttNameFollow = pablo.ScanThru(AttNameStart, lex.NameScan)
    436436                tag_Callouts.AttName_ends |= AttNameFollow
    437437                # Scan through WS to the expected '=' delimiter.
    438                 # EqExpected = bitutil.ScanThru(AttNameFollow, lex.WS)
     438                # EqExpected = pablo.ScanThru(AttNameFollow, lex.WS)
    439439                # But use if test to optimize.
    440440                if AttNameFollow & lex.WS:
    441                         EqExpected = bitutil.ScanThru(AttNameFollow, lex.WS)
     441                        EqExpected = pablo.ScanThru(AttNameFollow, lex.WS)
    442442                else: EqExpected = AttNameFollow
    443443                EqToCheck |= EqExpected
    444                 AttValPos = bitutil.ScanThru(EqExpected, EqExpected | lex.WS)
     444                AttValPos = pablo.ScanThru(EqExpected, EqExpected | lex.WS)
    445445                tag_Callouts.AttVal_starts |= AttValPos
    446446                DQuoteAttVal = AttValPos & lex.DQuote
    447447                SQuoteAttVal = AttValPos & lex.SQuote
    448                 DQuoteAttEnd = bitutil.ScanTo(DQuoteAttVal, DQuoteDelim &~ DQuoteAttVal)
    449                 SQuoteAttEnd = bitutil.ScanTo(SQuoteAttVal, SQuoteDelim &~ SQuoteAttVal)
     448                DQuoteAttEnd = pablo.ScanTo(DQuoteAttVal, DQuoteDelim &~ DQuoteAttVal)
     449                SQuoteAttEnd = pablo.ScanTo(SQuoteAttVal, SQuoteDelim &~ SQuoteAttVal)
    450450                AttValEnd = DQuoteAttEnd | SQuoteAttEnd
    451451                AttValEnds |= AttValEnd
    452                 AttValFollow = bitutil.Advance(AttValEnd)
     452                AttValFollow = pablo.Advance(AttValEnd)
    453453                tag_Callouts.AttVal_ends |= AttValFollow
    454                 #  AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
     454                #  AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
    455455                if AttValFollow & lex.WS:
    456                         AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
     456                        AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
    457457                else: AfterWS = AttValFollow
    458                 AfterWS = bitutil.ScanThru(AttValFollow, lex.WS)
     458                AfterWS = pablo.ScanThru(AttValFollow, lex.WS)
    459459                AttListEnd |= AfterWS & AttListDelim
    460460                AttNameStart = AfterWS & ~AttListDelim
     
    463463        STagEnds = AttListEnd & lex.RAngle
    464464        # Mark any "/" characters found as the ends of empty element tags.
    465         tag_Callouts.EmptyTag_marks = bitutil.Advance(AttListEnd & lex.Slash)
     465        tag_Callouts.EmptyTag_marks = pablo.Advance(AttListEnd & lex.Slash)
    466466
    467467        # Check for errors.
     
    475475        # End Tag Parsing
    476476
    477         EndTagEnds = bitutil.ScanThru(tag_Callouts.EndTag_marks, tag_Callouts.EndTag_marks | lex.NameScan)
     477        EndTagEnds = pablo.ScanThru(tag_Callouts.EndTag_marks, tag_Callouts.EndTag_marks | lex.NameScan)
    478478        if EndTagEnds & lex.WS:
    479                 EndTagEnds = bitutil.ScanThru(EndTagEnds, lex.WS)
     479                EndTagEnds = pablo.ScanThru(EndTagEnds, lex.WS)
    480480        ParseError |= EndTagEnds & ~lex.RAngle
    481481        if ParseError:
     
    498498        # All remaining "&" must be reference start characters; parse them.
    499499        if Ref1:
    500                 scope1.RefStart = bitutil.Advance(Ref1)
     500                scope1.RefStart = pablo.Advance(Ref1)
    501501                NumRef2 = scope1.RefStart & lex.Hash
    502502                ref_Callouts.GenRef_starts = scope1.RefStart &~ lex.Hash
    503                 NumRef3 = bitutil.Advance(NumRef2)
     503                NumRef3 = pablo.Advance(NumRef2)
    504504                HexRef3 = NumRef3 & lex.x
    505505                ref_Callouts.DecRef_starts = NumRef3 &~ lex.x
    506                 ref_Callouts.HexRef_starts = bitutil.Advance(HexRef3)
    507                 ref_Callouts.GenRef_ends = bitutil.ScanThru(ref_Callouts.GenRef_starts, lex.NameScan)
    508                 ref_Callouts.DecRef_ends = bitutil.ScanThru(ref_Callouts.DecRef_starts, lex.Digit)
    509                 ref_Callouts.HexRef_ends = bitutil.ScanThru(ref_Callouts.HexRef_starts, lex.Hex)
     506                ref_Callouts.HexRef_starts = pablo.Advance(HexRef3)
     507                ref_Callouts.GenRef_ends = pablo.ScanThru(ref_Callouts.GenRef_starts, lex.NameScan)
     508                ref_Callouts.DecRef_ends = pablo.ScanThru(ref_Callouts.DecRef_starts, lex.Digit)
     509                ref_Callouts.HexRef_ends = pablo.ScanThru(ref_Callouts.HexRef_starts, lex.Hex)
    510510                # Error checks
    511511                # At least one digit required for DecRef, one hex digit for HexRef.
     
    529529        ncname_stream = PI_names | GenRefs
    530530        name_stream = qname_stream | ncname_stream
    531         name_start = name_stream &~ bitutil.Advance(name_stream)
    532         name_cursor = name_stream & ~bitutil.Advance(name_stream)
     531        name_start = name_stream &~ pablo.Advance(name_stream)
     532        name_cursor = name_stream & ~pablo.Advance(name_stream)
    533533        void_prefix_err = name_cursor & lex.Colon
    534         namespace_sep = bitutil.ScanThru(name_cursor, lex.NameScan &~ lex.Colon) & lex.Colon
    535         local_part_start = bitutil.Advance(namespace_sep)
     534        namespace_sep = pablo.ScanThru(name_cursor, lex.NameScan &~ lex.Colon) & lex.Colon
     535        local_part_start = pablo.Advance(namespace_sep)
    536536        local_part_err = local_part_start &~ lex.NameScan
    537         colon2_err = bitutil.ScanThru(local_part_start, lex.NameScan &~ lex.Colon) & lex.Colon
     537        colon2_err = pablo.ScanThru(local_part_start, lex.NameScan &~ lex.Colon) & lex.Colon
    538538        ncname_err = ncname_stream & lex.Colon
    539539        namespace_error = void_prefix_err | local_part_err | colon2_err | ncname_err
     
    564564
    565565    # Group symbols of length 1
    566     starts_1 = bitutil.Advance(starts)
     566    starts_1 = pablo.Advance(starts)
    567567    tag_Callouts.ElemName_ends_1 = starts_1 & remaining_ends
    568568    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_1
    569569
    570570    # Group symbols of length 2
    571     starts_2 = bitutil.Advance(starts_1)
     571    starts_2 = pablo.Advance(starts_1)
    572572    tag_Callouts.ElemName_ends_2 = starts_2 & remaining_ends
    573573    remaining_ends = remaining_ends & ~tag_Callouts.ElemName_ends_2
     
    575575    # Group symbols of length 3_to_4
    576576    temp = starts_1 | starts_2
    577     temp32 = bitutil.Advance32(temp)
    578     #starts_3_to_4 = bitutil.Advance(bitutil.Advance(temp))
     577    temp32 = pablo.Advance32(temp)
     578    #starts_3_to_4 = pablo.Advance(pablo.Advance(temp))
    579579    starts_3_to_4 = interpose32 (temp, temp32, 2)
    580580    tag_Callouts.ElemName_ends_3_to_4 = starts_3_to_4 & remaining_ends
     
    583583    # Group symbols of length 5_to_8
    584584    temp = temp | starts_3_to_4
    585     temp32 = bitutil.Advance32(temp)
    586     #starts_5_to_8 = bitutil.Advance(bitutil.Advance(bitutil.Advance(bitutil.Advance(temp))))
     585    temp32 = pablo.Advance32(temp)
     586    #starts_5_to_8 = pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(temp))))
    587587    starts_5_to_8 = interpose32 (temp, temp32, 4)
    588588    tag_Callouts.ElemName_ends_5_to_8 = starts_5_to_8 & remaining_ends
     
    591591    # Group symbols of length 9_to_16
    592592    temp = temp | starts_5_to_8
    593     temp32 = bitutil.Advance32 (temp)
    594     #starts_9_to_16 = bitutil.Advance(bitutil.Advance(bitutil.Advance(bitutil.Advance(bitutil.Advance(bitutil.Advance(bitutil.Advance(bitutil.Advance(temp))))))))
     593    temp32 = pablo.Advance32 (temp)
     594    #starts_9_to_16 = pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(pablo.Advance(temp))))))))
    595595    starts_9_to_16 = interpose32 (temp, temp32, 8)
    596596    tag_Callouts.ElemName_ends_9_to_16 = starts_9_to_16 & remaining_ends
  • proto/SymbolTable/symtab_identity_template.cpp

    r1277 r1442  
    22#define TEMPLATED_SIMD_LIB
    33
    4 #define DEBUG 0
     4#define DEBUG 1
    55#define BLOCK_SIZE (sizeof(SIMD_type) * 8)
    66#define SEGMENT_BLOCKS 12
     
    110110{
    111111    BitBlock s = block[0];
    112     BitBlock temp = simd_and(s, simd<128>::sll(simd<1>::constant<1>(), sisd_from_int(pos)));
     112    BitBlock temp = simd_and(s, simd<128>::sll(simd<2>::constant<3>(), sisd_from_int(pos)));
    113113
    114114    if (bitblock_has_bit(temp))
  • proto/SymbolTable/symtab_pbgs_div_template.cpp

    r1387 r1442  
    111111{
    112112    BitBlock s = block[0];
    113     BitBlock temp = simd_and(s, simd<128>::sll(simd<1>::constant<1>(), sisd_from_int(pos)));
     113    BitBlock temp = simd_and(s, simd<128>::sll(simd<2>::constant<3>(), sisd_from_int(pos)));
    114114
    115115    if (bitblock_has_bit(temp))
  • proto/SymbolTable/symtab_pbgs_log_template.cpp

    r1278 r1442  
    126126{
    127127    BitBlock s = block[0];
    128     BitBlock temp = simd_and( s, simd_not(simd<128>::sll(simd<1>::constant<1>(), sisd_from_int(pos))) );
     128    BitBlock temp = simd_and( s, simd_not(simd<128>::sll(simd<2>::constant<3>(), sisd_from_int(pos))) );
    129129
    130130    if (bitblock_has_bit(temp))
  • proto/SymbolTable/symtab_pbgs_template.cpp

    r1232 r1442  
    103103{
    104104    BitBlock s = block[0];
    105     BitBlock temp = simd_and( s, simd_not(simd<128>::sll(simd<1>::constant<1>(), sisd_from_int(pos))) );
     105    BitBlock temp = simd_and( s, simd_not(simd<128>::sll(simd<2>::constant<3>(), sisd_from_int(pos))) );
    106106
    107107    if (bitblock_has_bit(temp))
  • proto/SymbolTable/symtab_stl_template.cpp

    r1232 r1442  
    363363             cout << gids[i] << " ";
    364364    }
     365    cout << endl;
    365366}
    366367
     
    486487    }
    487488
    488 //      print_GIDS();
     489#if DEBUG
     490    print_GIDS();
     491#endif
    489492}
    490493
Note: See TracChangeset for help on using the changeset viewer.