Changeset 2291 for trunk/symbol_table


Ignore:
Timestamp:
Aug 9, 2012, 8:36:06 PM (7 years ago)
Author:
ksherdy
Message:

Modified symbol table for integration with current ICXML structure.

Location:
trunk/symbol_table
Files:
12 edited

Legend:

Unmodified
Added
Removed
  • trunk/symbol_table/pablo/bit_byte_group_strms.py

    r2185 r2291  
    88        follows_0 = 0 # byte
    99
    10 def Gen_lgth_groups(markers, groups):
     10def Gen_lgth_groups(marker, groups):
    1111
    12         groups.starts = markers.starts
    13         groups.follows = markers.follows
     12        groups.starts = marker.starts
     13        groups.follows = marker.follows
    1414
    1515        # Groups symbols of length 1 to length 7
     
    3636
    3737
    38 def Main(markers, groups):
    39         Gen_lgth_groups(markers, groups)
     38def Main(marker, groups):
     39        Gen_lgth_groups(marker, groups)
    4040
  • trunk/symbol_table/pablo/demo_strms.py

    r2185 r2291  
    2121    lgth = len(u8data)
    2222
    23     markers = marker_strms.Markers()
    24     marker_strms.Classify_markers(basis_bits, markers)
    25     marker_strms.Generate_markers(markers)
     23    marker = marker_strms.Markers()
     24    marker_strms.Classify_marker(basis_bits, marker)
     25    marker_strms.Generate_marker(marker)
    2626
    2727    groups = div2_group_strms.Groups()
    28     div2_group_strms.Gen_lgth_groups(markers, groups)
     28    div2_group_strms.Gen_lgth_groups(marker, groups)
    2929
    3030    print_aligned_streams([('Input Data', u8data),
    31         ('markers.spans', bitstream2string(markers.spans, lgth)),
    32         ('markers.starts', bitstream2string(markers.starts, lgth)),
    33         ('markers.follows', bitstream2string(markers.follows, lgth)),
     31        ('marker.spans', bitstream2string(marker.spans, lgth)),
     32        ('marker.starts', bitstream2string(marker.starts, lgth)),
     33        ('marker.follows', bitstream2string(marker.follows, lgth)),
    3434        ('groups.starts', bitstream2string(groups.starts, lgth)),
    3535        ('groups.follows', bitstream2string(groups.follows, lgth)),
     
    4848    lgth = len(u8data)
    4949
    50     markers = marker_strms.Markers()
    51     marker_strms.Classify_markers(basis_bits, markers)
    52     marker_strms.Generate_markers(markers)
     50    marker = marker_strms.Markers()
     51    marker_strms.Classify_marker(basis_bits, marker)
     52    marker_strms.Generate_marker(marker)
    5353
    5454    groups = logbase2_group_strms.Groups()
    55     logbase2_group_strms.Gen_lgth_groups(markers, groups)
     55    logbase2_group_strms.Gen_lgth_groups(marker, groups)
    5656
    5757    print_aligned_streams([('Input Data', u8data),
    58         ('markers.spans', bitstream2string(markers.spans, lgth)),
    59         ('markers.starts', bitstream2string(markers.starts, lgth)),
    60         ('markers.follows', bitstream2string(markers.follows, lgth)),
     58        ('marker.spans', bitstream2string(marker.spans, lgth)),
     59        ('marker.starts', bitstream2string(marker.starts, lgth)),
     60        ('marker.follows', bitstream2string(marker.follows, lgth)),
    6161        ('groups.starts', bitstream2string(groups.starts, lgth)),
    6262        ('groups.follows', bitstream2string(groups.follows, lgth)),
     
    7272    lgth = len(u8data)
    7373
    74     markers = marker_strms.Markers()
    75     marker_strms.Classify_markers(basis_bits, markers)
    76     marker_strms.Generate_markers(markers)
     74    marker = marker_strms.Markers()
     75    marker_strms.Classify_marker(basis_bits, marker)
     76    marker_strms.Generate_marker(marker)
    7777
    7878    groups = div2_logbase2_group_strms.Groups()
    79     div2_logbase2_group_strms.Gen_lgth_groups(markers, groups)
     79    div2_logbase2_group_strms.Gen_lgth_groups(marker, groups)
    8080
    8181    print_aligned_streams([('Input Data', u8data),
    82         ('markers.spans', bitstream2string(markers.spans, lgth)),
    83         ('markers.starts', bitstream2string(markers.starts, lgth)),
    84         ('markers.follows', bitstream2string(markers.follows, lgth)),
     82        ('marker.spans', bitstream2string(marker.spans, lgth)),
     83        ('marker.starts', bitstream2string(marker.starts, lgth)),
     84        ('marker.follows', bitstream2string(marker.follows, lgth)),
    8585        ('groups.starts', bitstream2string(groups.starts, lgth)),
    8686        ('groups.follows', bitstream2string(groups.follows, lgth)),
     
    9696    lgth = len(u8data)
    9797
    98     markers = marker_strms.Markers()
    99     marker_strms.Classify_markers(basis_bits, markers)
    100     marker_strms.Generate_markers(markers)
     98    marker = marker_strms.Markers()
     99    marker_strms.Classify_marker(basis_bits, marker)
     100    marker_strms.Generate_marker(marker)
    101101
    102102    groups = bit_byte_group_strms.Groups()
    103     bit_byte_group_strms.Gen_lgth_groups(markers, groups)
     103    bit_byte_group_strms.Gen_lgth_groups(marker, groups)
    104104
    105105    print_aligned_streams([('Input Data', u8data),
    106         ('markers.spans', bitstream2string(markers.spans, lgth)),
    107         ('markers.starts', bitstream2string(markers.starts, lgth)),
    108         ('markers.follows', bitstream2string(markers.follows, lgth)),
     106        ('marker.spans', bitstream2string(marker.spans, lgth)),
     107        ('marker.starts', bitstream2string(marker.starts, lgth)),
     108        ('marker.follows', bitstream2string(marker.follows, lgth)),
    109109        ('groups.starts', bitstream2string(groups.starts, lgth)),
    110110        ('groups.follows', bitstream2string(groups.follows, lgth)),
  • trunk/symbol_table/pablo/div2_group_strms.py

    r2185 r2291  
    1515        follows_0 = 0
    1616
    17 def Gen_lgth_groups(markers, groups):
     17def Gen_lgth_groups(marker, groups):
    1818
    19         groups.starts = markers.starts
    20         groups.follows = markers.follows
     19        groups.starts = marker.starts
     20        groups.follows = marker.follows
    2121
    2222        ### Div2 Strategy ###
     
    6161        groups.follows_0 = follows_gt_16
    6262
    63 def Main(markers, groups):
    64         Gen_lgth_groups(markers, groups)
     63def Main(marker, groups):
     64        Gen_lgth_groups(marker, groups)
  • trunk/symbol_table/pablo/div2_logbase2_group_strms.py

    r2216 r2291  
    1111        follows_0 = 0
    1212
    13 def Gen_lgth_groups(markers, groups):
     13def Gen_lgth_groups(marker, groups):
    1414
    15         groups.starts = markers.starts
    16         groups.follows = markers.follows
     15        groups.starts = marker.starts
     16        groups.follows = marker.follows
    1717
    1818        ### Div2 Strategy ###
     
    6060        groups.follows_0 = follows_gt_16
    6161
    62 def Main(markers, groups):
    63         Gen_lgth_groups(markers, groups)
     62def Main(marker, groups):
     63        Gen_lgth_groups(marker, groups)
  • trunk/symbol_table/pablo/id_group_strms.py

    r2185 r2291  
    2323        follows_0 = 0
    2424
    25 def Gen_lgth_groups(markers, groups):
     25def Gen_lgth_groups(marker, groups):
    2626
    27         groups.starts = markers.starts
    28         groups.follows = markers.follows
     27        groups.starts = marker.starts
     28        groups.follows = marker.follows
    2929
    3030        cursor = groups.starts
     
    114114        groups.follows_0 = temp
    115115
    116 def Main(markers, groups):
    117         Gen_lgth_groups(markers, groups)
     116def Main(marker, groups):
     117        Gen_lgth_groups(marker, groups)
    118118
  • trunk/symbol_table/pablo/logbase2_group_strms.py

    r2216 r2291  
    1111        follows_0 = 0
    1212
    13 def Gen_lgth_groups(markers, groups):
     13def Gen_lgth_groups(marker, groups):
    1414
    15                 groups.starts = markers.starts
    16                 groups.follows = markers.follows
     15                groups.starts = marker.starts
     16                groups.follows = marker.follows
    1717
    1818                starts = groups.starts
     
    5656                groups.follows_0 = remaining_follows
    5757
    58 def Main(markers, groups):
    59                 Gen_lgth_groups(markers, groups)
     58def Main(marker, groups):
     59                Gen_lgth_groups(marker, groups)
  • trunk/symbol_table/pablo/marker_strms.py

    r2285 r2291  
    11from bitutil import *
    22
    3 class Markers():
     3class Marker():
    44    spans = 0
    55    starts = 0
     
    77
    88# Calculate Symbol spans
    9 # markers.spans = [-_:.a-zA-Z0-9] character set compiler def'n, matches 'st_test_file_generator.py' def'n
    10 def Classify_markers(basis_bits, markers):
     9# marker.spans = [-_:.a-zA-Z0-9] character set compiler def'n, matches 'st_test_file_generator.py' def'n
     10def Classify_marker(basis_bits, marker):
    1111    temp1 = (basis_bits.bit_0 | basis_bits.bit_1)
    1212    temp2 = (basis_bits.bit_2 &~ basis_bits.bit_3)
     
    4949    temp39 = (basis_bits.bit_4 & temp38)
    5050    temp40 = (temp16 &~ temp39)
    51     markers.spans = (temp37 | temp40)
     51    marker.spans = (temp37 | temp40)
    5252
    5353# Determine Symbol starts, follows
    54 def Gen_markers(markers):
    55     markers.starts = markers.spans &~ Advance(markers.spans)
    56     markers.follows = ~markers.spans & Advance(markers.spans)
    57     #ScanThru(markers.starts, markers.spans)
    58     #markers.follows = ScanThru(markers.starts, markers.spans)
     54def Gen_marker(marker):
     55    marker.starts = marker.spans &~ Advance(marker.spans)
     56    marker.follows = ~marker.spans & Advance(marker.spans)
     57    #ScanThru(marker.starts, marker.spans)
     58    #marker.follows = ScanThru(marker.starts, marker.spans)
    5959
    60 def Main(markers):
    61     Classify_markers(basis_bits, markers)
    62     Gen_markers(markers)
     60def Main(marker):
     61    Classify_marker(basis_bits, marker)
     62    Gen_marker(marker)
    6363
  • trunk/symbol_table/pablo/xmlwf_marker_strms.py

    r2285 r2291  
    11from bitutil import *
    22
    3 class Markers():
     3class Marker():
    44    starts = 0
    55    follows = 0
    66
    7 def Gen_markers(markers, tag_Callouts):
    8         markers.starts = tag_Callouts.ElemName_starts | tag_Callouts.AttName_starts
    9         markers.follows = tag_Callouts.ElemName_ends | tag_Callouts.AttName_ends
     7def Gen_marker(marker, tag_Callouts):
     8        marker.starts = tag_Callouts.ElemName_starts | tag_Callouts.AttName_starts
     9        marker.follows = tag_Callouts.ElemName_ends | tag_Callouts.AttName_ends
    1010
    11 def Main(markers):
    12         Gen_markers(markers, tag_Callouts)
     11def Main(marker):
     12        Gen_marker(marker, tag_Callouts)
    1313
  • trunk/symbol_table/src/symbol_table.hpp

    r2288 r2291  
    9393        }
    9494
    95         void resolve(uint8_t buffer [], Groups groups [],  BitBlock starts [], BitBlock follows_0 [],
    96                                  BitBlock h0 [], BitBlock h1 [], uint32_t segment_blocks, GIDS & gids) {
     95        void resolve(uint8_t buffer [], Groups groups [],  BitBlock starts [], BitBlock h0 [], BitBlock h1 [], uint32_t segment_blocks, GIDS & gids) {
    9796
    9897                        for(uint32_t blk = 0; blk < segment_blocks; blk++) {
     
    102101        }
    103102
    104         IDISA_ALWAYS_INLINE void resolve(uint32_t blk_offset, uint8_t buffer [], Groups & groups,  BitBlock starts[],
     103        IDISA_ALWAYS_INLINE void resolve(uint32_t blk_offset, uint8_t buffer [], Groups & groups,  BitBlock * starts,
    105104                                 BitBlock * h0, BitBlock * h1, GIDS & gids) {
    106105
  • trunk/symbol_table/templates/group_strms.template.hpp

    r2284 r2291  
    1010@group_strms_stream_stmts
    1111
    12 static IDISA_ALWAYS_INLINE void identity_group_do_block(Markers & markers, Groups & groups) {
     12static IDISA_ALWAYS_INLINE void group_strms_do_block(Marker & marker, Groups & groups) {
    1313        // groups.starts = markers.starts;
    1414        // groups.follows = markers.follows;
     
    1717}
    1818
    19 static IDISA_ALWAYS_INLINE void identity_group_do_final_block(Markers & markers, Groups & groups, BitBlock & EOF_mask) {
     19static IDISA_ALWAYS_INLINE void group_strms_do_final_block(Marker & marker, Groups & groups, BitBlock & EOF_mask) {
    2020        // groups.starts = markers.starts;
    2121        // groups.follows = markers.follows;
  • trunk/symbol_table/templates/main.template.cpp

    r2284 r2291  
    115115    ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_starts, starts);
    116116
    117     // follows_0 - Arbitrary length symbols
    118     BitBlock * COPYBACK_follows_0;
    119     BitBlock * follows_0;
    120     ALLOC_STATIC_ALIGNED_BITBLOCK_BUFFER_WITH_COPYBACK(COPYBACK_follows_0, follows_0);
    121 
    122117    ///////////////////////////////////////////////////////////////////////////
    123118    // Parallel Data Streams - No CopyBack
     
    125120
    126121    Basis_bits basis_bits[SEGMENT_BLOCKS];
    127     Markers markers[SEGMENT_BLOCKS];
     122    Marker marker[SEGMENT_BLOCKS];
    128123    Hash hash[SEGMENT_BLOCKS];
    129124    Groups groups[SEGMENT_BLOCKS];
     
    149144                for(blk=0;blk<SEGMENT_BLOCKS;blk++) {
    150145                        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);    // transpose
    151                         markers_do_block(basis_bits[blk], markers[blk]);                            // gen symbol spans, mark starts & follows
     146                        marker_do_block(basis_bits[blk], marker[blk]);                              // gen symbol spans, mark starts & follows
    152147                        hash_strms_do_block(basis_bits[blk], hash[blk]);                            // gen hash bit streams
    153                         identity_group_do_block(markers[blk], groups[blk]);                         // sort marker bit stream (identity)
     148                        group_strms_do_block(marker[blk], groups[blk]);                     // sort marker bit stream (identity)
    154149    }
    155150
     
    158153                        h1[blk] = hash[blk].h1;
    159154                        starts[blk] = groups[blk].starts;
    160                         follows_0[blk] = groups[blk].follows_0;
    161155    }
    162156
    163157    PERF_SEC_START(parser_timer);
    164     st.resolve(src_buffer, groups, starts, follows_0, h0, h1, SEGMENT_BLOCKS, gids);
     158    st.resolve(src_buffer, groups, starts, h0, h1, SEGMENT_BLOCKS, gids);
    165159    PERF_SEC_END(parser_timer, SEGMENT_SIZE);
    166160
     
    169163    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_h1,h1);
    170164    COPY_BACK_BITBLOCK_BUFFER(COPYBACK_starts,starts);
    171     COPY_BACK_BITBLOCK_BUFFER(COPYBACK_follows_0,follows_0);
    172165
    173166    is.read ((char *)(src_buffer), SEGMENT_SIZE);
     
    181174
    182175                        #ifdef INDEX_AT_STARTS
    183                                         ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].starts));
     176                                        ForwardScanner<BitBlock, scanword_t> scanner(&(marker[blk].starts));
    184177                        #else
    185                                         ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].follows));
     178                                        ForwardScanner<BitBlock, scanword_t> scanner(&(marker[blk].follows));
    186179                        #endif
    187180
     
    205198    while (remaining >= BLOCK_SIZE) {
    206199                        s2p_do_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk]);
    207                         markers_do_block(basis_bits[blk], markers[blk]);
     200                        marker_do_block(basis_bits[blk], marker[blk]);
    208201                        hash_strms_do_block(basis_bits[blk], hash[blk]);
    209                         identity_group_do_block(markers[blk], groups[blk]);
     202                        group_strms_do_block(marker[blk], groups[blk]);
    210203                        blk++;
    211204                        remaining -= BLOCK_SIZE;
     
    218211                        BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
    219212                        s2p_do_final_block((BytePack *) &src_buffer[blk*BLOCK_SIZE], basis_bits[blk], EOF_mask);
    220                         markers_do_final_block(basis_bits[blk], markers[blk], EOF_mask);
     213                        marker_do_final_block(basis_bits[blk], marker[blk], EOF_mask);
    221214                        hash_strms_do_final_block(basis_bits[blk], hash[blk], EOF_mask);
    222                         identity_group_do_final_block(markers[blk], groups[blk], EOF_mask);
     215                        group_strms_do_final_block(marker[blk], groups[blk], EOF_mask);
    223216                        blk++;
    224217    }
     
    229222                        h1[blk] = hash[blk].h1;
    230223                        starts[blk] = groups[blk].starts;
    231                         follows_0[blk] = groups[blk].follows_0;
    232224    }
    233225
    234226    // PERF_SEC_BIND(0);
    235227    PERF_SEC_START(parser_timer);
    236     st.resolve(src_buffer, groups, starts, follows_0, h0, h1, segment_blocks, gids);
     228    st.resolve(src_buffer, groups, starts, h0, h1, segment_blocks, gids);
    237229    PERF_SEC_END(parser_timer, chars_avail);
    238230
     
    244236
    245237                #ifdef INDEX_AT_STARTS
    246                           ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].starts));
     238                          ForwardScanner<BitBlock, scanword_t> scanner(&(marker[blk].starts));
    247239                #else
    248                           ForwardScanner<BitBlock, scanword_t> scanner(&(groups[blk].follows));
     240                          ForwardScanner<BitBlock, scanword_t> scanner(&(marker[blk].follows));
    249241                #endif
    250242
  • trunk/symbol_table/templates/marker_strms.template.hpp

    r2284 r2291  
    99@marker_strms_stream_stmts
    1010
    11 static IDISA_ALWAYS_INLINE void markers_do_block(Basis_bits & basis_bits, Markers & markers) {
     11static IDISA_ALWAYS_INLINE void marker_do_block(Basis_bits & basis_bits, Marker & marker) {
    1212        // GENERATED
    1313        @marker_strms_block_stmts
    1414}
    1515
    16 static IDISA_ALWAYS_INLINE void markers_do_final_block(Basis_bits & basis_bits, Markers & markers, BitBlock & EOF_mask) { //  BitBlock & starts, BitBlock & follows, BitBlock & EOF_mask) {
     16static IDISA_ALWAYS_INLINE void marker_do_final_block(Basis_bits & basis_bits, Marker & marker, BitBlock & EOF_mask) { //  BitBlock & starts, BitBlock & follows, BitBlock & EOF_mask) {
    1717        // GENERATED
    1818        @marker_strms_final_block_stmts
Note: See TracChangeset for help on using the changeset viewer.