Changeset 2180 for proto


Ignore:
Timestamp:
May 28, 2012, 6:55:03 PM (7 years ago)
Author:
ksherdy
Message:

Updated Makefile and template.

Location:
proto/parabix2
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • proto/parabix2/Makefile

    r1607 r2180  
    22PABLO_SRCFILE=parabix2_pablo.py
    33PABLO_TEMPLATE=pablo_template.cpp
     4PABLO_TEMPLATE_SYMBOL_TABLE=pablo_template.cpp
    45PABLO_COMPILER=../Compiler/pablomain.py
    56
     
    1011avx:    $(PABLO_SRCFILE)
    1112        python $(PABLO_COMPILER) $(PABLO_SRCFILE) -t pablo_template_avx.cpp -o $(OUTFILE)
     13
     14symbol_table: $(PABLO_SRCFILE) $(PABLO_TEMPLATE_SYMBOL_TABLE)
     15        python $(PABLO_COMPILER) $(PABLO_SRCFILE) -t $(PABLO_TEMPLATE_SYMBOL_TABLE) -o $(OUTFILE)
    1216
    1317#threads:       $(PABLO_SRCFILE)
  • proto/parabix2/pablo_template.cpp

    r2171 r2180  
    4343// Buffer Management
    4444//////////////////////////////////////////////////////////////////////////////////////////
    45 #define PADDING_BLOCKS 1
     45#define PADDING_BLOCKS 0
    4646#define PADDING_SIZE (BLOCK_SIZE * PADDING_BLOCKS)
    4747#define COPYBACK_BLOCKS 3
     
    5252#define SEGMENT_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS)
    5353#define BUFFER_SIZE (COPYBACK_SIZE + SEGMENT_SIZE + LOOKAHEAD_SIZE + PADDING_SIZE)
    54 
    55 #define OVERLAP_BUFSIZE PADDING_SIZE
    5654
    5755//////////////////////////////////////////////////////////////////////////////////////////
     
    118116
    119117    LineColTracker tracker;
    120     TagMatcher<SEGMENT_SIZE,OVERLAP_BUFSIZE> matcher;
    121 
    122     uint8_t * src_buf;
     118        TagMatcher<SEGMENT_SIZE,LOOKAHEAD_SIZE> matcher;
     119
    123120    int block_base=0;
    124121        int block_pos = 0;
     
    127124    int chars_avail = 0;
    128125        int chars_read = 0;
    129         int check_pos = 0;
    130126
    131127    //////////////////////////////////////////////////////////////////////////////////////////
    132128    // Buffer Management
    133129    //////////////////////////////////////////////////////////////////////////////////////////
    134         BitBlock buf[(BUFFER_SIZE)/sizeof(BitBlock)];
    135         src_buf = (uint8_t *)buf + COPYBACK_SIZE;
    136 
     130        //      BitBlock buf[(BUFFER_SIZE)/sizeof(BitBlock)];
     131        //      uint8_t * src_buffer = (uint8_t *)buf + COPYBACK_SIZE;
     132
     133        #include "../lib/buffer.hpp"
     134        uint8_t * COPYBACK;
     135        uint8_t * src_buffer;
     136        ALLOC_STATIC_ALIGNED_BYTE_BUFFER_WITH_COPYBACK(COPYBACK, src_buffer);
     137       
    137138    //////////////////////////////////////////////////////////////////////////////////////////
    138139    // XML Validation / Content Model
    139140    //////////////////////////////////////////////////////////////////////////////////////////
    140         chars_read = fread((void *)src_buf, 1, SEGMENT_SIZE, infile);
     141        chars_read = fread((void *)src_buffer, 1, SEGMENT_SIZE, infile);
    141142        chars_avail = chars_read;
    142143        if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
     
    148149
    149150    Entity_Info * e = new Entity_Info;
    150     e->AnalyzeSignature((unsigned char *)src_buf);
     151        e->AnalyzeSignature((unsigned char *)src_buffer);
    151152
    152153    if (e->code_unit_base == ASCII) {
    153154
    154                 XML_Decl_Parser<ASCII> decl_parser((unsigned char *)src_buf);
     155                XML_Decl_Parser<ASCII> decl_parser((unsigned char *)src_buffer);
    155156
    156157                decl_parser.ReadXMLInfo(*e);
     
    168169    if (e->content_start != 0) {
    169170
    170                 memmove(&src_buf[0], &src_buf[e->content_start], chars_avail - e->content_start);
     171                memmove(&src_buffer[0], &src_buffer[e->content_start], chars_avail - e->content_start);
    171172                buffer_pos = e->content_start;
    172173                if ((chars_avail-e->content_start) < SEGMENT_SIZE) {
    173                         // TODO - Overlap buffer size.
    174                         chars_read = chars_avail - e->content_start + fread(&src_buf[chars_avail-e->content_start], 1, e->content_start, infile);
     174                        chars_read = chars_avail - e->content_start + fread(&src_buffer[chars_avail-e->content_start], 1, e->content_start, infile);
    175175                        chars_avail = chars_read;
    176176                }
     
    178178    }
    179179
     180        //////////////////////////////////////////////////////////////////////////////////////////
     181        // Read OVERLAP bytes to support post processing validation lookahead.
     182        //////////////////////////////////////////////////////////////////////////////////////////
     183        chars_read = chars_avail + fread(&src_buffer[chars_avail], 1, LOOKAHEAD_SIZE, infile);
     184        chars_avail = chars_read;
     185        if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
     186
    180187    @stream_stmts
    181188
     
    183190    // Full Segments
    184191    //////////////////////////////////////////////////////////////////////////////////////////
    185     matcher.setSrc((char *)src_buf);
     192        matcher.setSrc((char *)src_buffer);
    186193        while (chars_avail >= SEGMENT_SIZE) {
    187194      PERF_SEC_START(parser_timer);
    188195      for (int blk = 0; blk < SEGMENT_BLOCKS; blk++) {
    189196          block_base = blk*BLOCK_SIZE;
    190           s2p_do_block((BytePack *) &src_buf[block_base], basis_bits);
     197                  s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits);
    191198
    192199          @block_stmts
    193200
    194201          tracker.StoreNewlines(lex.LF);
    195                   postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buf, buffer_base, block_base, chars_avail+2, tracker);
     202                  postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail+2, tracker);
    196203          matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
    197204          tracker.AdvanceBlock();
     
    201208      PERF_SEC_END(parser_timer, chars_avail);
    202209
    203           int bytes_left = chars_read - chars_avail;
    204 
    205           memmove(src_buf, &src_buf[SEGMENT_SIZE], bytes_left);
    206 
    207           // TODO - Look at this calculation. Define a segment size that incorporates OVERLAP_BUFSIZE.
    208 
    209           chars_read = fread(&src_buf[bytes_left], 1, SEGMENT_SIZE + OVERLAP_BUFSIZE - bytes_left, infile) + bytes_left;
    210       chars_avail = chars_read;
     210          memmove(src_buffer, &src_buffer[SEGMENT_SIZE], LOOKAHEAD_SIZE); // CopyBack Trailing Padding
     211
     212          // memmove(src_base, &src_buf[SEGMENT_SIZE-COPYBACK_SIZE], COPYBACK_SIZE); // Symbol Table CopyBack
     213          chars_read = fread(&src_buffer[LOOKAHEAD_SIZE], 1, SEGMENT_SIZE, infile) + LOOKAHEAD_SIZE;
     214          chars_avail = chars_read;
    211215          if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE;
    212       buffer_pos += chars_avail;
    213       buffer_base = buffer_pos;
     216          buffer_pos += chars_avail;
     217          buffer_base = buffer_pos;
    214218    }
    215219
     
    225229    while (remaining >= BLOCK_SIZE) {
    226230          block_base = block_pos;
    227           s2p_do_block((BytePack *) &src_buf[block_pos], basis_bits);
     231                  s2p_do_block((BytePack *) &src_buffer[block_pos], basis_bits);
    228232          @block_stmts
    229233          tracker.StoreNewlines(lex.LF);
    230           postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buf, buffer_base, block_base, chars_avail, tracker);
     234                  postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
    231235                  matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
    232236          tracker.AdvanceBlock();
     
    239243    if (remaining > 0 || @any_carry) {
    240244          EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining));
    241           s2p_do_final_block((BytePack *) &src_buf[block_pos], basis_bits, EOF_mask);
     245                  s2p_do_final_block((BytePack *) &src_buffer[block_pos], basis_bits, EOF_mask);
    242246          @final_block_stmts
    243247          tracker.StoreNewlines(lex.LF);
    244           postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buf, buffer_base, block_base, chars_avail, tracker);
     248                  postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker);
    245249          matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail);
    246250          tracker.AdvanceBlock();
     
    254258    matcher.Advance_buffer();
    255259
    256     PERF_SEC_END(parser_timer, chars_avail);
    257260    if (matcher.depth != 0) {
    258261      fprintf(stderr, "tag matching error (depth %i) at position %i\n", matcher.depth, buffer_base);
    259262      exit(-1);
    260263    }
     264        PERF_SEC_END(parser_timer, chars_avail);
    261265}
    262266
Note: See TracChangeset for help on using the changeset viewer.