Changeset 2153
- Timestamp:
- May 24, 2012, 5:54:15 PM (7 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
proto/parabix2/pablo_template.cpp
r2148 r2153 43 43 // Buffer Management 44 44 ////////////////////////////////////////////////////////////////////////////////////////// 45 #define PADDING_BLOCKS 1 46 #define PADDING_SIZE (BLOCK_SIZE * PADDING_BLOCKS ) 47 #define SEGMENT_BLOCKS 12 48 #define SEGMENT_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS) 49 #define BUFFER_SIZE PADDING_SIZE + SEGMENT_SIZE 45 //#define PADDING_BLOCKS 1 46 //#define PADDING_SIZE (BLOCK_SIZE * PADDING_BLOCKS ) 47 //#define SEGMENT_BLOCKS 12 48 //#define SEGMENT_SIZE (BLOCK_SIZE * SEGMENT_BLOCKS) 49 //#define BUFFER_SIZE PADDING_SIZE + SEGMENT_SIZE 50 #include "../lib/buffer.hpp" 50 51 51 52 ////////////////////////////////////////////////////////////////////////////////////////// … … 109 110 void do_process(FILE *infile, FILE *outfile) { 110 111 111 @decl 112 113 LineColTracker tracker; 114 TagMatcher<SEGMENT_SIZE,PADDING_SIZE> matcher; 115 116 char * source; 117 int block_base=0; 118 int buffer_base=0; 119 int buffer_pos = 0; 120 int block_pos = 0; 121 int chars_avail = 0; 122 int check_pos = 0; 123 int chars_read = 0; 124 125 ////////////////////////////////////////////////////////////////////////////////////////// 126 // Buffer Management 127 ////////////////////////////////////////////////////////////////////////////////////////// 128 BitBlock buf[(BUFFER_SIZE)/sizeof(BitBlock)]; 129 130 char * srcbuf = ((char *) buf); 131 buffer_base = buffer_pos; 132 source = srcbuf; 133 134 chars_read = fread((void *)srcbuf, 1, SEGMENT_SIZE, infile); 135 chars_avail = chars_read; 136 if (chars_avail > SEGMENT_SIZE) chars_avail = SEGMENT_SIZE; 137 138 if(chars_read<4){ 112 @decl 113 114 LineColTracker tracker; 115 TagMatcher<SEGMENT_SIZE,PADDING_SIZE> matcher; 116 117 uint8_t * src_buf; 118 int block_base=0; 119 int buffer_base=0; 120 int buffer_pos = 0; 121 int block_pos = 0; 122 int chars_avail = 0; 123 int check_pos = 0; 124 int chars_read = 0; 125 126 ////////////////////////////////////////////////////////////////////////////////////////// 127 // Buffer Management 128 ////////////////////////////////////////////////////////////////////////////////////////// 129 //BitBlock buf[(BUFFER_SIZE)/sizeof(BitBlock)]; 130 void * temp; 131 ALLOC_STATIC_ALIGNED_BYTE_BUFFER(temp); 132 src_buf = (uint8_t *)temp; 133 134 buffer_base = buffer_pos; 135 chars_read = fread((void *)src_buf, 1, SEGMENT_SIZE, infile); 136 chars_avail = chars_read; 137 if (chars_avail > SEGMENT_SIZE) chars_avail = SEGMENT_SIZE; 138 139 ////////////////////////////////////////////////////////////////////////////////////////// 140 // XML Validation / Content Model 141 ////////////////////////////////////////////////////////////////////////////////////////// 142 if(chars_read<4){ 139 143 fprintf(stderr,"File is too short. Not well formed.\n"); 140 144 exit(-1); 141 }142 143 Entity_Info * e = new Entity_Info;144 e->AnalyzeSignature((unsigned char *)srcbuf);145 146 if (e->code_unit_base == ASCII) {147 148 XML_Decl_Parser<ASCII> decl_parser((unsigned char *)src buf);145 } 146 147 Entity_Info * e = new Entity_Info; 148 e->AnalyzeSignature((unsigned char *)src_buf); 149 150 if (e->code_unit_base == ASCII) { 151 152 XML_Decl_Parser<ASCII> decl_parser((unsigned char *)src_buf); 149 153 150 154 decl_parser.ReadXMLInfo(*e); … … 154 158 exit(-1); 155 159 } 156 }157 else {160 } 161 else { 158 162 fprintf(stderr,"Sorry, this xmlwf demo does not process EBCDIC.\n"); 159 163 exit(-1); 160 }161 162 if (e->content_start != 0) {163 memmove(&srcbuf[0], &srcbuf[e->content_start], chars_read - e->content_start);164 } 165 166 if (e->content_start != 0) { 167 memmove(&src_buf[0], &src_buf[e->content_start], chars_read - e->content_start); 164 168 buffer_pos = e->content_start; 165 169 if (chars_avail == SEGMENT_SIZE) { 166 chars_read = chars_read - e->content_start + fread(&src buf[chars_read-e->content_start], 1, e->content_start, infile);167 170 chars_read = chars_read - e->content_start + fread(&src_buf[chars_read-e->content_start], 1, e->content_start, infile); 171 chars_avail = chars_read; 168 172 if (chars_avail > SEGMENT_SIZE) chars_avail = SEGMENT_SIZE; 169 } 170 else { 171 chars_read -=e->content_start; 172 chars_avail -=e->content_start; 173 } 174 } 175 176 matcher.setSrc(srcbuf); 177 178 @stream_stmts 179 180 /* Full Segments */ 173 } 174 else { 175 chars_read -=e->content_start; 176 chars_avail -=e->content_start; 177 } 178 } 179 180 @stream_stmts 181 182 183 ////////////////////////////////////////////////////////////////////////////////////////// 184 // Full Segments 185 ////////////////////////////////////////////////////////////////////////////////////////// 186 matcher.setSrc((char *)src_buf); 181 187 while (chars_avail == SEGMENT_SIZE) { 182 188 PERF_SEC_START(parser_timer); 183 189 for (int blk = 0; blk < SEGMENT_BLOCKS; blk++) { 184 190 block_base = blk*BLOCK_SIZE; 185 s2p_do_block((BytePack *) &srcbuf[block_base], basis_bits);191 s2p_do_block((BytePack *) &src_buf[block_base], basis_bits); 186 192 @block_stmts 187 193 tracker.StoreNewlines(lex.LF); 188 postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, source, buffer_base, block_base, chars_avail, tracker);194 postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buf, buffer_base, block_base, chars_avail, tracker); 189 195 matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail); 190 196 tracker.AdvanceBlock(); … … 195 201 196 202 int bytes_left = chars_read - chars_avail; 197 memmove(src buf, &srcbuf[SEGMENT_SIZE], bytes_left);198 chars_read = fread(&src buf[bytes_left], 1, SEGMENT_SIZE - bytes_left, infile) + bytes_left;203 memmove(src_buf, &src_buf[SEGMENT_SIZE], bytes_left); 204 chars_read = fread(&src_buf[bytes_left], 1, SEGMENT_SIZE - bytes_left, infile) + bytes_left; 199 205 chars_avail = chars_read; 200 206 if (chars_avail > SEGMENT_SIZE) chars_avail = SEGMENT_SIZE; … … 203 209 } 204 210 205 /* Final Partial Segment */ 211 ////////////////////////////////////////////////////////////////////////////////////////// 212 // Final Partial Segment 213 ////////////////////////////////////////////////////////////////////////////////////////// 206 214 PERF_SEC_START(parser_timer); 207 215 … … 211 219 /* Full Blocks */ 212 220 while (remaining >= BLOCK_SIZE) { 213 214 s2p_do_block((BytePack *) &srcbuf[block_pos], basis_bits);221 block_base = block_pos; 222 s2p_do_block((BytePack *) &src_buf[block_pos], basis_bits); 215 223 @block_stmts 216 224 tracker.StoreNewlines(lex.LF); 217 postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, source, buffer_base, block_base, chars_avail, tracker);225 postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buf, buffer_base, block_base, chars_avail, tracker); 218 226 matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail); 219 227 tracker.AdvanceBlock(); … … 226 234 if (remaining > 0 || @any_carry) { 227 235 EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining)); 228 s2p_do_final_block((BytePack *) &src buf[block_pos], basis_bits, EOF_mask);236 s2p_do_final_block((BytePack *) &src_buf[block_pos], basis_bits, EOF_mask); 229 237 @final_block_stmts 230 238 tracker.StoreNewlines(lex.LF); 231 postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, source, buffer_base, block_base, chars_avail, tracker);239 postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buf, buffer_base, block_base, chars_avail, tracker); 232 240 matcher.store_streams(check_streams.tag_marks, check_streams.name_follows, check_streams.misc_mask, chars_avail); 233 241 tracker.AdvanceBlock(); … … 240 248 matcher.StreamScan(chars_avail); 241 249 matcher.Advance_buffer(); 242 243 250 244 251 PERF_SEC_END(parser_timer, chars_avail);
Note: See TracChangeset
for help on using the changeset viewer.