Changeset 2639
- Timestamp:
- Nov 15, 2012, 11:44:25 AM (6 years ago)
- Location:
- proto/PDF
- Files:
-
- 2 edited
-
pablo_template_pdf.cpp (modified) (14 diffs)
-
pdf.py (modified) (2 diffs)
Legend:
- Unmodified
- Added
- Removed
-
proto/PDF/pablo_template_pdf.cpp
r2629 r2639 18 18 19 19 static unsigned int depth = 0; 20 static string outstring=""; 21 static int startpos=0; 22 static int val=0; 23 static int cur_state; 20 24 21 25 static inline void inc() { … … 26 30 depth--; 27 31 } 32 #define scan_to_next_and_test(scanner, state) \ 33 scanner.scan_to_next();\ 34 if (scanner.is_done()) {\ 35 cur_state=state;\ 36 return;\ 37 }\ 38 39 40 //if (scanner.is_done()) 28 41 29 42 static BitBlock lt_zero() { … … 184 197 return out; 185 198 } 186 void do_process(FILE *infile, FILE *outfile) { 187 @decl 188 189 /* Extras */ 190 parity.odd = simd<2>::constant<1>(); 191 parity.even = simd<2>::constant<2>(); 192 193 int block_base = 0; 194 int block_pos = 0; 195 int chars_avail = 0; 196 int chars_read = 0; 197 int padding_read = 0; 198 int remaining = 0; 199 200 /* Tracked for post processing */ 201 //int buffer_base=0; 202 //int buffer_pos = 0; 203 204 ////////////////////////////////////////////////////////////////////////////////////////// 205 // Buffer Management 206 ////////////////////////////////////////////////////////////////////////////////////////// 207 uint8_t * src_buffer; 208 ALLOC_STATIC_ALIGNED_BYTE_BUFFER(src_buffer, BUFFER_SIZE); 209 210 ////////////////////////////////////////////////////////////////////////////////////////// 211 // Initial file read 212 ////////////////////////////////////////////////////////////////////////////////////////// 213 chars_read = fread((void *)src_buffer, 1, SEGMENT_SIZE + LOOKAHEAD_SIZE, infile); 214 remaining = chars_read; 215 chars_avail = chars_read; 216 padding_read = chars_read - SEGMENT_SIZE; 217 218 if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE; 219 220 @stream_stmts 221 222 ////////////////////////////////////////////////////////////////////////////////////////// 223 // Processs Full Segments 224 ////////////////////////////////////////////////////////////////////////////////////////// 225 while (chars_avail >= SEGMENT_SIZE) { 226 PERF_SEC_START(parser_timer); 227 for (int blk = 0; blk < SEGMENT_BLOCKS; blk++) { 228 block_base = blk*BLOCK_SIZE; 229 s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits); 230 231 #ifdef PDF_DEBUG 232 for (int i = 0; i < BLOCK_SIZE; i++) { 233 cout << src_buffer[block_base + i]; 234 } 235 #endif 236 237 @block_stmts 238 239 //postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base,chars_avail, tracker); 240 241 } 242 PERF_SEC_END(parser_timer, chars_avail); 243 244 remaining -= SEGMENT_SIZE; 245 memmove(src_buffer, &src_buffer[SEGMENT_SIZE], padding_read); 246 chars_read = fread(&src_buffer[padding_read], 1, (SEGMENT_SIZE + LOOKAHEAD_SIZE - padding_read), infile); 247 248 padding_read = SEGMENT_SIZE + LOOKAHEAD_SIZE - padding_read ;//chars_read; 249 250 remaining += chars_read; 251 chars_avail = remaining; 252 253 if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE; 254 255 /* Tracked for post processing */ 256 // buffer_pos += chars_avail; 257 // buffer_base = buffer_pos; 258 259 } 260 261 ////////////////////////////////////////////////////////////////////////////////////////// 262 // Final Partial Segment 263 ////////////////////////////////////////////////////////////////////////////////////////// 264 PERF_SEC_START(parser_timer); 265 266 block_base = 0; 267 268 /* Full Blocks */ 269 uint32_t blk = 0; 270 while (remaining >= BLOCK_SIZE) { 271 272 #ifdef PDF_DEBUG 273 for(int i = 0; i < BLOCK_SIZE; i++) { 274 cout << src_buffer[block_base + i]; 275 } 276 #endif 277 278 s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits); 279 @block_stmts 280 281 // postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker); 282 283 block_base += BLOCK_SIZE; 284 remaining -= BLOCK_SIZE; 285 blk++; 286 } 287 288 /* Partial Block or Any Carry */ 289 if (remaining > 0 || @any_carry) { 290 291 #ifdef PDF_DEBUG 292 for(int i = 0; i < remaining; i++) 293 { 294 std::cout << src_buffer[block_base+i]; 295 } 296 #endif 297 298 EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining)); 299 s2p_do_final_block((BytePack *) &src_buffer[block_base], basis_bits, EOF_mask); 300 @final_block_stmts 301 //print_register2("markstarts",marker.starts,remaining); 302 303 304 string outstring; 305 int startpos; 199 200 void postprocess_do_block(Marker & marker, uint8_t * src_buffer, int block_base) { 201 306 202 int ch; 307 int val; 308 ForwardScanner<BitBlock, ScanWord> scanner(&(marker.starts)); 309 scanner.scan_to_next(); 203 ForwardScanner<BitBlock, ScanWord> scanner; 204 ForwardScanner<BitBlock, ScanWord> starts_scanner(&(marker.starts)); 205 ForwardScanner<BitBlock, ScanWord> others_scanner(&(marker.others)); 206 scanner = starts_scanner; 207 switch(cur_state){ 208 case 0: 209 scanner = starts_scanner; 210 goto pos0; 211 case 1: 212 scanner = others_scanner; 213 goto pos1; 214 case 2: 215 scanner = others_scanner; 216 goto pos2; 217 case 3: 218 scanner = others_scanner; 219 goto pos3; 220 case 4: 221 scanner = others_scanner; 222 goto pos4; 223 case 5: 224 scanner = others_scanner; 225 goto pos5; 226 case 6: 227 scanner = others_scanner; 228 goto pos6; 229 case 7: 230 scanner = others_scanner; 231 goto pos7; 232 case 8: 233 scanner = others_scanner; 234 goto pos8; 235 case 9: 236 scanner = others_scanner; 237 goto pos9; 238 case 10: 239 scanner = starts_scanner; 240 goto pos10; 241 } 242 243 244 pos0: scan_to_next_and_test(scanner, 0); 310 245 while(!scanner.is_done()) { 311 switch (src_buffer[ scanner.get_pos()])246 switch (src_buffer[block_base+scanner.get_pos()]) 312 247 { 313 248 case '/': 314 249 cout<<"LexName:"; 315 250 outstring = ""; 316 startpos = scanner.get_pos();251 startpos = block_base+scanner.get_pos(); 317 252 scanner.set_strm(&(marker.others)); 318 253 scanner.move_to(scanner.get_pos()); 319 scanner.scan_to_next();320 321 322 ch = src_buffer[ scanner.get_pos()];254 pos1: scan_to_next_and_test(scanner, 1); 255 256 257 ch = src_buffer[block_base+scanner.get_pos()]; 323 258 while (('0'<=ch && ch<='9') || ('A'<=ch && ch<='F') ||('a'<=ch && ch<='f')) { 324 259 outstring+=get_substring(src_buffer,startpos+1, scanner.get_pos()-2); 325 startpos = scanner.get_pos();260 startpos = block_base+scanner.get_pos(); 326 261 if (ch<='9') val = ch - '0'; 327 262 else if (ch<='F') val = 10 + ch - 'A'; 328 263 else val = 10 + ch - 'a'; 329 ch = src_buffer[ scanner.get_pos()-1];264 ch = src_buffer[block_base+scanner.get_pos()-1]; 330 265 331 266 if (ch<='9') val = val + 16*(ch - '0'); … … 334 269 outstring+=val; 335 270 scanner.scan_to_next(); 336 ch = src_buffer[ scanner.get_pos()];337 } 338 outstring+=get_substring(src_buffer,startpos+1, scanner.get_pos());271 ch = src_buffer[block_base+scanner.get_pos()]; 272 } 273 outstring+=get_substring(src_buffer,startpos+1,block_base+scanner.get_pos()); 339 274 scanner.set_strm(&(marker.starts)); 340 275 scanner.move_to(scanner.get_pos()-1); … … 346 281 cout<<"LexLiteralString:"; 347 282 outstring=""; 348 startpos = scanner.get_pos();283 startpos = block_base+scanner.get_pos(); 349 284 scanner.set_strm(&(marker.others)); 350 285 scanner.move_to(scanner.get_pos()); 351 scanner.scan_to_next();286 pos2: scan_to_next_and_test(scanner, 2); 352 287 while (!scanner.is_done()) { 353 outstring+=get_substring(src_buffer, startpos+1, scanner.get_pos());354 if (src_buffer[ scanner.get_pos()]==')') {288 outstring+=get_substring(src_buffer, startpos+1,block_base+scanner.get_pos()); 289 if (src_buffer[block_base+scanner.get_pos()]==')') { 355 290 cout<<outstring<<endl; 356 291 break; 357 292 } 358 startpos= scanner.get_pos();359 if (src_buffer[ scanner.get_pos()]=='\\') {360 ch = src_buffer[ scanner.get_pos()+1];293 startpos=block_base+scanner.get_pos(); 294 if (src_buffer[block_base+scanner.get_pos()]=='\\') { 295 ch = src_buffer[block_base+scanner.get_pos()+1]; 361 296 switch (ch) { 362 297 case 'n': … … 380 315 case 6: case 7: 381 316 val = ch - '0'; 382 if ((src_buffer[ scanner.get_pos()+2]) >= '0' && (src_buffer[scanner.get_pos()+2]<='8')) {383 val = val*8 + src_buffer[ scanner.get_pos()+2]-'0';317 if ((src_buffer[block_base+scanner.get_pos()+2]) >= '0' && (src_buffer[block_base+scanner.get_pos()+2]<='8')) { 318 val = val*8 + src_buffer[block_base+scanner.get_pos()+2]-'0'; 384 319 startpos+=1; 385 if ((src_buffer[ scanner.get_pos()+3]) >= '0' && (src_buffer[scanner.get_pos()+3]<='8')) {386 val = val*8 + src_buffer[ scanner.get_pos()+3]-'0';320 if ((src_buffer[block_base+scanner.get_pos()+3]) >= '0' && (src_buffer[block_base+scanner.get_pos()+3]<='8')) { 321 val = val*8 + src_buffer[block_base+scanner.get_pos()+3]-'0'; 387 322 startpos+=1; 388 323 } … … 392 327 break; 393 328 case 13: 394 if (src_buffer[ scanner.get_pos()+2]==10) {329 if (src_buffer[block_base+scanner.get_pos()+2]==10) { 395 330 startpos+=1; 396 331 } … … 407 342 else { 408 343 outstring+=0x0A; 409 if (src_buffer[ scanner.get_pos()+1]==0x0a) {344 if (src_buffer[block_base+scanner.get_pos()+1]==0x0a) { 410 345 startpos+=1; 411 346 } 412 347 } 413 scanner.scan_to_next(); 348 //CHANGE FOR FINAL BLOCK 349 pos3: scan_to_next_and_test(scanner, 3); 414 350 } 415 351 … … 429 365 cout << "LexNumber:"; 430 366 431 startpos = scanner.get_pos();367 startpos = block_base+scanner.get_pos(); 432 368 scanner.set_strm(&(marker.others)); 433 369 scanner.move_to(scanner.get_pos()); 434 scanner.scan_to_next();435 436 outstring=get_substring(src_buffer,startpos, scanner.get_pos());370 pos4: scan_to_next_and_test(scanner, 4); 371 372 outstring=get_substring(src_buffer,startpos,block_base+scanner.get_pos()); 437 373 scanner.set_strm(&(marker.starts)); 438 374 scanner.move_to(scanner.get_pos()-1); … … 444 380 scanner.set_strm(&(marker.others)); 445 381 scanner.move_to(scanner.get_pos()); 446 scanner.scan_to_next();447 ch = src_buffer[ scanner.get_pos()];382 pos5: scan_to_next_and_test(scanner, 5); 383 ch = src_buffer[block_base+scanner.get_pos()]; 448 384 while (('0'<=ch && ch<='9') || ('A'<=ch && ch<='F') ||('a'<=ch && ch<='f')) { 449 385 if (ch<='9') val = 16*(ch - '0'); 450 386 else if (ch<='F') val = 16*(10 + ch - 'A'); 451 387 else val = 16*(10 + ch - 'a'); 452 scanner.scan_to_next();453 ch = src_buffer[ scanner.get_pos()];388 pos6: scan_to_next_and_test(scanner, 6); 389 ch = src_buffer[block_base+scanner.get_pos()]; 454 390 if (('0'<=ch && ch<='9') || ('A'<=ch && ch<='F') ||('a'<=ch && ch<='f')) { 455 391 if (ch<='9') val += (ch - '0'); 456 392 else if (ch<='F') val += (10 + ch - 'A'); 457 393 else val += 10 + ch - 'a'; 458 scanner.scan_to_next(); 459 ch = src_buffer[scanner.get_pos()]; 394 395 //CHANGE FOR FINAL BLOCK (If scanner.is_done(), BASE_ASSERT) 396 pos7: scan_to_next_and_test(scanner, 7); 397 398 399 ch = src_buffer[block_base+scanner.get_pos()]; 460 400 } 461 401 outstring+=val; … … 478 418 scanner.set_strm(&(marker.others)); 479 419 scanner.move_to(scanner.get_pos()); 480 scanner.scan_to_next();420 pos8: scan_to_next_and_test(scanner, 8); 481 421 scanner.set_strm(&(marker.starts)); 482 422 … … 485 425 default: 486 426 cout<<"LexDefault:"; 487 startpos = scanner.get_pos();427 startpos = block_base+scanner.get_pos(); 488 428 scanner.set_strm(&(marker.others)); 489 429 scanner.move_to(scanner.get_pos()); 490 scanner.scan_to_next();491 outstring = get_substring(src_buffer,startpos, scanner.get_pos());430 pos9: scan_to_next_and_test(scanner, 9); 431 outstring = get_substring(src_buffer,startpos,block_base+scanner.get_pos()); 492 432 if (outstring=="null") { 493 433 cout << "NULL OBJECT"<<endl; … … 506 446 507 447 if(!scanner.is_done()) { 508 scanner.scan_to_next();448 pos10: scan_to_next_and_test(scanner, 10); 509 449 } 510 450 } … … 520 460 521 461 522 523 524 // postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base, chars_avail, tracker); 462 463 464 465 466 467 } 468 469 void do_process(FILE *infile, FILE *outfile) { 470 @decl 471 472 /* Extras */ 473 parity.odd = simd<2>::constant<1>(); 474 parity.even = simd<2>::constant<2>(); 475 476 int block_base = 0; 477 int block_pos = 0; 478 int chars_avail = 0; 479 int chars_read = 0; 480 int padding_read = 0; 481 int remaining = 0; 482 483 /* Tracked for post processing */ 484 //int buffer_base=0; 485 //int buffer_pos = 0; 486 487 ////////////////////////////////////////////////////////////////////////////////////////// 488 // Buffer Management 489 ////////////////////////////////////////////////////////////////////////////////////////// 490 uint8_t * src_buffer; 491 ALLOC_STATIC_ALIGNED_BYTE_BUFFER(src_buffer, BUFFER_SIZE); 492 493 ////////////////////////////////////////////////////////////////////////////////////////// 494 // Initial file read 495 ////////////////////////////////////////////////////////////////////////////////////////// 496 chars_read = fread((void *)src_buffer, 1, SEGMENT_SIZE + LOOKAHEAD_SIZE, infile); 497 remaining = chars_read; 498 chars_avail = chars_read; 499 padding_read = chars_read - SEGMENT_SIZE; 500 501 if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE; 502 503 @stream_stmts 504 505 ////////////////////////////////////////////////////////////////////////////////////////// 506 // Processs Full Segments 507 ////////////////////////////////////////////////////////////////////////////////////////// 508 while (chars_avail >= SEGMENT_SIZE) { 509 PERF_SEC_START(parser_timer); 510 for (int blk = 0; blk < SEGMENT_BLOCKS; blk++) { 511 block_base = blk*BLOCK_SIZE; 512 s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits); 513 514 #ifdef PDF_DEBUG 515 for (int i = 0; i < BLOCK_SIZE; i++) { 516 cout << src_buffer[block_base + i]; 517 } 518 #endif 519 520 @block_stmts 521 522 //postprocess_do_block(lex, ctCDPI_Callouts, ref_Callouts, check_streams, (char *)src_buffer, buffer_base, block_base,chars_avail, tracker); 523 524 } 525 PERF_SEC_END(parser_timer, chars_avail); 526 527 remaining -= SEGMENT_SIZE; 528 memmove(src_buffer, &src_buffer[SEGMENT_SIZE], padding_read); 529 chars_read = fread(&src_buffer[padding_read], 1, (SEGMENT_SIZE + LOOKAHEAD_SIZE - padding_read), infile); 530 531 padding_read = SEGMENT_SIZE + LOOKAHEAD_SIZE - padding_read ;//chars_read; 532 533 remaining += chars_read; 534 chars_avail = remaining; 535 536 if (chars_avail >= SEGMENT_SIZE) chars_avail = SEGMENT_SIZE; 537 538 /* Tracked for post processing */ 539 // buffer_pos += chars_avail; 540 // buffer_base = buffer_pos; 541 542 } 543 544 ////////////////////////////////////////////////////////////////////////////////////////// 545 // Final Partial Segment 546 ////////////////////////////////////////////////////////////////////////////////////////// 547 PERF_SEC_START(parser_timer); 548 549 block_base = 0; 550 551 /* Full Blocks */ 552 uint32_t blk = 0; 553 while (remaining >= BLOCK_SIZE) { 554 555 #ifdef PDF_DEBUG 556 for(int i = 0; i < BLOCK_SIZE; i++) { 557 cout << src_buffer[block_base + i]; 558 } 559 #endif 560 561 s2p_do_block((BytePack *) &src_buffer[block_base], basis_bits); 562 @block_stmts 563 564 postprocess_do_block(marker, src_buffer, block_base); 565 566 block_base += BLOCK_SIZE; 567 remaining -= BLOCK_SIZE; 568 blk++; 569 } 570 571 /* Partial Block or Any Carry */ 572 if (remaining > 0 || @any_carry) { 573 574 #ifdef PDF_DEBUG 575 for(int i = 0; i < remaining; i++) 576 { 577 std::cout << src_buffer[block_base+i]; 578 } 579 #endif 580 581 EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-remaining)); 582 s2p_do_final_block((BytePack *) &src_buffer[block_base], basis_bits, EOF_mask); 583 @final_block_stmts 584 //print_register2("markstarts",marker.starts,remaining); 585 586 587 588 589 postprocess_do_block(marker, src_buffer, block_base); 525 590 blk++; 526 591 } -
proto/PDF/pdf.py
r2629 r2639 272 272 273 273 if string_cursor : 274 #There should be a better way to do this 274 275 if (string_cursor & escaped_Callouts.UELParen) : 275 276 if is_zero(): … … 277 278 278 279 inc() 279 280 if (string_cursor & escaped_Callouts.UELParen) : 280 281 cursor = pablo.ScanTo(pablo.Advance(string_cursor), escaped_Callouts.UELParen | escaped_Callouts.UERParen) 281 282 if (string_cursor & escaped_Callouts.UERParen) : 282 283 283 dec() 284 284 if (string_cursor & escaped_Callouts.UERParen) : 285 285 if is_zero(): 286 286 literalStrings_Callouts.ends|=string_cursor
Note: See TracChangeset
for help on using the changeset viewer.
