Changeset 3738


Ignore:
Timestamp:
Mar 21, 2014, 2:43:21 AM (5 years ago)
Author:
ksherdy
Message:

Added tests. Added -b -o grep flags.

Location:
proto/RE/demo
Files:
1 added
5 edited

Legend:

Unmodified
Added
Removed
  • proto/RE/demo/grep_demo.py

    r3737 r3738  
    143143def MatchLines2(lex, output): # verbose with temporary streams
    144144        line_start      = 0
    145         line_end                = 0
     145        line_end        = 0
    146146        line_start      = pablo.Advance(~line_start) ^ ~line_start # line_start == pablo.First()
    147147        output.line_starts = line_start
  • proto/RE/demo/grep_template_segment.cpp

    r3736 r3738  
    4141
    4242// Fixed pattern.
    43 const char * delim          = ":";
    44 const char * newline        = "\n";
    4543const char * fixed_pattern  = "apple";
    4644const int pattern_size      = strlen(fixed_pattern);
     
    6159//const int BUFFER_SEGMENTS  = CACHE_SIZE / SEGMENT_SIZE;
    6260//const int BUFFER_SIZE      = BUFFER_SEGMENTS * SEGMENT_SIZE; // SEGMENT_SIZE; //
    63 
    64 BitBlock make_EOF_mask(int size);
    6561
    6662// @ global - Parameter replaced with C++ translation of stream structs
     
    127123
    128124int match_offset            = 0;
    129 int start_offset            = 0;   
    130 int end_offset              = -1;
    131 
    132 int starts_final_offset     = 0;
    133 int ends_final_offset       = 0;
     125int line_start_offset            = 0;   
     126int line_end_offset              = -1;
     127
     128int line_starts_final_offset     = 0;
     129int line_ends_final_offset       = 0;
    134130
    135131int main(int argc, char * argv[]) {
     
    219215    bytes_avail     = bytes_read + copy_back_size;
    220216    bytes_remaining = bytes_avail;
    221 
    222     if(feof(infile)) { bytes_remaining--; }
     217   
     218    //if(feof(infile)) { KH: No! OPC /*bytes_remaining--;*/ }
    223219    if(ferror(infile)) { perror( "io error" ); exit(1); }
    224220
     
    248244        }
    249245
    250         line_starts_scanner.load_block(output.line_starts, block_index);
    251         line_ends_scanner.load_block(output.line_ends, block_index);
     246        if(!only_matching) {
     247            line_starts_scanner.load_block(output.line_starts, block_index);
     248            line_ends_scanner.load_block(output.line_ends, block_index);
     249        }
    252250      }
    253251   
     
    260258          }
    261259         
    262           // KH: Lookahead or dies at segment boundary.
    263           //fwrite(&buffer[match_offset], 1, pattern_size, outfile);
    264          
    265           fprintf(outfile, "%s\n", fixed_pattern);
    266           //fprintf(outfile, "\n");
    267         }
     260          // KH: Lookahead.
     261          fwrite(&buffer[match_offset], 1, pattern_size, outfile);
     262         
     263          //fprintf(outfile, "%s\n", fixed_pattern);
     264          fprintf(outfile, "\n");
     265        }
     266       
     267        copy_back_size      = pattern_size;           
     268        copy_back_offset    = bytes_avail - copy_back_size;           
     269       
    268270      }
    269271 
    270       // both, either, or none (in the general case)     
    271       bool has_line_start   = line_starts_scanner.has_next();
    272       bool has_line_end     = line_ends_scanner.has_next();
     272      if(!only_matching) {
     273         
     274          print_register<BitBlock>("starts", output.line_starts);
     275          print_register<BitBlock>("starts", output.line_ends);
    273276     
    274       //if(has_line_start) {
    275           starts_final_offset = line_starts_scanner.get_final_pos();
    276       //}
    277       //if(has_line_end) {
    278           ends_final_offset = line_ends_scanner.get_final_pos();
    279       //}
    280       assert(("Line length exceeds segment size.", has_line_end));
    281       // if(!has_line_start && !has_line_end) {/* Set flag to buffer entire segment. */;}
    282      
    283       while(line_starts_scanner.has_next() && line_ends_scanner.has_next()) {
    284           start_offset  = line_starts_scanner.scan_to_next();
    285           end_offset    = line_ends_scanner.scan_to_next();
    286           ////printf("(%d,%d)\n", start_offset, end_offset);
    287       }
    288      
    289       copy_back_offset   = (starts_final_offset > ends_final_offset) ? starts_final_offset : (ends_final_offset + 1);
    290       copy_back_size  = bytes_avail - copy_back_offset;   
    291      
    292       //cout << copy_back_offset << ", " << copy_back_size << endl;
    293      
    294       bytes_remaining -= SEGMENT_SIZE;
     277          // both, either, or none (in the general case)     
     278          bool has_line_start   = line_starts_scanner.has_next();
     279          bool has_line_end     = line_ends_scanner.has_next();
     280         
     281          //if(has_line_start) {
     282              line_starts_final_offset = line_starts_scanner.get_final_pos();
     283          //}
     284          //if(has_line_end) {
     285              line_ends_final_offset = line_ends_scanner.get_final_pos();
     286          //}
     287          assert(("Line length exceeds segment size.", has_line_end));
     288          // if(!has_line_start && !has_line_end) {/* Set flag to buffer entire segment. */;}
     289         
     290          while(line_starts_scanner.has_next() && line_ends_scanner.has_next()) {
     291             
     292              line_start_offset  = line_starts_scanner.scan_to_next();
     293              line_end_offset    = line_ends_scanner.scan_to_next();
     294             
     295              if(byte_offset) {
     296                  fprintf(outfile, "%d:", stream_base + line_start_offset);
     297              }
     298             
     299              fwrite(&buffer[line_start_offset], 1, line_end_offset - line_start_offset, outfile);
     300              fprintf(outfile, "\n");
     301          }
     302         
     303          copy_back_offset   = (line_starts_final_offset > line_ends_final_offset) ? line_starts_final_offset : (line_ends_final_offset + 1);
     304          copy_back_size  = bytes_avail - copy_back_offset;   
     305         
     306          bytes_remaining -= SEGMENT_SIZE;
     307      }   
    295308     
    296309    }
     
    327340
    328341        // Process the final partial block. // KH: Not required, remove.
    329         if(bytes_remaining > 0) {
    330           BitBlock EOF_mask = make_EOF_mask(bytes_remaining);
     342        if(bytes_remaining > 0) {           
     343          BitBlock EOF_mask = bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-bytes_remaining));
     344         
    331345          byte_data = &buffer[block_base];
    332346          //Replaced with C++ stream function 'do_final_block()' calls.
     
    338352         
    339353          line_starts_scanner.load_block(output.line_starts & EOF_mask, block_index);
    340           line_ends_scanner.load_block(output.line_ends & EOF_mask, block_index);
     354          line_ends_scanner.load_block(output.line_ends & EOF_mask, block_index);         
     355         
     356          if(0) {
     357              print_register<BitBlock>("starts", output.line_starts);
     358              print_register<BitBlock>("ends", output.line_ends);
     359              print_register<BitBlock>("lex.LF", lex.LF);
     360              print_register<BitBlock>("mask", EOF_mask);
     361          }
    341362        }
    342363
     
    349370            }
    350371           
    351             // KH: Lookahead or dies at segment boundary.
    352             //fwrite(&buffer[match_offset], 1, pattern_size, outfile);
    353             fprintf(outfile, "%s\n", fixed_pattern);
    354             //fprintf(outfile, "\n");
     372            // KH: Lookahead.
     373            fwrite(&buffer[match_offset], 1, pattern_size, outfile);
     374            //fprintf(outfile, "%s\n", fixed_pattern);
     375            fprintf(outfile, "\n");
    355376          }
    356         }
    357        
    358         // both, either, or none (in the general case)     
    359         bool has_line_start   = line_starts_scanner.has_next();
    360         bool has_line_end     = line_ends_scanner.has_next();
    361        
    362         //if(has_line_start) {
    363             starts_final_offset = line_starts_scanner.get_final_pos();
    364         //}
    365         //if(has_line_end) {
    366             ends_final_offset = line_ends_scanner.get_final_pos();
    367         //}
    368         assert(("Line length exceeds segment size.", has_line_end));
    369         // if(!has_line_start && !has_line_end) {/* Set flag to buffer entire segment. */;}
    370        
    371         while(line_starts_scanner.has_next() && line_ends_scanner.has_next()) {
    372             start_offset  = line_starts_scanner.scan_to_next();
    373             end_offset    = line_ends_scanner.scan_to_next();
    374             ////printf("(%d,%d)\n", start_offset, end_offset);
    375         }
    376        
    377         copy_back_offset    = (starts_final_offset > ends_final_offset) ? starts_final_offset : (ends_final_offset + 1);
    378         copy_back_size      = bytes_avail - copy_back_offset;   
    379        
    380         //cout << copy_back_offset << ", " << copy_back_size << endl;
     377         
     378          copy_back_size      = pattern_size;           
     379          copy_back_offset    = bytes_avail - copy_back_size;           
     380         
     381        }
     382     
     383        if(!only_matching) {
     384       
     385            // both, either, or none (in the general case)     
     386            bool has_line_start   = line_starts_scanner.has_next();
     387            bool has_line_end     = line_ends_scanner.has_next();
     388           
     389            //if(has_line_start) {
     390                line_starts_final_offset = line_starts_scanner.get_final_pos();
     391            //}
     392            //if(has_line_end) {
     393                line_ends_final_offset = line_ends_scanner.get_final_pos();
     394            //}
     395            //assert(("Line length exceeds segment size.", has_line_end));
     396            // if(!has_line_start && !has_line_end) {/* Set flag to buffer entire segment. */;}
     397           
     398            while(line_starts_scanner.has_next() && line_ends_scanner.has_next()) {
     399               
     400                line_start_offset  = line_starts_scanner.scan_to_next();
     401                line_end_offset    = line_ends_scanner.scan_to_next();
     402               
     403                if(byte_offset) {
     404                    fprintf(outfile, "%d:", stream_base + line_start_offset);
     405                }
     406               
     407                fwrite(&buffer[line_start_offset], 1, line_end_offset - line_start_offset, outfile);
     408                fprintf(outfile, "\n");
     409            }
     410
     411            copy_back_offset    = (line_starts_final_offset > line_ends_final_offset) ? line_starts_final_offset : (line_ends_final_offset + 1);
     412            copy_back_size      = bytes_avail - copy_back_offset;           
     413           
     414        }
     415
    381416    }
    382417     
     
    391426  return 0;
    392427}
    393 
    394 BitBlock make_EOF_mask(int size) {
    395   return bitblock::srl(simd<1>::constant<1>(), convert(BLOCK_SIZE-size));
    396 }
    397 
  • proto/RE/demo/src/Makefile

    r3736 r3738  
    22FIXED_PATTERN=apple
    33OUTFILE=grep
     4TESTDIR=../test
    45TESTFILE0=../test/fruitlist0.dat
    56TESTFILE1=../test/fruitlist1.dat
     
    2829grep:   $(SRCFILE)
    2930        $(CC) -o $(OUTFILE) $(SRCFILE) $(INCLUDES) $(AFLAGS)
    30        
    31 grep_test_b_o: $(OUTFILE)
    32         ./$(OUTFILE) $(TESTFILE0) -o -b > 0     
    33         grep $(FIXED_PATTERN) $(TESTFILE0) -o -b > 1
    34         diff 0 1
    35         ./$(OUTFILE) $(TESTFILE1) -o -b > 0     
    36         grep $(FIXED_PATTERN) $(TESTFILE1) -o -b > 1
    37         diff 0 1
    38         ./$(OUTFILE) $(TESTFILE2) -o -b > 0     
    39         grep $(FIXED_PATTERN) $(TESTFILE2) -o -b > 1
    40         diff 0 1
    41         ./$(OUTFILE) $(TESTFILE3) -o -b > 0     
    42         grep $(FIXED_PATTERN) $(TESTFILE3) -o -b > 1
    43         diff 0 1
    44         ./$(OUTFILE) $(TESTFILE4) -o -b > 0     
    45         grep $(FIXED_PATTERN) $(TESTFILE4) -o -b > 1
    46         diff 0 1       
     31
     32test: $(OUTFILE)
     33        for file in ../test/* ; do \
     34                # echo $$file ; \
     35        ./$(OUTFILE) $$file > 0 ; \
     36        grep $(FIXED_PATTERN) $$file > 1 ; \
     37        diff 0 1 > 2; \
     38        done   
     39
     40test_all: test test_b test_o test_b_o
     41         
     42
     43test_b: $(OUTFILE)
     44        for file in ../test/* ; do \
     45                # echo $$file ; \
     46        ./$(OUTFILE) $$file -b > 0 ; \
     47        grep $(FIXED_PATTERN) $$file -b > 1 ; \
     48        diff 0 1 > 2; \
     49        done   
     50
     51test_o: $(OUTFILE)
     52        for file in ../test/* ; do \
     53                # echo $$file ; \
     54        ./$(OUTFILE) $$file -o > 0 ; \
     55        grep $(FIXED_PATTERN) $$file -o > 1 ; \
     56        diff 0 1 > 2; \
     57        done   
     58   
     59test_b_o: $(OUTFILE)
     60        for file in ../test/* ; do \
     61                # echo $$file ; \
     62        ./$(OUTFILE) $$file -o -b > 0 ; \
     63        grep $(FIXED_PATTERN) $$file -o -b > 1 ; \
     64        diff 0 1 > 2; \
     65        done   
    4766
    4867clean:
  • proto/RE/demo/test/fruitlist3.dat

    r3730 r3738  
    66banana
    77 pineapple
     8 
  • proto/RE/demo/test/fruitlist4.dat

    r3736 r3738  
    77applepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepineapplepinepineap
    88crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple crab apple
    9 
Note: See TracChangeset for help on using the changeset viewer.