source: icXML/icXML-devel/tests/src/xinclude/harness.pl @ 2726

Last change on this file since 2726 was 2726, checked in by cameron, 7 years ago

Add original Xerces tests and samples directories

File size: 7.2 KB
Line 
1#use strict;
2
3# usage perl harness.pl [testNum]
4#   where testNum is an integer from 1 to 41
5#   testNum may also be a list of integers denoting tests to be run
6#   ommitting testNum will lead to all tests being run
7#
8#  this harness outputs the results to the file results.html as well
9#   as reporting them to stderr.
10#
11#  note that the XInclude sample application must be discoverable on
12#   the current PATH.
13
14#
15# Find out the platform from 'uname -s'
16#
17open(PLATFORM, "uname -s|");
18$platform = <PLATFORM>;
19chomp($platform);
20close (PLATFORM);
21
22if ($platform =~ m/Windows/i  || $platform =~ m/CYGWIN/i) {
23    $pathsep = "\\";
24}
25else {
26    $pathsep = "/";
27}
28
29@CORRECT_TEST_RESULTS = (
30        -3,
31        1, 1, 1, 1, 1, 1, 0, 1, 0, 1,
32        0, 0, 0, 0, 1, 1, 1, 0, 1, 1,
33        1, 0, 1, 0, 0, 0, 1, 1, 0, 1,
34        1, 0, 1, 1, 1, 1, 1, 1, 0, 0,
35        1
36);
37
38#    private static boolean[] TEST_RESULTS = new boolean[] {
39#        // one value for each test
40#        true, true, true, true, true, true, false, true, false, true, // 10
41#        false, false, false, false, true, true, true, false, true, true, // 20
42#        true, false, true, false, false, false, true, true, false, true, // 30
43#        true, false, true, true, true, true, true, true, false, false, // 40
44#        true, };
45
46#no need to add one to this value since there is a dummy value in the tables
47$NUM_TESTS = $#CORRECT_TEST_RESULTS;
48
49@ACTUAL_TEST_RESULTS = (
50        -3,
51        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 
52        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 
53        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 
54        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 
55        -1
56);
57
58print "Running XInclude Tests\n";
59
60$dateAndTime = localtime();
61
62open(outfile, ">results.html");
63print outfile "<html><head><title>XInclude results generated at $dateAndTime </title>";
64print outfile "</head><body bgcolor=\"black\" text=\"white\"><center>";
65
66print outfile "<table border cellspacing=\"0\" cellpadding=\"5\"><caption align=\"bottom\">Results of XI Test Run</caption>";
67print outfile "<tr><th>Test Number</th><th>Test Input File</th><th>Generated Output File</th>";
68print outfile "<th>Reference Result File</th><th>Result</th><th>Expected Result</th><th>Test Status</th></tr>";
69
70$successes = 0;
71$correct = 0;
72$numTestsToRun = $#ARGV + 1;
73while (<@ARGV>){
74   $i = $_;
75   if ($i < 10) {
76      $testNum = "0$i";
77   } else {
78      $testNum = "$i";
79   }
80   $input_file =  "tests".$pathsep."xinclude".$pathsep."tests".$pathsep."test$testNum.xml";
81   if ($CORRECT_TEST_RESULTS[$i] == 1) {
82      $output_file = "tests".$pathsep."xinclude".$pathsep."written".$pathsep."test$testNum.xml";
83      $expected_result_file = "tests".$pathsep."xinclude".$pathsep."cppoutput".$pathsep."test$testNum.xml";
84   } else {
85      $output_file = "tests".$pathsep."xinclude".$pathsep."written".$pathsep."test$testNum.txt";
86      $expected_result_file = "tests".$pathsep."xinclude".$pathsep."cppoutput".$pathsep."test$testNum.txt";
87   }
88
89   # run the tests and apture the output
90   $command = "XInclude $input_file $output_file";
91   print "R[$command]\n";
92   system("$command");
93
94   $outcome = 0;
95   if (! -e $output_file) {
96    $ACTUAL_TEST_RESULTS[$i] = 0;
97    # we didn't generate an output, we are correct if the test was expected to fail
98    if($CORRECT_TEST_RESULTS[$i] eq 0) {
99      $outcome = 1;
100    }
101   } else {
102    $ACTUAL_TEST_RESULTS[$i] = 1;
103    # we generated an output, we are correct if our result matches
104    if($CORRECT_TEST_RESULTS[$i] eq 1) {
105      $outcome = &compareFiles($expected_result_file, $output_file);
106    }
107   }
108
109   print outfile "<tr bgcolor=";
110   if ($outcome eq 1) {
111      print outfile "green";
112   } else {
113      print outfile "red";
114   }
115
116   print outfile ">";
117   #test number
118   print outfile "<td>$testNum</td>";
119   print outfile "<td>";
120   $anchor = &createHTMLAnchor("$input_file");
121   print outfile "$anchor";
122   print outfile "</td><td>";
123   $anchor = &createHTMLAnchor("$output_file");
124   print outfile "$anchor";
125   print outfile "</td><td>";
126   $anchor = &createHTMLAnchor("$expected_result_file");
127   print outfile "$anchor";
128   print outfile "</td>";
129   # actual result
130   $result = (($ACTUAL_TEST_RESULTS[$i])?"true":"false");
131   print outfile "<td>$result</td>";
132
133   # correct result
134   $result = (($CORRECT_TEST_RESULTS[$i])?"true":"false");
135   print outfile "<td>$result</td>";
136
137   if ($outcome eq 1) {
138      $successes++;
139      $result = "Passed";
140      print "[test $testNum PASSED]\n";
141      $correct++;
142   } else {
143      $result = "Failed";
144      print "[test $testNum FAILED]\n";     
145   }
146   print outfile "<td>$result</td></tr>";
147}
148
149print outfile "</table>";
150
151print "Tests Passed: $correct\n";
152$percentSucceeding = $correct / $numTestsToRun * 100.0;
153print outfile "<hr/><h2>$percentSucceeding% of tests that were run passed ($correct out of $numTestsToRun).</h2>";
154
155$shPass = 0;
156$shFail = 0;
157for ($i = 1; $i < $NUM_TESTS+1; $i++){
158   if ($CORRECT_TEST_RESULTS[$i]){
159       $shPass++;
160   } else {
161       $shFail++;
162   }
163}
164
165print outfile "<h2>$shPass should pass ($successes) did</h2>";
166$failures = $NUM_TESTS - $successes;
167print outfile "<h2>$shFail should fail ($failures) did</h2>";
168
169$percentSucceeding = $correct / $NUM_TESTS * 100.0; 
170print outfile "<h2>$percentSucceeding% of all available tests passed ($correct out of $NUM_TESTS).</h2>";
171
172$percentSucceeding = $numTestsToRun / $NUM_TESTS * 100.0;
173print outfile "<h2>$percentSucceeding% of all available tests were run ($numTestsToRun out of $NUM_TESTS).</h2>";
174
175print outfile "</center><hr/></body></html>";
176close(outfile);
177
178# quick and dirty but functional
179sub compareFiles 
180{
181   $result = 1;
182   print "C[$_[0]]:[$_[1]]\n";
183
184   if (! -e $_[0]) {
185      print "No such file as $_[0] - cannot compare\n";
186      return 0;
187   }
188
189   if (! -e $_[1]) {
190      print "No such file as $_[1] - cannot compare\n";
191      return 0;
192   }
193
194   open(expected, "<$_[0]");
195   open(actual, "<$_[1]");
196
197   #compare the files
198   @expectedData = <expected>;
199   @actualData = <actual>;
200
201   close(expected);
202   close(actual);
203
204   for ($x = 0, $a = 0; $x < $#expectedData && $a < $#actualData; $x++, $a++){
205      $dataLineEx = $expectedData[$x];
206      $dataLineAc = $actualData[$a];
207      chomp($dataLineEx);
208      chomp($dataLineAc);
209      if ($dataLineEx ne $dataLineAc) {
210         # check if its a warning and can be ignored at this stage
211         if ($dataLineEx =~ /Warning/) {
212            print "probably just a warning line:\n";
213            print "[$dataLineEx]\n";
214            $a--
215         } elsif ( $dataLineEx =~ m/^\s*$/ && $dataLineAc =~ m/^\s*$/ ) {
216            $line_num = $x + 1;
217            print "Line $line_num ws difference: \n[$dataLineEx] != \n[$dataLineAc]\n";
218            print "probably just white space on both lines\n";
219            $a--
220         } else {
221            $line_num = $x + 1;
222            print "Line $line_num difference: \n[$dataLineEx] != \n[$dataLineAc]\n";
223            print "From files: $_[0] $_[1]\n";
224            return 0;
225         }
226      }
227   }
228
229   return 1;
230}
231
232sub createHTMLAnchor {
233   "<a target=\"_blank\" href=\"$_[0]\">$_[0]</a>";
234}
Note: See TracBrowser for help on using the repository browser.