Commit | Line | Data |
---|---|---|
395cf596 SDJ |
1 | #!/usr/bin/python |
2 | # | |
3 | # Copyright (C) 2014 Free Software Foundation, Inc. | |
4 | # | |
5 | # This script is free software; you can redistribute it and/or modify | |
6 | # it under the terms of the GNU General Public License as published by | |
7 | # the Free Software Foundation; either version 3, or (at your option) | |
8 | # any later version. | |
9 | ||
10 | import sys | |
11 | import getopt | |
12 | import re | |
13 | import io | |
14 | from datetime import datetime | |
15 | from operator import attrgetter | |
16 | ||
17 | # True if unrecognised lines should cause a fatal error. Might want to turn | |
18 | # this on by default later. | |
19 | strict = False | |
20 | ||
21 | # True if the order of .log segments should match the .sum file, false if | |
22 | # they should keep the original order. | |
23 | sort_logs = True | |
24 | ||
25 | # A version of open() that is safe against whatever binary output | |
26 | # might be added to the log. | |
27 | def safe_open (filename): | |
28 | if sys.version_info >= (3, 0): | |
29 | return open (filename, 'r', errors = 'surrogateescape') | |
30 | return open (filename, 'r') | |
31 | ||
32 | # Force stdout to handle escape sequences from a safe_open file. | |
33 | if sys.version_info >= (3, 0): | |
34 | sys.stdout = io.TextIOWrapper (sys.stdout.buffer, | |
35 | errors = 'surrogateescape') | |
36 | ||
37 | class Named: | |
38 | def __init__ (self, name): | |
39 | self.name = name | |
40 | ||
41 | class ToolRun (Named): | |
42 | def __init__ (self, name): | |
43 | Named.__init__ (self, name) | |
44 | # The variations run for this tool, mapped by --target_board name. | |
45 | self.variations = dict() | |
46 | ||
47 | # Return the VariationRun for variation NAME. | |
48 | def get_variation (self, name): | |
49 | if name not in self.variations: | |
50 | self.variations[name] = VariationRun (name) | |
51 | return self.variations[name] | |
52 | ||
53 | class VariationRun (Named): | |
54 | def __init__ (self, name): | |
55 | Named.__init__ (self, name) | |
56 | # A segment of text before the harness runs start, describing which | |
57 | # baseboard files were loaded for the target. | |
58 | self.header = None | |
59 | # The harnesses run for this variation, mapped by filename. | |
60 | self.harnesses = dict() | |
61 | # A list giving the number of times each type of result has | |
62 | # been seen. | |
63 | self.counts = [] | |
64 | ||
65 | # Return the HarnessRun for harness NAME. | |
66 | def get_harness (self, name): | |
67 | if name not in self.harnesses: | |
68 | self.harnesses[name] = HarnessRun (name) | |
69 | return self.harnesses[name] | |
70 | ||
71 | class HarnessRun (Named): | |
72 | def __init__ (self, name): | |
73 | Named.__init__ (self, name) | |
74 | # Segments of text that make up the harness run, mapped by a test-based | |
75 | # key that can be used to order them. | |
76 | self.segments = dict() | |
77 | # Segments of text that make up the harness run but which have | |
78 | # no recognized test results. These are typically harnesses that | |
79 | # are completely skipped for the target. | |
80 | self.empty = [] | |
81 | # A list of results. Each entry is a pair in which the first element | |
82 | # is a unique sorting key and in which the second is the full | |
83 | # PASS/FAIL line. | |
84 | self.results = [] | |
85 | ||
86 | # Add a segment of text to the harness run. If the segment includes | |
87 | # test results, KEY is an example of one of them, and can be used to | |
88 | # combine the individual segments in order. If the segment has no | |
89 | # test results (e.g. because the harness doesn't do anything for the | |
90 | # current configuration) then KEY is None instead. In that case | |
91 | # just collect the segments in the order that we see them. | |
92 | def add_segment (self, key, segment): | |
93 | if key: | |
94 | assert key not in self.segments | |
95 | self.segments[key] = segment | |
96 | else: | |
97 | self.empty.append (segment) | |
98 | ||
99 | class Segment: | |
100 | def __init__ (self, filename, start): | |
101 | self.filename = filename | |
102 | self.start = start | |
103 | self.lines = 0 | |
104 | ||
105 | class Prog: | |
106 | def __init__ (self): | |
107 | # The variations specified on the command line. | |
108 | self.variations = [] | |
109 | # The variations seen in the input files. | |
110 | self.known_variations = set() | |
111 | # The tools specified on the command line. | |
112 | self.tools = [] | |
113 | # Whether to create .sum rather than .log output. | |
114 | self.do_sum = True | |
115 | # Regexps used while parsing. | |
116 | self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$') | |
117 | self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$') | |
118 | self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED' | |
119 | r'|WARNING|ERROR|UNSUPPORTED|UNTESTED' | |
120 | r'|KFAIL):\s*(.+)') | |
121 | self.completed_re = re.compile (r'.* completed at (.*)') | |
122 | # Pieces of text to write at the head of the output. | |
123 | # start_line is a pair in which the first element is a datetime | |
124 | # and in which the second is the associated 'Test Run By' line. | |
125 | self.start_line = None | |
126 | self.native_line = '' | |
127 | self.target_line = '' | |
128 | self.host_line = '' | |
129 | self.acats_premable = '' | |
130 | # Pieces of text to write at the end of the output. | |
131 | # end_line is like start_line but for the 'runtest completed' line. | |
132 | self.acats_failures = [] | |
133 | self.version_output = '' | |
134 | self.end_line = None | |
135 | # Known summary types. | |
136 | self.count_names = [ | |
137 | '# of expected passes\t\t', | |
138 | '# of unexpected failures\t', | |
139 | '# of unexpected successes\t', | |
140 | '# of expected failures\t\t', | |
141 | '# of unknown successes\t\t', | |
142 | '# of known failures\t\t', | |
143 | '# of untested testcases\t\t', | |
144 | '# of unresolved testcases\t', | |
145 | '# of unsupported tests\t\t' | |
146 | ] | |
147 | self.runs = dict() | |
148 | ||
149 | def usage (self): | |
150 | name = sys.argv[0] | |
151 | sys.stderr.write ('Usage: ' + name | |
152 | + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ... | |
153 | ||
154 | tool The tool (e.g. g++, libffi) for which to create a | |
155 | new test summary file. If not specified then output | |
156 | is created for all tools. | |
157 | variant-list One or more test variant names. If the list is | |
158 | not specified then one is constructed from all | |
159 | variants in the files for <tool>. | |
160 | sum-file A test summary file with the format of those | |
161 | created by runtest from DejaGnu. | |
162 | If -L is used, merge *.log files instead of *.sum. In this | |
163 | mode the exact order of lines may not be preserved, just different | |
164 | Running *.exp chunks should be in correct order. | |
165 | ''') | |
166 | sys.exit (1) | |
167 | ||
168 | def fatal (self, what, string): | |
169 | if not what: | |
170 | what = sys.argv[0] | |
171 | sys.stderr.write (what + ': ' + string + '\n') | |
172 | sys.exit (1) | |
173 | ||
174 | # Parse the command-line arguments. | |
175 | def parse_cmdline (self): | |
176 | try: | |
177 | (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L') | |
178 | if len (self.files) == 0: | |
179 | self.usage() | |
180 | for (option, value) in options: | |
181 | if option == '-l': | |
182 | self.variations.append (value) | |
183 | elif option == '-t': | |
184 | self.tools.append (value) | |
185 | else: | |
186 | self.do_sum = False | |
187 | except getopt.GetoptError as e: | |
188 | self.fatal (None, e.msg) | |
189 | ||
190 | # Try to parse time string TIME, returning an arbitrary time on failure. | |
191 | # Getting this right is just a nice-to-have so failures should be silent. | |
192 | def parse_time (self, time): | |
193 | try: | |
194 | return datetime.strptime (time, '%c') | |
195 | except ValueError: | |
196 | return datetime.now() | |
197 | ||
198 | # Parse an integer and abort on failure. | |
199 | def parse_int (self, filename, value): | |
200 | try: | |
201 | return int (value) | |
202 | except ValueError: | |
203 | self.fatal (filename, 'expected an integer, got: ' + value) | |
204 | ||
205 | # Return a list that represents no test results. | |
206 | def zero_counts (self): | |
207 | return [0 for x in self.count_names] | |
208 | ||
209 | # Return the ToolRun for tool NAME. | |
210 | def get_tool (self, name): | |
211 | if name not in self.runs: | |
212 | self.runs[name] = ToolRun (name) | |
213 | return self.runs[name] | |
214 | ||
215 | # Add the result counts in list FROMC to TOC. | |
216 | def accumulate_counts (self, toc, fromc): | |
217 | for i in range (len (self.count_names)): | |
218 | toc[i] += fromc[i] | |
219 | ||
220 | # Parse the list of variations after 'Schedule of variations:'. | |
221 | # Return the number seen. | |
222 | def parse_variations (self, filename, file): | |
223 | num_variations = 0 | |
224 | while True: | |
225 | line = file.readline() | |
226 | if line == '': | |
227 | self.fatal (filename, 'could not parse variation list') | |
228 | if line == '\n': | |
229 | break | |
230 | self.known_variations.add (line.strip()) | |
231 | num_variations += 1 | |
232 | return num_variations | |
233 | ||
234 | # Parse from the first line after 'Running target ...' to the end | |
235 | # of the run's summary. | |
236 | def parse_run (self, filename, file, tool, variation, num_variations): | |
237 | header = None | |
238 | harness = None | |
239 | segment = None | |
240 | final_using = 0 | |
241 | ||
242 | # If this is the first run for this variation, add any text before | |
243 | # the first harness to the header. | |
244 | if not variation.header: | |
245 | segment = Segment (filename, file.tell()) | |
246 | variation.header = segment | |
247 | ||
248 | # Parse up until the first line of the summary. | |
249 | if num_variations == 1: | |
250 | end = '\t\t=== ' + tool.name + ' Summary ===\n' | |
251 | else: | |
252 | end = ('\t\t=== ' + tool.name + ' Summary for ' | |
253 | + variation.name + ' ===\n') | |
254 | while True: | |
255 | line = file.readline() | |
256 | if line == '': | |
257 | self.fatal (filename, 'no recognised summary line') | |
258 | if line == end: | |
259 | break | |
260 | ||
261 | # Look for the start of a new harness. | |
262 | if line.startswith ('Running ') and line.endswith (' ...\n'): | |
263 | # Close off the current harness segment, if any. | |
264 | if harness: | |
265 | segment.lines -= final_using | |
266 | harness.add_segment (first_key, segment) | |
267 | name = line[len ('Running '):-len(' ...\n')] | |
268 | harness = variation.get_harness (name) | |
269 | segment = Segment (filename, file.tell()) | |
270 | first_key = None | |
271 | final_using = 0 | |
272 | continue | |
273 | ||
274 | # Record test results. Associate the first test result with | |
275 | # the harness segment, so that if a run for a particular harness | |
276 | # has been split up, we can reassemble the individual segments | |
277 | # in a sensible order. | |
278 | # | |
279 | # dejagnu sometimes issues warnings about the testing environment | |
280 | # before running any tests. Treat them as part of the header | |
281 | # rather than as a test result. | |
282 | match = self.result_re.match (line) | |
283 | if match and (harness or not line.startswith ('WARNING:')): | |
284 | if not harness: | |
285 | self.fatal (filename, 'saw test result before harness name') | |
286 | name = match.group (2) | |
287 | # Ugly hack to get the right order for gfortran. | |
288 | if name.startswith ('gfortran.dg/g77/'): | |
289 | name = 'h' + name | |
290 | key = (name, len (harness.results)) | |
291 | harness.results.append ((key, line)) | |
292 | if not first_key and sort_logs: | |
293 | first_key = key | |
294 | ||
295 | # 'Using ...' lines are only interesting in a header. Splitting | |
296 | # the test up into parallel runs leads to more 'Using ...' lines | |
297 | # than there would be in a single log. | |
298 | if line.startswith ('Using '): | |
299 | final_using += 1 | |
300 | else: | |
301 | final_using = 0 | |
302 | ||
303 | # Add other text to the current segment, if any. | |
304 | if segment: | |
305 | segment.lines += 1 | |
306 | ||
307 | # Close off the final harness segment, if any. | |
308 | if harness: | |
309 | segment.lines -= final_using | |
310 | harness.add_segment (first_key, segment) | |
311 | ||
312 | # Parse the rest of the summary (the '# of ' lines). | |
313 | if len (variation.counts) == 0: | |
314 | variation.counts = self.zero_counts() | |
315 | while True: | |
316 | before = file.tell() | |
317 | line = file.readline() | |
318 | if line == '': | |
319 | break | |
320 | if line == '\n': | |
321 | continue | |
322 | if not line.startswith ('# '): | |
323 | file.seek (before) | |
324 | break | |
325 | found = False | |
326 | for i in range (len (self.count_names)): | |
327 | if line.startswith (self.count_names[i]): | |
328 | count = line[len (self.count_names[i]):-1].strip() | |
329 | variation.counts[i] += self.parse_int (filename, count) | |
330 | found = True | |
331 | break | |
332 | if not found: | |
333 | self.fatal (filename, 'unknown test result: ' + line[:-1]) | |
334 | ||
335 | # Parse an acats run, which uses a different format from dejagnu. | |
336 | # We have just skipped over '=== acats configuration ==='. | |
337 | def parse_acats_run (self, filename, file): | |
338 | # Parse the preamble, which describes the configuration and logs | |
339 | # the creation of support files. | |
340 | record = (self.acats_premable == '') | |
341 | if record: | |
342 | self.acats_premable = '\t\t=== acats configuration ===\n' | |
343 | while True: | |
344 | line = file.readline() | |
345 | if line == '': | |
346 | self.fatal (filename, 'could not parse acats preamble') | |
347 | if line == '\t\t=== acats tests ===\n': | |
348 | break | |
349 | if record: | |
350 | self.acats_premable += line | |
351 | ||
352 | # Parse the test results themselves, using a dummy variation name. | |
353 | tool = self.get_tool ('acats') | |
354 | variation = tool.get_variation ('none') | |
355 | self.parse_run (filename, file, tool, variation, 1) | |
356 | ||
357 | # Parse the failure list. | |
358 | while True: | |
359 | before = file.tell() | |
360 | line = file.readline() | |
361 | if line.startswith ('*** FAILURES: '): | |
362 | self.acats_failures.append (line[len ('*** FAILURES: '):-1]) | |
363 | continue | |
364 | file.seek (before) | |
365 | break | |
366 | ||
367 | # Parse the final summary at the end of a log in order to capture | |
368 | # the version output that follows it. | |
369 | def parse_final_summary (self, filename, file): | |
370 | record = (self.version_output == '') | |
371 | while True: | |
372 | line = file.readline() | |
373 | if line == '': | |
374 | break | |
375 | if line.startswith ('# of '): | |
376 | continue | |
377 | if record: | |
378 | self.version_output += line | |
379 | if line == '\n': | |
380 | break | |
381 | ||
382 | # Parse a .log or .sum file. | |
383 | def parse_file (self, filename, file): | |
384 | tool = None | |
385 | target = None | |
386 | num_variations = 1 | |
387 | while True: | |
388 | line = file.readline() | |
389 | if line == '': | |
390 | return | |
391 | ||
392 | # Parse the list of variations, which comes before the test | |
393 | # runs themselves. | |
394 | if line.startswith ('Schedule of variations:'): | |
395 | num_variations = self.parse_variations (filename, file) | |
396 | continue | |
397 | ||
398 | # Parse a testsuite run for one tool/variation combination. | |
399 | if line.startswith ('Running target '): | |
400 | name = line[len ('Running target '):-1] | |
401 | if not tool: | |
402 | self.fatal (filename, 'could not parse tool name') | |
403 | if name not in self.known_variations: | |
404 | self.fatal (filename, 'unknown target: ' + name) | |
405 | self.parse_run (filename, file, tool, | |
406 | tool.get_variation (name), | |
407 | num_variations) | |
408 | # If there is only one variation then there is no separate | |
409 | # summary for it. Record any following version output. | |
410 | if num_variations == 1: | |
411 | self.parse_final_summary (filename, file) | |
412 | continue | |
413 | ||
414 | # Parse the start line. In the case where several files are being | |
415 | # parsed, pick the one with the earliest time. | |
416 | match = self.test_run_re.match (line) | |
417 | if match: | |
418 | time = self.parse_time (match.group (2)) | |
419 | if not self.start_line or self.start_line[0] > time: | |
420 | self.start_line = (time, line) | |
421 | continue | |
422 | ||
423 | # Parse the form used for native testing. | |
424 | if line.startswith ('Native configuration is '): | |
425 | self.native_line = line | |
426 | continue | |
427 | ||
428 | # Parse the target triplet. | |
429 | if line.startswith ('Target is '): | |
430 | self.target_line = line | |
431 | continue | |
432 | ||
433 | # Parse the host triplet. | |
434 | if line.startswith ('Host is '): | |
435 | self.host_line = line | |
436 | continue | |
437 | ||
438 | # Parse the acats premable. | |
439 | if line == '\t\t=== acats configuration ===\n': | |
440 | self.parse_acats_run (filename, file) | |
441 | continue | |
442 | ||
443 | # Parse the tool name. | |
444 | match = self.tool_re.match (line) | |
445 | if match: | |
446 | tool = self.get_tool (match.group (1)) | |
447 | continue | |
448 | ||
449 | # Skip over the final summary (which we instead create from | |
450 | # individual runs) and parse the version output. | |
451 | if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n': | |
452 | if file.readline() != '\n': | |
453 | self.fatal (filename, 'expected blank line after summary') | |
454 | self.parse_final_summary (filename, file) | |
455 | continue | |
456 | ||
457 | # Parse the completion line. In the case where several files | |
458 | # are being parsed, pick the one with the latest time. | |
459 | match = self.completed_re.match (line) | |
460 | if match: | |
461 | time = self.parse_time (match.group (1)) | |
462 | if not self.end_line or self.end_line[0] < time: | |
463 | self.end_line = (time, line) | |
464 | continue | |
465 | ||
466 | # Sanity check to make sure that important text doesn't get | |
467 | # dropped accidentally. | |
468 | if strict and line.strip() != '': | |
469 | self.fatal (filename, 'unrecognised line: ' + line[:-1]) | |
470 | ||
471 | # Output a segment of text. | |
472 | def output_segment (self, segment): | |
473 | with safe_open (segment.filename) as file: | |
474 | file.seek (segment.start) | |
475 | for i in range (segment.lines): | |
476 | sys.stdout.write (file.readline()) | |
477 | ||
478 | # Output a summary giving the number of times each type of result has | |
479 | # been seen. | |
480 | def output_summary (self, tool, counts): | |
481 | for i in range (len (self.count_names)): | |
482 | name = self.count_names[i] | |
483 | # dejagnu only prints result types that were seen at least once, | |
484 | # but acats always prints a number of unexpected failures. | |
485 | if (counts[i] > 0 | |
486 | or (tool.name == 'acats' | |
487 | and name.startswith ('# of unexpected failures'))): | |
488 | sys.stdout.write ('%s%d\n' % (name, counts[i])) | |
489 | ||
490 | # Output unified .log or .sum information for a particular variation, | |
491 | # with a summary at the end. | |
492 | def output_variation (self, tool, variation): | |
493 | self.output_segment (variation.header) | |
494 | for harness in sorted (variation.harnesses.values(), | |
495 | key = attrgetter ('name')): | |
496 | sys.stdout.write ('Running ' + harness.name + ' ...\n') | |
497 | if self.do_sum: | |
498 | harness.results.sort() | |
499 | for (key, line) in harness.results: | |
500 | sys.stdout.write (line) | |
501 | else: | |
502 | # Rearrange the log segments into test order (but without | |
503 | # rearranging text within those segments). | |
504 | for key in sorted (harness.segments.keys()): | |
505 | self.output_segment (harness.segments[key]) | |
506 | for segment in harness.empty: | |
507 | self.output_segment (segment) | |
508 | if len (self.variations) > 1: | |
509 | sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for ' | |
510 | + variation.name + ' ===\n\n') | |
511 | self.output_summary (tool, variation.counts) | |
512 | ||
513 | # Output unified .log or .sum information for a particular tool, | |
514 | # with a summary at the end. | |
515 | def output_tool (self, tool): | |
516 | counts = self.zero_counts() | |
517 | if tool.name == 'acats': | |
518 | # acats doesn't use variations, so just output everything. | |
519 | # It also has a different approach to whitespace. | |
520 | sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n') | |
521 | for variation in tool.variations.values(): | |
522 | self.output_variation (tool, variation) | |
523 | self.accumulate_counts (counts, variation.counts) | |
524 | sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n') | |
525 | else: | |
526 | # Output the results in the usual dejagnu runtest format. | |
527 | sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n' | |
528 | 'Schedule of variations:\n') | |
529 | for name in self.variations: | |
530 | if name in tool.variations: | |
531 | sys.stdout.write (' ' + name + '\n') | |
532 | sys.stdout.write ('\n') | |
533 | for name in self.variations: | |
534 | if name in tool.variations: | |
535 | variation = tool.variations[name] | |
536 | sys.stdout.write ('Running target ' | |
537 | + variation.name + '\n') | |
538 | self.output_variation (tool, variation) | |
539 | self.accumulate_counts (counts, variation.counts) | |
540 | sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n') | |
541 | self.output_summary (tool, counts) | |
542 | ||
543 | def main (self): | |
544 | self.parse_cmdline() | |
545 | try: | |
546 | # Parse the input files. | |
547 | for filename in self.files: | |
548 | with safe_open (filename) as file: | |
549 | self.parse_file (filename, file) | |
550 | ||
551 | # Decide what to output. | |
552 | if len (self.variations) == 0: | |
553 | self.variations = sorted (self.known_variations) | |
554 | else: | |
555 | for name in self.variations: | |
556 | if name not in self.known_variations: | |
557 | self.fatal (None, 'no results for ' + name) | |
558 | if len (self.tools) == 0: | |
559 | self.tools = sorted (self.runs.keys()) | |
560 | ||
561 | # Output the header. | |
562 | if self.start_line: | |
563 | sys.stdout.write (self.start_line[1]) | |
564 | sys.stdout.write (self.native_line) | |
565 | sys.stdout.write (self.target_line) | |
566 | sys.stdout.write (self.host_line) | |
567 | sys.stdout.write (self.acats_premable) | |
568 | ||
569 | # Output the main body. | |
570 | for name in self.tools: | |
571 | if name not in self.runs: | |
572 | self.fatal (None, 'no results for ' + name) | |
573 | self.output_tool (self.runs[name]) | |
574 | ||
575 | # Output the footer. | |
576 | if len (self.acats_failures) > 0: | |
577 | sys.stdout.write ('*** FAILURES: ' | |
578 | + ' '.join (self.acats_failures) + '\n') | |
579 | sys.stdout.write (self.version_output) | |
580 | if self.end_line: | |
581 | sys.stdout.write (self.end_line[1]) | |
582 | except IOError as e: | |
583 | self.fatal (e.filename, e.strerror) | |
584 | ||
585 | Prog().main() |