contrib/dg-extract-results.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (C) 2014 Free Software Foundation, Inc.
   4 #
   5 # This script is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3, or (at your option)
   8 # any later version.
   9
  10 import sys
  11 import getopt
  12 import re
  13 import io
  14 from datetime import datetime
  15 from operator import attrgetter
  16
  17 # True if unrecognised lines should cause a fatal error.  Might want to turn
  18 # this on by default later.
  19 strict = False
  20
  21 # True if the order of .log segments should match the .sum file, false if
  22 # they should keep the original order.
  23 sort_logs = True
  24
  25 # A version of open() that is safe against whatever binary output
  26 # might be added to the log.
  27 def safe_open (filename):
  28     if sys.version_info >= (3, 0):
  29         return open (filename, 'r', errors = 'surrogateescape')
  30     return open (filename, 'r')
  31
  32 # Force stdout to handle escape sequences from a safe_open file.
  33 if sys.version_info >= (3, 0):
  34     sys.stdout = io.TextIOWrapper (sys.stdout.buffer,
  35                                    errors = 'surrogateescape')
  36
  37 class Named:
  38     def __init__ (self, name):
  39         self.name = name
  40
  41 class ToolRun (Named):
  42     def __init__ (self, name):
  43         Named.__init__ (self, name)
  44         # The variations run for this tool, mapped by --target_board name.
  45         self.variations = dict()
  46
  47     # Return the VariationRun for variation NAME.
  48     def get_variation (self, name):
  49         if name not in self.variations:
  50             self.variations[name] = VariationRun (name)
  51         return self.variations[name]
  52
  53 class VariationRun (Named):
  54     def __init__ (self, name):
  55         Named.__init__ (self, name)
  56         # A segment of text before the harness runs start, describing which
  57         # baseboard files were loaded for the target.
  58         self.header = None
  59         # The harnesses run for this variation, mapped by filename.
  60         self.harnesses = dict()
  61         # A list giving the number of times each type of result has
  62         # been seen.
  63         self.counts = []
  64
  65     # Return the HarnessRun for harness NAME.
  66     def get_harness (self, name):
  67         if name not in self.harnesses:
  68             self.harnesses[name] = HarnessRun (name)
  69         return self.harnesses[name]
  70
  71 class HarnessRun (Named):
  72     def __init__ (self, name):
  73         Named.__init__ (self, name)
  74         # Segments of text that make up the harness run, mapped by a test-based
  75         # key that can be used to order them.
  76         self.segments = dict()
  77         # Segments of text that make up the harness run but which have
  78         # no recognized test results.  These are typically harnesses that
  79         # are completely skipped for the target.
  80         self.empty = []
  81         # A list of results.  Each entry is a pair in which the first element
  82         # is a unique sorting key and in which the second is the full
  83         # PASS/FAIL line.
  84         self.results = []
  85
  86     # Add a segment of text to the harness run.  If the segment includes
  87     # test results, KEY is an example of one of them, and can be used to
  88     # combine the individual segments in order.  If the segment has no
  89     # test results (e.g. because the harness doesn't do anything for the
  90     # current configuration) then KEY is None instead.  In that case
  91     # just collect the segments in the order that we see them.
  92     def add_segment (self, key, segment):
  93         if key:
  94             assert key not in self.segments
  95             self.segments[key] = segment
  96         else:
  97             self.empty.append (segment)
  98
  99 class Segment:
 100     def __init__ (self, filename, start):
 101         self.filename = filename
 102         self.start = start
 103         self.lines = 0
 104
 105 class Prog:
 106     def __init__ (self):
 107         # The variations specified on the command line.
 108         self.variations = []
 109         # The variations seen in the input files.
 110         self.known_variations = set()
 111         # The tools specified on the command line.
 112         self.tools = []
 113         # Whether to create .sum rather than .log output.
 114         self.do_sum = True
 115         # Regexps used while parsing.
 116         self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$')
 117         self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$')
 118         self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
 119                                      r'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
 120                                      r'|KFAIL|KPASS):\s*(.+)')
 121         self.completed_re = re.compile (r'.* completed at (.*)')
 122         # Pieces of text to write at the head of the output.
 123         # start_line is a pair in which the first element is a datetime
 124         # and in which the second is the associated 'Test Run By' line.
 125         self.start_line = None
 126         self.native_line = ''
 127         self.target_line = ''
 128         self.host_line = ''
 129         self.acats_premable = ''
 130         # Pieces of text to write at the end of the output.
 131         # end_line is like start_line but for the 'runtest completed' line.
 132         self.acats_failures = []
 133         self.version_output = ''
 134         self.end_line = None
 135         # Known summary types.
 136         self.count_names = [
 137             '# of DejaGnu errors\t\t',
 138             '# of expected passes\t\t',
 139             '# of unexpected failures\t',
 140             '# of unexpected successes\t',
 141             '# of expected failures\t\t',
 142             '# of unknown successes\t\t',
 143             '# of known failures\t\t',
 144             '# of untested testcases\t\t',
 145             '# of unresolved testcases\t',
 146             '# of unsupported tests\t\t'
 147         ]
 148         self.runs = dict()
 149
 150     def usage (self):
 151         name = sys.argv[0]
 152         sys.stderr.write ('Usage: ' + name
 153                           + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
 154
 155     tool           The tool (e.g. g++, libffi) for which to create a
 156                    new test summary file.  If not specified then output
 157                    is created for all tools.
 158     variant-list   One or more test variant names.  If the list is
 159                    not specified then one is constructed from all
 160                    variants in the files for <tool>.
 161     sum-file       A test summary file with the format of those
 162                    created by runtest from DejaGnu.
 163     If -L is used, merge *.log files instead of *.sum.  In this
 164     mode the exact order of lines may not be preserved, just different
 165     Running *.exp chunks should be in correct order.
 166 ''')
 167         sys.exit (1)
 168
 169     def fatal (self, what, string):
 170         if not what:
 171             what = sys.argv[0]
 172         sys.stderr.write (what + ': ' + string + '\n')
 173         sys.exit (1)
 174
 175     # Parse the command-line arguments.
 176     def parse_cmdline (self):
 177         try:
 178             (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L')
 179             if len (self.files) == 0:
 180                 self.usage()
 181             for (option, value) in options:
 182                 if option == '-l':
 183                     self.variations.append (value)
 184                 elif option == '-t':
 185                     self.tools.append (value)
 186                 else:
 187                     self.do_sum = False
 188         except getopt.GetoptError as e:
 189             self.fatal (None, e.msg)
 190
 191     # Try to parse time string TIME, returning an arbitrary time on failure.
 192     # Getting this right is just a nice-to-have so failures should be silent.
 193     def parse_time (self, time):
 194         try:
 195             return datetime.strptime (time, '%c')
 196         except ValueError:
 197             return datetime.now()
 198
 199     # Parse an integer and abort on failure.
 200     def parse_int (self, filename, value):
 201         try:
 202             return int (value)
 203         except ValueError:
 204             self.fatal (filename, 'expected an integer, got: ' + value)
 205
 206     # Return a list that represents no test results.
 207     def zero_counts (self):
 208         return [0 for x in self.count_names]
 209
 210     # Return the ToolRun for tool NAME.
 211     def get_tool (self, name):
 212         if name not in self.runs:
 213             self.runs[name] = ToolRun (name)
 214         return self.runs[name]
 215
 216     # Add the result counts in list FROMC to TOC.
 217     def accumulate_counts (self, toc, fromc):
 218         for i in range (len (self.count_names)):
 219             toc[i] += fromc[i]
 220
 221     # Parse the list of variations after 'Schedule of variations:'.
 222     # Return the number seen.
 223     def parse_variations (self, filename, file):
 224         num_variations = 0
 225         while True:
 226             line = file.readline()
 227             if line == '':
 228                 self.fatal (filename, 'could not parse variation list')
 229             if line == '\n':
 230                 break
 231             self.known_variations.add (line.strip())
 232             num_variations += 1
 233         return num_variations
 234
 235     # Parse from the first line after 'Running target ...' to the end
 236     # of the run's summary.
 237     def parse_run (self, filename, file, tool, variation, num_variations):
 238         header = None
 239         harness = None
 240         segment = None
 241         final_using = 0
 242         has_warning = 0
 243
 244         # If this is the first run for this variation, add any text before
 245         # the first harness to the header.
 246         if not variation.header:
 247             segment = Segment (filename, file.tell())
 248             variation.header = segment
 249
 250         # Parse the rest of the summary (the '# of ' lines).
 251         if len (variation.counts) == 0:
 252             variation.counts = self.zero_counts()
 253
 254         # Parse up until the first line of the summary.
 255         if num_variations == 1:
 256             end = '\t\t=== ' + tool.name + ' Summary ===\n'
 257         else:
 258             end = ('\t\t=== ' + tool.name + ' Summary for '
 259                    + variation.name + ' ===\n')
 260         while True:
 261             line = file.readline()
 262             if line == '':
 263                 self.fatal (filename, 'no recognised summary line')
 264             if line == end:
 265                 break
 266
 267             # Look for the start of a new harness.
 268             if line.startswith ('Running ') and line.endswith (' ...\n'):
 269                 # Close off the current harness segment, if any.
 270                 if harness:
 271                     segment.lines -= final_using
 272                     harness.add_segment (first_key, segment)
 273                 name = line[len ('Running '):-len(' ...\n')]
 274                 harness = variation.get_harness (name)
 275                 segment = Segment (filename, file.tell())
 276                 first_key = None
 277                 final_using = 0
 278                 continue
 279
 280             # Record test results.  Associate the first test result with
 281             # the harness segment, so that if a run for a particular harness
 282             # has been split up, we can reassemble the individual segments
 283             # in a sensible order.
 284             #
 285             # dejagnu sometimes issues warnings about the testing environment
 286             # before running any tests.  Treat them as part of the header
 287             # rather than as a test result.
 288             match = self.result_re.match (line)
 289             if match and (harness or not line.startswith ('WARNING:')):
 290                 if not harness:
 291                     self.fatal (filename, 'saw test result before harness name')
 292                 name = match.group (2)
 293                 # Ugly hack to get the right order for gfortran.
 294                 if name.startswith ('gfortran.dg/g77/'):
 295                     name = 'h' + name
 296                 # If we have a time out warning, make sure it appears
 297                 # before the following testcase diagnostic: we insert
 298                 # the testname before 'program' so that sort faces a
 299                 # list of testnames.
 300                 if line.startswith ('WARNING: program timed out'):
 301                   has_warning = 1
 302                 else:
 303                   if has_warning == 1:
 304                       key = (name, len (harness.results))
 305                       myline = 'WARNING: %s program timed out.\n' % name
 306                       harness.results.append ((key, myline))
 307                       has_warning = 0
 308                   key = (name, len (harness.results))
 309                   harness.results.append ((key, line))
 310                   if not first_key and sort_logs:
 311                       first_key = key
 312                 if line.startswith ('ERROR: (DejaGnu)'):
 313                     for i in range (len (self.count_names)):
 314                         if 'DejaGnu errors' in self.count_names[i]:
 315                             variation.counts[i] += 1
 316                             break
 317
 318             # 'Using ...' lines are only interesting in a header.  Splitting
 319             # the test up into parallel runs leads to more 'Using ...' lines
 320             # than there would be in a single log.
 321             if line.startswith ('Using '):
 322                 final_using += 1
 323             else:
 324                 final_using = 0
 325
 326             # Add other text to the current segment, if any.
 327             if segment:
 328                 segment.lines += 1
 329
 330         # Close off the final harness segment, if any.
 331         if harness:
 332             segment.lines -= final_using
 333             harness.add_segment (first_key, segment)
 334
 335         while True:
 336             before = file.tell()
 337             line = file.readline()
 338             if line == '':
 339                 break
 340             if line == '\n':
 341                 continue
 342             if not line.startswith ('# '):
 343                 file.seek (before)
 344                 break
 345             found = False
 346             for i in range (len (self.count_names)):
 347                 if line.startswith (self.count_names[i]):
 348                     count = line[len (self.count_names[i]):-1].strip()
 349                     variation.counts[i] += self.parse_int (filename, count)
 350                     found = True
 351                     break
 352             if not found:
 353                 self.fatal (filename, 'unknown test result: ' + line[:-1])
 354
 355     # Parse an acats run, which uses a different format from dejagnu.
 356     # We have just skipped over '=== acats configuration ==='.
 357     def parse_acats_run (self, filename, file):
 358         # Parse the preamble, which describes the configuration and logs
 359         # the creation of support files.
 360         record = (self.acats_premable == '')
 361         if record:
 362             self.acats_premable = '\t\t=== acats configuration ===\n'
 363         while True:
 364             line = file.readline()
 365             if line == '':
 366                 self.fatal (filename, 'could not parse acats preamble')
 367             if line == '\t\t=== acats tests ===\n':
 368                 break
 369             if record:
 370                 self.acats_premable += line
 371
 372         # Parse the test results themselves, using a dummy variation name.
 373         tool = self.get_tool ('acats')
 374         variation = tool.get_variation ('none')
 375         self.parse_run (filename, file, tool, variation, 1)
 376
 377         # Parse the failure list.
 378         while True:
 379             before = file.tell()
 380             line = file.readline()
 381             if line.startswith ('*** FAILURES: '):
 382                 self.acats_failures.append (line[len ('*** FAILURES: '):-1])
 383                 continue
 384             file.seek (before)
 385             break
 386
 387     # Parse the final summary at the end of a log in order to capture
 388     # the version output that follows it.
 389     def parse_final_summary (self, filename, file):
 390         record = (self.version_output == '')
 391         while True:
 392             line = file.readline()
 393             if line == '':
 394                 break
 395             if line.startswith ('# of '):
 396                 continue
 397             if record:
 398                 self.version_output += line
 399             if line == '\n':
 400                 break
 401
 402     # Parse a .log or .sum file.
 403     def parse_file (self, filename, file):
 404         tool = None
 405         target = None
 406         num_variations = 1
 407         while True:
 408             line = file.readline()
 409             if line == '':
 410                 return
 411
 412             # Parse the list of variations, which comes before the test
 413             # runs themselves.
 414             if line.startswith ('Schedule of variations:'):
 415                 num_variations = self.parse_variations (filename, file)
 416                 continue
 417
 418             # Parse a testsuite run for one tool/variation combination.
 419             if line.startswith ('Running target '):
 420                 name = line[len ('Running target '):-1]
 421                 if not tool:
 422                     self.fatal (filename, 'could not parse tool name')
 423                 if name not in self.known_variations:
 424                     self.fatal (filename, 'unknown target: ' + name)
 425                 self.parse_run (filename, file, tool,
 426                                 tool.get_variation (name),
 427                                 num_variations)
 428                 # If there is only one variation then there is no separate
 429                 # summary for it.  Record any following version output.
 430                 if num_variations == 1:
 431                     self.parse_final_summary (filename, file)
 432                 continue
 433
 434             # Parse the start line.  In the case where several files are being
 435             # parsed, pick the one with the earliest time.
 436             match = self.test_run_re.match (line)
 437             if match:
 438                 time = self.parse_time (match.group (2))
 439                 if not self.start_line or self.start_line[0] > time:
 440                     self.start_line = (time, line)
 441                 continue
 442
 443             # Parse the form used for native testing.
 444             if line.startswith ('Native configuration is '):
 445                 self.native_line = line
 446                 continue
 447
 448             # Parse the target triplet.
 449             if line.startswith ('Target is '):
 450                 self.target_line = line
 451                 continue
 452
 453             # Parse the host triplet.
 454             if line.startswith ('Host   is '):
 455                 self.host_line = line
 456                 continue
 457
 458             # Parse the acats premable.
 459             if line == '\t\t=== acats configuration ===\n':
 460                 self.parse_acats_run (filename, file)
 461                 continue
 462
 463             # Parse the tool name.
 464             match = self.tool_re.match (line)
 465             if match:
 466                 tool = self.get_tool (match.group (1))
 467                 continue
 468
 469             # Skip over the final summary (which we instead create from
 470             # individual runs) and parse the version output.
 471             if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n':
 472                 if file.readline() != '\n':
 473                     self.fatal (filename, 'expected blank line after summary')
 474                 self.parse_final_summary (filename, file)
 475                 continue
 476
 477             # Parse the completion line.  In the case where several files
 478             # are being parsed, pick the one with the latest time.
 479             match = self.completed_re.match (line)
 480             if match:
 481                 time = self.parse_time (match.group (1))
 482                 if not self.end_line or self.end_line[0] < time:
 483                     self.end_line = (time, line)
 484                 continue
 485
 486             # Sanity check to make sure that important text doesn't get
 487             # dropped accidentally.
 488             if strict and line.strip() != '':
 489                 self.fatal (filename, 'unrecognised line: ' + line[:-1])
 490
 491     # Output a segment of text.
 492     def output_segment (self, segment):
 493         with safe_open (segment.filename) as file:
 494             file.seek (segment.start)
 495             for i in range (segment.lines):
 496                 sys.stdout.write (file.readline())
 497
 498     # Output a summary giving the number of times each type of result has
 499     # been seen.
 500     def output_summary (self, tool, counts):
 501         for i in range (len (self.count_names)):
 502             name = self.count_names[i]
 503             # dejagnu only prints result types that were seen at least once,
 504             # but acats always prints a number of unexpected failures.
 505             if (counts[i] > 0
 506                 or (tool.name == 'acats'
 507                     and name.startswith ('# of unexpected failures'))):
 508                 sys.stdout.write ('%s%d\n' % (name, counts[i]))
 509
 510     # Output unified .log or .sum information for a particular variation,
 511     # with a summary at the end.
 512     def output_variation (self, tool, variation):
 513         self.output_segment (variation.header)
 514         for harness in sorted (variation.harnesses.values(),
 515                                key = attrgetter ('name')):
 516             sys.stdout.write ('Running ' + harness.name + ' ...\n')
 517             if self.do_sum:
 518                 harness.results.sort()
 519                 for (key, line) in harness.results:
 520                     sys.stdout.write (line)
 521             else:
 522                 # Rearrange the log segments into test order (but without
 523                 # rearranging text within those segments).
 524                 for key in sorted (harness.segments.keys()):
 525                     self.output_segment (harness.segments[key])
 526                 for segment in harness.empty:
 527                     self.output_segment (segment)
 528         if len (self.variations) > 1:
 529             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for '
 530                               + variation.name + ' ===\n\n')
 531             self.output_summary (tool, variation.counts)
 532
 533     # Output unified .log or .sum information for a particular tool,
 534     # with a summary at the end.
 535     def output_tool (self, tool):
 536         counts = self.zero_counts()
 537         if tool.name == 'acats':
 538             # acats doesn't use variations, so just output everything.
 539             # It also has a different approach to whitespace.
 540             sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n')
 541             for variation in tool.variations.values():
 542                 self.output_variation (tool, variation)
 543                 self.accumulate_counts (counts, variation.counts)
 544             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n')
 545         else:
 546             # Output the results in the usual dejagnu runtest format.
 547             sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n'
 548                               'Schedule of variations:\n')
 549             for name in self.variations:
 550                 if name in tool.variations:
 551                     sys.stdout.write ('    ' + name + '\n')
 552             sys.stdout.write ('\n')
 553             for name in self.variations:
 554                 if name in tool.variations:
 555                     variation = tool.variations[name]
 556                     sys.stdout.write ('Running target '
 557                                       + variation.name + '\n')
 558                     self.output_variation (tool, variation)
 559                     self.accumulate_counts (counts, variation.counts)
 560             sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n')
 561         self.output_summary (tool, counts)
 562
 563     def main (self):
 564         self.parse_cmdline()
 565         try:
 566             # Parse the input files.
 567             for filename in self.files:
 568                 with safe_open (filename) as file:
 569                     self.parse_file (filename, file)
 570
 571             # Decide what to output.
 572             if len (self.variations) == 0:
 573                 self.variations = sorted (self.known_variations)
 574             else:
 575                 for name in self.variations:
 576                     if name not in self.known_variations:
 577                         self.fatal (None, 'no results for ' + name)
 578             if len (self.tools) == 0:
 579                 self.tools = sorted (self.runs.keys())
 580
 581             # Output the header.
 582             if self.start_line:
 583                 sys.stdout.write (self.start_line[1])
 584             sys.stdout.write (self.native_line)
 585             sys.stdout.write (self.target_line)
 586             sys.stdout.write (self.host_line)
 587             sys.stdout.write (self.acats_premable)
 588
 589             # Output the main body.
 590             for name in self.tools:
 591                 if name not in self.runs:
 592                     self.fatal (None, 'no results for ' + name)
 593                 self.output_tool (self.runs[name])
 594
 595             # Output the footer.
 596             if len (self.acats_failures) > 0:
 597                 sys.stdout.write ('*** FAILURES: '
 598                                   + ' '.join (self.acats_failures) + '\n')
 599             sys.stdout.write (self.version_output)
 600             if self.end_line:
 601                 sys.stdout.write (self.end_line[1])
 602         except IOError as e:
 603             self.fatal (e.filename, e.strerror)
 604
 605 Prog().main()