| #!/usr/bin/python |
| # |
| # Copyright (C) 2014 Free Software Foundation, Inc. |
| # |
| # This script is free software; you can redistribute it and/or modify |
| # it under the terms of the GNU General Public License as published by |
| # the Free Software Foundation; either version 3, or (at your option) |
| # any later version. |
| |
| import sys |
| import getopt |
| import re |
| import io |
| from datetime import datetime |
| from operator import attrgetter |
| |
| # True if unrecognised lines should cause a fatal error. Might want to turn |
| # this on by default later. |
| strict = False |
| |
| # True if the order of .log segments should match the .sum file, false if |
| # they should keep the original order. |
| sort_logs = True |
| |
| # A version of open() that is safe against whatever binary output |
| # might be added to the log. |
| def safe_open (filename): |
| if sys.version_info >= (3, 0): |
| return open (filename, 'r', errors = 'surrogateescape') |
| return open (filename, 'r') |
| |
| # Force stdout to handle escape sequences from a safe_open file. |
| if sys.version_info >= (3, 0): |
| sys.stdout = io.TextIOWrapper (sys.stdout.buffer, |
| errors = 'surrogateescape') |
| |
| class Named: |
| def __init__ (self, name): |
| self.name = name |
| |
| class ToolRun (Named): |
| def __init__ (self, name): |
| Named.__init__ (self, name) |
| # The variations run for this tool, mapped by --target_board name. |
| self.variations = dict() |
| |
| # Return the VariationRun for variation NAME. |
| def get_variation (self, name): |
| if name not in self.variations: |
| self.variations[name] = VariationRun (name) |
| return self.variations[name] |
| |
| class VariationRun (Named): |
| def __init__ (self, name): |
| Named.__init__ (self, name) |
| # A segment of text before the harness runs start, describing which |
| # baseboard files were loaded for the target. |
| self.header = None |
| # The harnesses run for this variation, mapped by filename. |
| self.harnesses = dict() |
| # A list giving the number of times each type of result has |
| # been seen. |
| self.counts = [] |
| |
| # Return the HarnessRun for harness NAME. |
| def get_harness (self, name): |
| if name not in self.harnesses: |
| self.harnesses[name] = HarnessRun (name) |
| return self.harnesses[name] |
| |
| class HarnessRun (Named): |
| def __init__ (self, name): |
| Named.__init__ (self, name) |
| # Segments of text that make up the harness run, mapped by a test-based |
| # key that can be used to order them. |
| self.segments = dict() |
| # Segments of text that make up the harness run but which have |
| # no recognized test results. These are typically harnesses that |
| # are completely skipped for the target. |
| self.empty = [] |
| # A list of results. Each entry is a pair in which the first element |
| # is a unique sorting key and in which the second is the full |
| # PASS/FAIL line. |
| self.results = [] |
| |
| # Add a segment of text to the harness run. If the segment includes |
| # test results, KEY is an example of one of them, and can be used to |
| # combine the individual segments in order. If the segment has no |
| # test results (e.g. because the harness doesn't do anything for the |
| # current configuration) then KEY is None instead. In that case |
| # just collect the segments in the order that we see them. |
| def add_segment (self, key, segment): |
| if key: |
| assert key not in self.segments |
| self.segments[key] = segment |
| else: |
| self.empty.append (segment) |
| |
| class Segment: |
| def __init__ (self, filename, start): |
| self.filename = filename |
| self.start = start |
| self.lines = 0 |
| |
| class Prog: |
| def __init__ (self): |
| # The variations specified on the command line. |
| self.variations = [] |
| # The variations seen in the input files. |
| self.known_variations = set() |
| # The tools specified on the command line. |
| self.tools = [] |
| # Whether to create .sum rather than .log output. |
| self.do_sum = True |
| # Regexps used while parsing. |
| self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$') |
| self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$') |
| self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED' |
| r'|WARNING|ERROR|UNSUPPORTED|UNTESTED' |
| r'|KFAIL):\s*(.+)') |
| self.completed_re = re.compile (r'.* completed at (.*)') |
| # Pieces of text to write at the head of the output. |
| # start_line is a pair in which the first element is a datetime |
| # and in which the second is the associated 'Test Run By' line. |
| self.start_line = None |
| self.native_line = '' |
| self.target_line = '' |
| self.host_line = '' |
| self.acats_premable = '' |
| # Pieces of text to write at the end of the output. |
| # end_line is like start_line but for the 'runtest completed' line. |
| self.acats_failures = [] |
| self.version_output = '' |
| self.end_line = None |
| # Known summary types. |
| self.count_names = [ |
| '# of DejaGnu errors\t\t', |
| '# of expected passes\t\t', |
| '# of unexpected failures\t', |
| '# of unexpected successes\t', |
| '# of expected failures\t\t', |
| '# of unknown successes\t\t', |
| '# of known failures\t\t', |
| '# of untested testcases\t\t', |
| '# of unresolved testcases\t', |
| '# of unsupported tests\t\t' |
| ] |
| self.runs = dict() |
| |
| def usage (self): |
| name = sys.argv[0] |
| sys.stderr.write ('Usage: ' + name |
| + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ... |
| |
| tool The tool (e.g. g++, libffi) for which to create a |
| new test summary file. If not specified then output |
| is created for all tools. |
| variant-list One or more test variant names. If the list is |
| not specified then one is constructed from all |
| variants in the files for <tool>. |
| sum-file A test summary file with the format of those |
| created by runtest from DejaGnu. |
| If -L is used, merge *.log files instead of *.sum. In this |
| mode the exact order of lines may not be preserved, just different |
| Running *.exp chunks should be in correct order. |
| ''') |
| sys.exit (1) |
| |
| def fatal (self, what, string): |
| if not what: |
| what = sys.argv[0] |
| sys.stderr.write (what + ': ' + string + '\n') |
| sys.exit (1) |
| |
| # Parse the command-line arguments. |
| def parse_cmdline (self): |
| try: |
| (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L') |
| if len (self.files) == 0: |
| self.usage() |
| for (option, value) in options: |
| if option == '-l': |
| self.variations.append (value) |
| elif option == '-t': |
| self.tools.append (value) |
| else: |
| self.do_sum = False |
| except getopt.GetoptError as e: |
| self.fatal (None, e.msg) |
| |
| # Try to parse time string TIME, returning an arbitrary time on failure. |
| # Getting this right is just a nice-to-have so failures should be silent. |
| def parse_time (self, time): |
| try: |
| return datetime.strptime (time, '%c') |
| except ValueError: |
| return datetime.now() |
| |
| # Parse an integer and abort on failure. |
| def parse_int (self, filename, value): |
| try: |
| return int (value) |
| except ValueError: |
| self.fatal (filename, 'expected an integer, got: ' + value) |
| |
| # Return a list that represents no test results. |
| def zero_counts (self): |
| return [0 for x in self.count_names] |
| |
| # Return the ToolRun for tool NAME. |
| def get_tool (self, name): |
| if name not in self.runs: |
| self.runs[name] = ToolRun (name) |
| return self.runs[name] |
| |
| # Add the result counts in list FROMC to TOC. |
| def accumulate_counts (self, toc, fromc): |
| for i in range (len (self.count_names)): |
| toc[i] += fromc[i] |
| |
| # Parse the list of variations after 'Schedule of variations:'. |
| # Return the number seen. |
| def parse_variations (self, filename, file): |
| num_variations = 0 |
| while True: |
| line = file.readline() |
| if line == '': |
| self.fatal (filename, 'could not parse variation list') |
| if line == '\n': |
| break |
| self.known_variations.add (line.strip()) |
| num_variations += 1 |
| return num_variations |
| |
| # Parse from the first line after 'Running target ...' to the end |
| # of the run's summary. |
| def parse_run (self, filename, file, tool, variation, num_variations): |
| header = None |
| harness = None |
| segment = None |
| final_using = 0 |
| has_warning = 0 |
| |
| # If this is the first run for this variation, add any text before |
| # the first harness to the header. |
| if not variation.header: |
| segment = Segment (filename, file.tell()) |
| variation.header = segment |
| |
| # Parse the rest of the summary (the '# of ' lines). |
| if len (variation.counts) == 0: |
| variation.counts = self.zero_counts() |
| |
| # Parse up until the first line of the summary. |
| if num_variations == 1: |
| end = '\t\t=== ' + tool.name + ' Summary ===\n' |
| else: |
| end = ('\t\t=== ' + tool.name + ' Summary for ' |
| + variation.name + ' ===\n') |
| while True: |
| line = file.readline() |
| if line == '': |
| self.fatal (filename, 'no recognised summary line') |
| if line == end: |
| break |
| |
| # Look for the start of a new harness. |
| if line.startswith ('Running ') and line.endswith (' ...\n'): |
| # Close off the current harness segment, if any. |
| if harness: |
| segment.lines -= final_using |
| harness.add_segment (first_key, segment) |
| name = line[len ('Running '):-len(' ...\n')] |
| harness = variation.get_harness (name) |
| segment = Segment (filename, file.tell()) |
| first_key = None |
| final_using = 0 |
| continue |
| |
| # Record test results. Associate the first test result with |
| # the harness segment, so that if a run for a particular harness |
| # has been split up, we can reassemble the individual segments |
| # in a sensible order. |
| # |
| # dejagnu sometimes issues warnings about the testing environment |
| # before running any tests. Treat them as part of the header |
| # rather than as a test result. |
| match = self.result_re.match (line) |
| if match and (harness or not line.startswith ('WARNING:')): |
| if not harness: |
| self.fatal (filename, 'saw test result before harness name') |
| name = match.group (2) |
| # Ugly hack to get the right order for gfortran. |
| if name.startswith ('gfortran.dg/g77/'): |
| name = 'h' + name |
| # If we have a time out warning, make sure it appears |
| # before the following testcase diagnostic: we insert |
| # the testname before 'program' so that sort faces a |
| # list of testnames. |
| if line.startswith ('WARNING: program timed out'): |
| has_warning = 1 |
| else: |
| if has_warning == 1: |
| key = (name, len (harness.results)) |
| myline = 'WARNING: %s program timed out.\n' % name |
| harness.results.append ((key, myline)) |
| has_warning = 0 |
| key = (name, len (harness.results)) |
| harness.results.append ((key, line)) |
| if not first_key and sort_logs: |
| first_key = key |
| if line.startswith ('ERROR: (DejaGnu)'): |
| for i in range (len (self.count_names)): |
| if 'DejaGnu errors' in self.count_names[i]: |
| variation.counts[i] += 1 |
| break |
| |
| # 'Using ...' lines are only interesting in a header. Splitting |
| # the test up into parallel runs leads to more 'Using ...' lines |
| # than there would be in a single log. |
| if line.startswith ('Using '): |
| final_using += 1 |
| else: |
| final_using = 0 |
| |
| # Add other text to the current segment, if any. |
| if segment: |
| segment.lines += 1 |
| |
| # Close off the final harness segment, if any. |
| if harness: |
| segment.lines -= final_using |
| harness.add_segment (first_key, segment) |
| |
| while True: |
| before = file.tell() |
| line = file.readline() |
| if line == '': |
| break |
| if line == '\n': |
| continue |
| if not line.startswith ('# '): |
| file.seek (before) |
| break |
| found = False |
| for i in range (len (self.count_names)): |
| if line.startswith (self.count_names[i]): |
| count = line[len (self.count_names[i]):-1].strip() |
| variation.counts[i] += self.parse_int (filename, count) |
| found = True |
| break |
| if not found: |
| self.fatal (filename, 'unknown test result: ' + line[:-1]) |
| |
| # Parse an acats run, which uses a different format from dejagnu. |
| # We have just skipped over '=== acats configuration ==='. |
| def parse_acats_run (self, filename, file): |
| # Parse the preamble, which describes the configuration and logs |
| # the creation of support files. |
| record = (self.acats_premable == '') |
| if record: |
| self.acats_premable = '\t\t=== acats configuration ===\n' |
| while True: |
| line = file.readline() |
| if line == '': |
| self.fatal (filename, 'could not parse acats preamble') |
| if line == '\t\t=== acats tests ===\n': |
| break |
| if record: |
| self.acats_premable += line |
| |
| # Parse the test results themselves, using a dummy variation name. |
| tool = self.get_tool ('acats') |
| variation = tool.get_variation ('none') |
| self.parse_run (filename, file, tool, variation, 1) |
| |
| # Parse the failure list. |
| while True: |
| before = file.tell() |
| line = file.readline() |
| if line.startswith ('*** FAILURES: '): |
| self.acats_failures.append (line[len ('*** FAILURES: '):-1]) |
| continue |
| file.seek (before) |
| break |
| |
| # Parse the final summary at the end of a log in order to capture |
| # the version output that follows it. |
| def parse_final_summary (self, filename, file): |
| record = (self.version_output == '') |
| while True: |
| line = file.readline() |
| if line == '': |
| break |
| if line.startswith ('# of '): |
| continue |
| if record: |
| self.version_output += line |
| if line == '\n': |
| break |
| |
| # Parse a .log or .sum file. |
| def parse_file (self, filename, file): |
| tool = None |
| target = None |
| num_variations = 1 |
| while True: |
| line = file.readline() |
| if line == '': |
| return |
| |
| # Parse the list of variations, which comes before the test |
| # runs themselves. |
| if line.startswith ('Schedule of variations:'): |
| num_variations = self.parse_variations (filename, file) |
| continue |
| |
| # Parse a testsuite run for one tool/variation combination. |
| if line.startswith ('Running target '): |
| name = line[len ('Running target '):-1] |
| if not tool: |
| self.fatal (filename, 'could not parse tool name') |
| if name not in self.known_variations: |
| self.fatal (filename, 'unknown target: ' + name) |
| self.parse_run (filename, file, tool, |
| tool.get_variation (name), |
| num_variations) |
| # If there is only one variation then there is no separate |
| # summary for it. Record any following version output. |
| if num_variations == 1: |
| self.parse_final_summary (filename, file) |
| continue |
| |
| # Parse the start line. In the case where several files are being |
| # parsed, pick the one with the earliest time. |
| match = self.test_run_re.match (line) |
| if match: |
| time = self.parse_time (match.group (2)) |
| if not self.start_line or self.start_line[0] > time: |
| self.start_line = (time, line) |
| continue |
| |
| # Parse the form used for native testing. |
| if line.startswith ('Native configuration is '): |
| self.native_line = line |
| continue |
| |
| # Parse the target triplet. |
| if line.startswith ('Target is '): |
| self.target_line = line |
| continue |
| |
| # Parse the host triplet. |
| if line.startswith ('Host is '): |
| self.host_line = line |
| continue |
| |
| # Parse the acats premable. |
| if line == '\t\t=== acats configuration ===\n': |
| self.parse_acats_run (filename, file) |
| continue |
| |
| # Parse the tool name. |
| match = self.tool_re.match (line) |
| if match: |
| tool = self.get_tool (match.group (1)) |
| continue |
| |
| # Skip over the final summary (which we instead create from |
| # individual runs) and parse the version output. |
| if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n': |
| if file.readline() != '\n': |
| self.fatal (filename, 'expected blank line after summary') |
| self.parse_final_summary (filename, file) |
| continue |
| |
| # Parse the completion line. In the case where several files |
| # are being parsed, pick the one with the latest time. |
| match = self.completed_re.match (line) |
| if match: |
| time = self.parse_time (match.group (1)) |
| if not self.end_line or self.end_line[0] < time: |
| self.end_line = (time, line) |
| continue |
| |
| # Sanity check to make sure that important text doesn't get |
| # dropped accidentally. |
| if strict and line.strip() != '': |
| self.fatal (filename, 'unrecognised line: ' + line[:-1]) |
| |
| # Output a segment of text. |
| def output_segment (self, segment): |
| with safe_open (segment.filename) as file: |
| file.seek (segment.start) |
| for i in range (segment.lines): |
| sys.stdout.write (file.readline()) |
| |
| # Output a summary giving the number of times each type of result has |
| # been seen. |
| def output_summary (self, tool, counts): |
| for i in range (len (self.count_names)): |
| name = self.count_names[i] |
| # dejagnu only prints result types that were seen at least once, |
| # but acats always prints a number of unexpected failures. |
| if (counts[i] > 0 |
| or (tool.name == 'acats' |
| and name.startswith ('# of unexpected failures'))): |
| sys.stdout.write ('%s%d\n' % (name, counts[i])) |
| |
| # Output unified .log or .sum information for a particular variation, |
| # with a summary at the end. |
| def output_variation (self, tool, variation): |
| self.output_segment (variation.header) |
| for harness in sorted (variation.harnesses.values(), |
| key = attrgetter ('name')): |
| sys.stdout.write ('Running ' + harness.name + ' ...\n') |
| if self.do_sum: |
| harness.results.sort() |
| for (key, line) in harness.results: |
| sys.stdout.write (line) |
| else: |
| # Rearrange the log segments into test order (but without |
| # rearranging text within those segments). |
| for key in sorted (harness.segments.keys()): |
| self.output_segment (harness.segments[key]) |
| for segment in harness.empty: |
| self.output_segment (segment) |
| if len (self.variations) > 1: |
| sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for ' |
| + variation.name + ' ===\n\n') |
| self.output_summary (tool, variation.counts) |
| |
| # Output unified .log or .sum information for a particular tool, |
| # with a summary at the end. |
| def output_tool (self, tool): |
| counts = self.zero_counts() |
| if tool.name == 'acats': |
| # acats doesn't use variations, so just output everything. |
| # It also has a different approach to whitespace. |
| sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n') |
| for variation in tool.variations.values(): |
| self.output_variation (tool, variation) |
| self.accumulate_counts (counts, variation.counts) |
| sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n') |
| else: |
| # Output the results in the usual dejagnu runtest format. |
| sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n' |
| 'Schedule of variations:\n') |
| for name in self.variations: |
| if name in tool.variations: |
| sys.stdout.write (' ' + name + '\n') |
| sys.stdout.write ('\n') |
| for name in self.variations: |
| if name in tool.variations: |
| variation = tool.variations[name] |
| sys.stdout.write ('Running target ' |
| + variation.name + '\n') |
| self.output_variation (tool, variation) |
| self.accumulate_counts (counts, variation.counts) |
| sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n') |
| self.output_summary (tool, counts) |
| |
| def main (self): |
| self.parse_cmdline() |
| try: |
| # Parse the input files. |
| for filename in self.files: |
| with safe_open (filename) as file: |
| self.parse_file (filename, file) |
| |
| # Decide what to output. |
| if len (self.variations) == 0: |
| self.variations = sorted (self.known_variations) |
| else: |
| for name in self.variations: |
| if name not in self.known_variations: |
| self.fatal (None, 'no results for ' + name) |
| if len (self.tools) == 0: |
| self.tools = sorted (self.runs.keys()) |
| |
| # Output the header. |
| if self.start_line: |
| sys.stdout.write (self.start_line[1]) |
| sys.stdout.write (self.native_line) |
| sys.stdout.write (self.target_line) |
| sys.stdout.write (self.host_line) |
| sys.stdout.write (self.acats_premable) |
| |
| # Output the main body. |
| for name in self.tools: |
| if name not in self.runs: |
| self.fatal (None, 'no results for ' + name) |
| self.output_tool (self.runs[name]) |
| |
| # Output the footer. |
| if len (self.acats_failures) > 0: |
| sys.stdout.write ('*** FAILURES: ' |
| + ' '.join (self.acats_failures) + '\n') |
| sys.stdout.write (self.version_output) |
| if self.end_line: |
| sys.stdout.write (self.end_line[1]) |
| except IOError as e: |
| self.fatal (e.filename, e.strerror) |
| |
| Prog().main() |
