report_utils.py

# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Generate XML and HTML test reports."""

from __future__ import print_function

try:
  from mako.runtime import Context
  from mako.template import Template
  from mako import exceptions
except (ImportError):
  pass  # Mako not installed but it is ok.
import datetime
import os
import string
import xml.etree.cElementTree as ET
import six


def _filter_msg(msg, output_format):
  """Filters out nonprintable and illegal characters from the message."""
  if output_format in ['XML', 'HTML']:
    # keep whitespaces but remove formfeed and vertical tab characters
    # that make XML report unparseable.
    filtered_msg = filter(
        lambda x: x in string.printable and x != '\f' and x != '\v',
        msg.decode('UTF-8', 'ignore'))
    if output_format == 'HTML':
      filtered_msg = filtered_msg.replace('"', '"')
    return filtered_msg
  else:
    return msg


def new_junit_xml_tree():
  return ET.ElementTree(ET.Element('testsuites'))

def render_junit_xml_report(resultset, report_file, suite_package='grpc',
                            suite_name='tests'):
  """Generate JUnit-like XML report."""
  tree = new_junit_xml_tree()
  append_junit_xml_results(tree, resultset, suite_package, suite_name, '1')
  create_xml_report_file(tree, report_file)

def create_xml_report_file(tree, report_file):
  """Generate JUnit-like report file from xml tree ."""
  # ensure the report directory exists
  report_dir = os.path.dirname(os.path.abspath(report_file))
  if not os.path.exists(report_dir):
    os.makedirs(report_dir)
  tree.write(report_file, encoding='UTF-8')

def append_junit_xml_results(tree, resultset, suite_package, suite_name, id):
  """Append a JUnit-like XML report tree with test results as a new suite."""
  testsuite = ET.SubElement(tree.getroot(), 'testsuite',
                            id=id, package=suite_package, name=suite_name,
                            timestamp=datetime.datetime.now().isoformat())
  failure_count  = 0
  error_count = 0
  for shortname, results in six.iteritems(resultset):
    for result in results:
      xml_test = ET.SubElement(testsuite, 'testcase', name=shortname)
      if result.elapsed_time:
        xml_test.set('time', str(result.elapsed_time))
      filtered_msg =  _filter_msg(result.message, 'XML')
      if result.state == 'FAILED':
        ET.SubElement(xml_test, 'failure', message='Failure').text = filtered_msg
        failure_count += 1
      elif result.state == 'TIMEOUT':
        ET.SubElement(xml_test, 'error', message='Timeout').text = filtered_msg
        error_count += 1
      elif result.state == 'SKIPPED':
        ET.SubElement(xml_test, 'skipped', message='Skipped')
  testsuite.set('failures', str(failure_count))
  testsuite.set('errors', str(error_count))

def render_interop_html_report(
  client_langs, server_langs, test_cases, auth_test_cases, http2_cases,
  http2_server_cases, resultset,
  num_failures, cloud_to_prod, prod_servers, http2_interop):
  """Generate HTML report for interop tests."""
  template_file = 'tools/run_tests/interop/interop_html_report.template'
  try:
    mytemplate = Template(filename=template_file, format_exceptions=True)
  except NameError:
    print('Mako template is not installed. Skipping HTML report generation.')
    return
  except IOError as e:
    print('Failed to find the template %s: %s' % (template_file, e))
    return

  sorted_test_cases = sorted(test_cases)
  sorted_auth_test_cases = sorted(auth_test_cases)
  sorted_http2_cases = sorted(http2_cases)
  sorted_http2_server_cases = sorted(http2_server_cases)
  sorted_client_langs = sorted(client_langs)
  sorted_server_langs = sorted(server_langs)
  sorted_prod_servers = sorted(prod_servers)

  args = {'client_langs': sorted_client_langs,
          'server_langs': sorted_server_langs,
          'test_cases': sorted_test_cases,
          'auth_test_cases': sorted_auth_test_cases,
          'http2_cases': sorted_http2_cases,
          'http2_server_cases': sorted_http2_server_cases,
          'resultset': resultset,
          'num_failures': num_failures,
          'cloud_to_prod': cloud_to_prod,
          'prod_servers': sorted_prod_servers,
          'http2_interop': http2_interop}

  html_report_out_dir = 'reports'
  if not os.path.exists(html_report_out_dir):
    os.mkdir(html_report_out_dir)
  html_file_path = os.path.join(html_report_out_dir, 'index.html')
  try:
    with open(html_file_path, 'w') as output_file:
      mytemplate.render_context(Context(output_file, **args))
  except:
    print(exceptions.text_error_template().render())
    raise

def render_perf_profiling_results(output_filepath, profile_names):
  with open(output_filepath, 'w') as output_file:
    output_file.write('<ul>\n')
    for name in profile_names:
      output_file.write('<li><a href=%s>%s</a></li>\n' % (name, name))
    output_file.write('</ul>\n')