forked from OSchip/llvm-project
483 lines
17 KiB
Python
483 lines
17 KiB
Python
"""
|
|
The LLVM Compiler Infrastructure
|
|
|
|
This file is distributed under the University of Illinois Open Source
|
|
License. See LICENSE.TXT for details.
|
|
|
|
Provides a class to build Python test event data structures.
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
from __future__ import absolute_import
|
|
|
|
# System modules
|
|
import inspect
|
|
import time
|
|
import traceback
|
|
|
|
# Third-party modules
|
|
|
|
# LLDB modules
|
|
from . import build_exception
|
|
|
|
|
|
class EventBuilder(object):
|
|
"""Helper class to build test result event dictionaries."""
|
|
|
|
BASE_DICTIONARY = None
|
|
|
|
# Test Event Types
|
|
TYPE_JOB_RESULT = "job_result"
|
|
TYPE_TEST_RESULT = "test_result"
|
|
TYPE_TEST_START = "test_start"
|
|
TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
|
|
TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure"
|
|
TYPE_SESSION_TERMINATE = "terminate"
|
|
|
|
RESULT_TYPES = {TYPE_JOB_RESULT, TYPE_TEST_RESULT}
|
|
|
|
# Test/Job Status Tags
|
|
STATUS_EXCEPTIONAL_EXIT = "exceptional_exit"
|
|
STATUS_SUCCESS = "success"
|
|
STATUS_FAILURE = "failure"
|
|
STATUS_EXPECTED_FAILURE = "expected_failure"
|
|
STATUS_EXPECTED_TIMEOUT = "expected_timeout"
|
|
STATUS_UNEXPECTED_SUCCESS = "unexpected_success"
|
|
STATUS_SKIP = "skip"
|
|
STATUS_ERROR = "error"
|
|
STATUS_TIMEOUT = "timeout"
|
|
|
|
"""Test methods or jobs with a status matching any of these
|
|
status values will cause a testrun failure, unless
|
|
the test methods rerun and do not trigger an issue when rerun."""
|
|
TESTRUN_ERROR_STATUS_VALUES = {
|
|
STATUS_ERROR,
|
|
STATUS_EXCEPTIONAL_EXIT,
|
|
STATUS_FAILURE,
|
|
STATUS_TIMEOUT}
|
|
|
|
@staticmethod
|
|
def _get_test_name_info(test):
|
|
"""Returns (test-class-name, test-method-name) from a test case instance.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@return tuple containing (test class name, test method name)
|
|
"""
|
|
test_class_components = test.id().split(".")
|
|
test_class_name = ".".join(test_class_components[:-1])
|
|
test_name = test_class_components[-1]
|
|
return test_class_name, test_name
|
|
|
|
@staticmethod
|
|
def bare_event(event_type):
|
|
"""Creates an event with default additions, event type and timestamp.
|
|
|
|
@param event_type the value set for the "event" key, used
|
|
to distinguish events.
|
|
|
|
@returns an event dictionary with all default additions, the "event"
|
|
key set to the passed in event_type, and the event_time value set to
|
|
time.time().
|
|
"""
|
|
if EventBuilder.BASE_DICTIONARY is not None:
|
|
# Start with a copy of the "always include" entries.
|
|
event = dict(EventBuilder.BASE_DICTIONARY)
|
|
else:
|
|
event = {}
|
|
|
|
event.update({
|
|
"event": event_type,
|
|
"event_time": time.time()
|
|
})
|
|
return event
|
|
|
|
@staticmethod
|
|
def _assert_is_python_sourcefile(test_filename):
|
|
if test_filename is not None:
|
|
if not test_filename.endswith(".py"):
|
|
raise Exception(
|
|
"source python filename has unexpected extension: {}".format(test_filename))
|
|
return test_filename
|
|
|
|
@staticmethod
|
|
def _event_dictionary_common(test, event_type):
|
|
"""Returns an event dictionary setup with values for the given event type.
|
|
|
|
@param test the unittest.TestCase instance
|
|
|
|
@param event_type the name of the event type (string).
|
|
|
|
@return event dictionary with common event fields set.
|
|
"""
|
|
test_class_name, test_name = EventBuilder._get_test_name_info(test)
|
|
|
|
# Determine the filename for the test case. If there is an attribute
|
|
# for it, use it. Otherwise, determine from the TestCase class path.
|
|
if hasattr(test, "test_filename"):
|
|
test_filename = EventBuilder._assert_is_python_sourcefile(
|
|
test.test_filename)
|
|
else:
|
|
test_filename = EventBuilder._assert_is_python_sourcefile(
|
|
inspect.getsourcefile(test.__class__))
|
|
|
|
event = EventBuilder.bare_event(event_type)
|
|
event.update({
|
|
"test_class": test_class_name,
|
|
"test_name": test_name,
|
|
"test_filename": test_filename
|
|
})
|
|
|
|
return event
|
|
|
|
@staticmethod
|
|
def _error_tuple_class(error_tuple):
|
|
"""Returns the unittest error tuple's error class as a string.
|
|
|
|
@param error_tuple the error tuple provided by the test framework.
|
|
|
|
@return the error type (typically an exception) raised by the
|
|
test framework.
|
|
"""
|
|
type_var = error_tuple[0]
|
|
module = inspect.getmodule(type_var)
|
|
if module:
|
|
return "{}.{}".format(module.__name__, type_var.__name__)
|
|
else:
|
|
return type_var.__name__
|
|
|
|
@staticmethod
|
|
def _error_tuple_message(error_tuple):
|
|
"""Returns the unittest error tuple's error message.
|
|
|
|
@param error_tuple the error tuple provided by the test framework.
|
|
|
|
@return the error message provided by the test framework.
|
|
"""
|
|
return str(error_tuple[1])
|
|
|
|
@staticmethod
|
|
def _error_tuple_traceback(error_tuple):
|
|
"""Returns the unittest error tuple's error message.
|
|
|
|
@param error_tuple the error tuple provided by the test framework.
|
|
|
|
@return the error message provided by the test framework.
|
|
"""
|
|
return error_tuple[2]
|
|
|
|
@staticmethod
|
|
def _event_dictionary_test_result(test, status):
|
|
"""Returns an event dictionary with common test result fields set.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@param status the status/result of the test
|
|
(e.g. "success", "failure", etc.)
|
|
|
|
@return the event dictionary
|
|
"""
|
|
event = EventBuilder._event_dictionary_common(
|
|
test, EventBuilder.TYPE_TEST_RESULT)
|
|
event["status"] = status
|
|
return event
|
|
|
|
@staticmethod
|
|
def _event_dictionary_issue(test, status, error_tuple):
|
|
"""Returns an event dictionary with common issue-containing test result
|
|
fields set.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@param status the status/result of the test
|
|
(e.g. "success", "failure", etc.)
|
|
|
|
@param error_tuple the error tuple as reported by the test runner.
|
|
This is of the form (type<error>, error).
|
|
|
|
@return the event dictionary
|
|
"""
|
|
event = EventBuilder._event_dictionary_test_result(test, status)
|
|
event["issue_class"] = EventBuilder._error_tuple_class(error_tuple)
|
|
event["issue_message"] = EventBuilder._error_tuple_message(error_tuple)
|
|
backtrace = EventBuilder._error_tuple_traceback(error_tuple)
|
|
if backtrace is not None:
|
|
event["issue_backtrace"] = traceback.format_tb(backtrace)
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_start(test):
|
|
"""Returns an event dictionary for the test start event.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@return the event dictionary
|
|
"""
|
|
return EventBuilder._event_dictionary_common(
|
|
test, EventBuilder.TYPE_TEST_START)
|
|
|
|
@staticmethod
|
|
def event_for_success(test):
|
|
"""Returns an event dictionary for a successful test.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@return the event dictionary
|
|
"""
|
|
return EventBuilder._event_dictionary_test_result(
|
|
test, EventBuilder.STATUS_SUCCESS)
|
|
|
|
@staticmethod
|
|
def event_for_unexpected_success(test, bugnumber):
|
|
"""Returns an event dictionary for a test that succeeded but was
|
|
expected to fail.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@param bugnumber the issue identifier for the bug tracking the
|
|
fix request for the test expected to fail (but is in fact
|
|
passing here).
|
|
|
|
@return the event dictionary
|
|
|
|
"""
|
|
event = EventBuilder._event_dictionary_test_result(
|
|
test, EventBuilder.STATUS_UNEXPECTED_SUCCESS)
|
|
if bugnumber:
|
|
event["bugnumber"] = str(bugnumber)
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_failure(test, error_tuple):
|
|
"""Returns an event dictionary for a test that failed.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@param error_tuple the error tuple as reported by the test runner.
|
|
This is of the form (type<error>, error).
|
|
|
|
@return the event dictionary
|
|
"""
|
|
return EventBuilder._event_dictionary_issue(
|
|
test, EventBuilder.STATUS_FAILURE, error_tuple)
|
|
|
|
@staticmethod
|
|
def event_for_expected_failure(test, error_tuple, bugnumber):
|
|
"""Returns an event dictionary for a test that failed as expected.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@param error_tuple the error tuple as reported by the test runner.
|
|
This is of the form (type<error>, error).
|
|
|
|
@param bugnumber the issue identifier for the bug tracking the
|
|
fix request for the test expected to fail.
|
|
|
|
@return the event dictionary
|
|
|
|
"""
|
|
event = EventBuilder._event_dictionary_issue(
|
|
test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple)
|
|
if bugnumber:
|
|
event["bugnumber"] = str(bugnumber)
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_skip(test, reason):
|
|
"""Returns an event dictionary for a test that was skipped.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@param reason the reason why the test is being skipped.
|
|
|
|
@return the event dictionary
|
|
"""
|
|
event = EventBuilder._event_dictionary_test_result(
|
|
test, EventBuilder.STATUS_SKIP)
|
|
event["skip_reason"] = reason
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_error(test, error_tuple):
|
|
"""Returns an event dictionary for a test that hit a test execution error.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@param error_tuple the error tuple as reported by the test runner.
|
|
This is of the form (type<error>, error).
|
|
|
|
@return the event dictionary
|
|
"""
|
|
event = EventBuilder._event_dictionary_issue(
|
|
test, EventBuilder.STATUS_ERROR, error_tuple)
|
|
event["issue_phase"] = "test"
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_build_error(test, error_tuple):
|
|
"""Returns an event dictionary for a test that hit a test execution error
|
|
during the test cleanup phase.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@param error_tuple the error tuple as reported by the test runner.
|
|
This is of the form (type<error>, error).
|
|
|
|
@return the event dictionary
|
|
"""
|
|
event = EventBuilder._event_dictionary_issue(
|
|
test, EventBuilder.STATUS_ERROR, error_tuple)
|
|
event["issue_phase"] = "build"
|
|
|
|
build_error = error_tuple[1]
|
|
event["build_command"] = build_error.command
|
|
event["build_error"] = build_error.build_error
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_cleanup_error(test, error_tuple):
|
|
"""Returns an event dictionary for a test that hit a test execution error
|
|
during the test cleanup phase.
|
|
|
|
@param test a unittest.TestCase instance.
|
|
|
|
@param error_tuple the error tuple as reported by the test runner.
|
|
This is of the form (type<error>, error).
|
|
|
|
@return the event dictionary
|
|
"""
|
|
event = EventBuilder._event_dictionary_issue(
|
|
test, EventBuilder.STATUS_ERROR, error_tuple)
|
|
event["issue_phase"] = "cleanup"
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_job_test_add_error(test_filename, exception, backtrace):
|
|
event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
|
|
event["status"] = EventBuilder.STATUS_ERROR
|
|
if test_filename is not None:
|
|
event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
|
|
test_filename)
|
|
if exception is not None and "__class__" in dir(exception):
|
|
event["issue_class"] = exception.__class__
|
|
event["issue_message"] = exception
|
|
if backtrace is not None:
|
|
event["issue_backtrace"] = backtrace
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_job_exceptional_exit(
|
|
pid, worker_index, exception_code, exception_description,
|
|
test_filename, command_line):
|
|
"""Creates an event for a job (i.e. process) exit due to signal.
|
|
|
|
@param pid the process id for the job that failed
|
|
@param worker_index optional id for the job queue running the process
|
|
@param exception_code optional code
|
|
(e.g. SIGTERM integer signal number)
|
|
@param exception_description optional string containing symbolic
|
|
representation of the issue (e.g. "SIGTERM")
|
|
@param test_filename the path to the test filename that exited
|
|
in some exceptional way.
|
|
@param command_line the Popen()-style list provided as the command line
|
|
for the process that timed out.
|
|
|
|
@return an event dictionary coding the job completion description.
|
|
"""
|
|
event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
|
|
event["status"] = EventBuilder.STATUS_EXCEPTIONAL_EXIT
|
|
if pid is not None:
|
|
event["pid"] = pid
|
|
if worker_index is not None:
|
|
event["worker_index"] = int(worker_index)
|
|
if exception_code is not None:
|
|
event["exception_code"] = exception_code
|
|
if exception_description is not None:
|
|
event["exception_description"] = exception_description
|
|
if test_filename is not None:
|
|
event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
|
|
test_filename)
|
|
if command_line is not None:
|
|
event["command_line"] = command_line
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_job_timeout(pid, worker_index, test_filename, command_line):
|
|
"""Creates an event for a job (i.e. process) timeout.
|
|
|
|
@param pid the process id for the job that timed out
|
|
@param worker_index optional id for the job queue running the process
|
|
@param test_filename the path to the test filename that timed out.
|
|
@param command_line the Popen-style list provided as the command line
|
|
for the process that timed out.
|
|
|
|
@return an event dictionary coding the job completion description.
|
|
"""
|
|
event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
|
|
event["status"] = "timeout"
|
|
if pid is not None:
|
|
event["pid"] = pid
|
|
if worker_index is not None:
|
|
event["worker_index"] = int(worker_index)
|
|
if test_filename is not None:
|
|
event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
|
|
test_filename)
|
|
if command_line is not None:
|
|
event["command_line"] = command_line
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_mark_test_rerun_eligible(test):
|
|
"""Creates an event that indicates the specified test is explicitly
|
|
eligible for rerun.
|
|
|
|
Note there is a mode that will enable test rerun eligibility at the
|
|
global level. These markings for explicit rerun eligibility are
|
|
intended for the mode of running where only explicitly re-runnable
|
|
tests are rerun upon hitting an issue.
|
|
|
|
@param test the TestCase instance to which this pertains.
|
|
|
|
@return an event that specifies the given test as being eligible to
|
|
be rerun.
|
|
"""
|
|
event = EventBuilder._event_dictionary_common(
|
|
test,
|
|
EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE)
|
|
return event
|
|
|
|
@staticmethod
|
|
def event_for_mark_test_expected_failure(test):
|
|
"""Creates an event that indicates the specified test is expected
|
|
to fail.
|
|
|
|
@param test the TestCase instance to which this pertains.
|
|
|
|
@return an event that specifies the given test is expected to fail.
|
|
"""
|
|
event = EventBuilder._event_dictionary_common(
|
|
test,
|
|
EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE)
|
|
return event
|
|
|
|
@staticmethod
|
|
def add_entries_to_all_events(entries_dict):
|
|
"""Specifies a dictionary of entries to add to all test events.
|
|
|
|
This provides a mechanism for, say, a parallel test runner to
|
|
indicate to each inferior dotest.py that it should add a
|
|
worker index to each.
|
|
|
|
Calling this method replaces all previous entries added
|
|
by a prior call to this.
|
|
|
|
Event build methods will overwrite any entries that collide.
|
|
Thus, the passed in dictionary is the base, which gets merged
|
|
over by event building when keys collide.
|
|
|
|
@param entries_dict a dictionary containing key and value
|
|
pairs that should be merged into all events created by the
|
|
event generator. May be None to clear out any extra entries.
|
|
"""
|
|
EventBuilder.BASE_DICTIONARY = dict(entries_dict)
|