From 5578d008d9e06bb531fb3e62dd17096d9fd9c853 Mon Sep 17 00:00:00 2001 From: Brendan Higgins Date: Tue, 11 Aug 2020 14:27:55 -0700 Subject: [PATCH 1/5] kunit: tool: fix running kunit_tool from outside kernel tree Currently kunit_tool does not work correctly when executed from a path outside of the kernel tree, so make sure that the current working directory is correct and the kunit_dir is properly initialized before running. Signed-off-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 425ef40067e7..e2caf4e24ecb 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -237,9 +237,13 @@ def main(argv, linux=None): cli_args = parser.parse_args(argv) + if get_kernel_root_path(): + os.chdir(get_kernel_root_path()) + if cli_args.subcommand == 'run': if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir) + create_default_kunitconfig() if not linux: linux = kunit_kernel.LinuxSourceTree() @@ -257,6 +261,7 @@ def main(argv, linux=None): if cli_args.build_dir: if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir) + create_default_kunitconfig() if not linux: linux = kunit_kernel.LinuxSourceTree() @@ -270,10 +275,6 @@ def main(argv, linux=None): if result.status != KunitStatus.SUCCESS: sys.exit(1) elif cli_args.subcommand == 'build': - if cli_args.build_dir: - if not os.path.exists(cli_args.build_dir): - os.mkdir(cli_args.build_dir) - if not linux: linux = kunit_kernel.LinuxSourceTree() @@ -288,10 +289,6 @@ def main(argv, linux=None): if result.status != KunitStatus.SUCCESS: sys.exit(1) elif cli_args.subcommand == 'exec': - if cli_args.build_dir: - if not os.path.exists(cli_args.build_dir): - os.mkdir(cli_args.build_dir) - if not linux: linux = kunit_kernel.LinuxSourceTree() From 21a6d1780d5bbfca0ce9b8104ca6233502fcbf86 Mon Sep 17 00:00:00 2001 From: Heidi Fahim Date: Tue, 11 Aug 2020 14:27:56 -0700 Subject: [PATCH 2/5] kunit: tool: allow generating test results in JSON Add a --json flag, which when specified generates JSON formatted test results conforming to the KernelCI API test_group spec[1]. The user can use the new flag to specify a filename to print the json formatted results to. Link[1]: https://api.kernelci.org/schema-test-group.html#post Signed-off-by: Heidi Fahim Signed-off-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 35 +++++++++++--- tools/testing/kunit/kunit_json.py | 63 ++++++++++++++++++++++++++ tools/testing/kunit/kunit_tool_test.py | 33 ++++++++++++++ 3 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 tools/testing/kunit/kunit_json.py diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index e2caf4e24ecb..3c95a0eb0d04 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -17,6 +17,7 @@ from collections import namedtuple from enum import Enum, auto import kunit_config +import kunit_json import kunit_kernel import kunit_parser @@ -30,9 +31,9 @@ KunitBuildRequest = namedtuple('KunitBuildRequest', KunitExecRequest = namedtuple('KunitExecRequest', ['timeout', 'build_dir', 'alltests']) KunitParseRequest = namedtuple('KunitParseRequest', - ['raw_output', 'input_data']) + ['raw_output', 'input_data', 'build_dir', 'json']) KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', - 'build_dir', 'alltests', + 'build_dir', 'alltests', 'json', 'make_options']) KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] @@ -113,12 +114,22 @@ def parse_tests(request: KunitParseRequest) -> KunitResult: test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, [], 'Tests not Parsed.') + if request.raw_output: kunit_parser.raw_output(request.input_data) else: test_result = kunit_parser.parse_run_tests(request.input_data) parse_end = time.time() + if request.json: + json_obj = kunit_json.get_json_result( + test_result=test_result, + def_config='kunit_defconfig', + build_dir=request.build_dir, + json_path=request.json) + if request.json == 'stdout': + print(json_obj) + if test_result.status != kunit_parser.TestStatus.SUCCESS: return KunitResult(KunitStatus.TEST_FAILURE, test_result, parse_end - parse_start) @@ -151,7 +162,9 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, return exec_result parse_request = KunitParseRequest(request.raw_output, - exec_result.result) + exec_result.result, + request.build_dir, + request.json) parse_result = parse_tests(parse_request) run_end = time.time() @@ -195,7 +208,12 @@ def add_exec_opts(parser): def add_parse_opts(parser): parser.add_argument('--raw_output', help='don\'t format output from kernel', action='store_true') - + parser.add_argument('--json', + nargs='?', + help='Stores test results in a JSON, and either ' + 'prints to stdout or saves to file if a ' + 'filename is specified', + type=str, const='stdout', default=None) def main(argv, linux=None): parser = argparse.ArgumentParser( @@ -253,6 +271,7 @@ def main(argv, linux=None): cli_args.jobs, cli_args.build_dir, cli_args.alltests, + cli_args.json, cli_args.make_options) result = run_tests(linux, request) if result.status != KunitStatus.SUCCESS: @@ -297,7 +316,9 @@ def main(argv, linux=None): cli_args.alltests) exec_result = exec_tests(linux, exec_request) parse_request = KunitParseRequest(cli_args.raw_output, - exec_result.result) + exec_result.result, + cli_args.build_dir, + cli_args.json) result = parse_tests(parse_request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % ( @@ -311,7 +332,9 @@ def main(argv, linux=None): with open(cli_args.file, 'r') as f: kunit_output = f.read().splitlines() request = KunitParseRequest(cli_args.raw_output, - kunit_output) + kunit_output, + cli_args.build_dir, + cli_args.json) result = parse_tests(request) if result.status != KunitStatus.SUCCESS: sys.exit(1) diff --git a/tools/testing/kunit/kunit_json.py b/tools/testing/kunit/kunit_json.py new file mode 100644 index 000000000000..624b31b2dbd6 --- /dev/null +++ b/tools/testing/kunit/kunit_json.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Generates JSON from KUnit results according to +# KernelCI spec: https://github.com/kernelci/kernelci-doc/wiki/Test-API +# +# Copyright (C) 2020, Google LLC. +# Author: Heidi Fahim + +import json +import os + +import kunit_parser + +from kunit_parser import TestStatus + +def get_json_result(test_result, def_config, build_dir, json_path): + sub_groups = [] + + # Each test suite is mapped to a KernelCI sub_group + for test_suite in test_result.suites: + sub_group = { + "name": test_suite.name, + "arch": "UM", + "defconfig": def_config, + "build_environment": build_dir, + "test_cases": [], + "lab_name": None, + "kernel": None, + "job": None, + "git_branch": "kselftest", + } + test_cases = [] + # TODO: Add attachments attribute in test_case with detailed + # failure message, see https://api.kernelci.org/schema-test-case.html#get + for case in test_suite.cases: + test_case = {"name": case.name, "status": "FAIL"} + if case.status == TestStatus.SUCCESS: + test_case["status"] = "PASS" + elif case.status == TestStatus.TEST_CRASHED: + test_case["status"] = "ERROR" + test_cases.append(test_case) + sub_group["test_cases"] = test_cases + sub_groups.append(sub_group) + test_group = { + "name": "KUnit Test Group", + "arch": "UM", + "defconfig": def_config, + "build_environment": build_dir, + "sub_groups": sub_groups, + "lab_name": None, + "kernel": None, + "job": None, + "git_branch": "kselftest", + } + json_obj = json.dumps(test_group, indent=4) + if json_path != 'stdout': + with open(json_path, 'w') as result_path: + result_path.write(json_obj) + root = __file__.split('tools/testing/kunit/')[0] + kunit_parser.print_with_timestamp( + "Test results stored in %s" % + os.path.join(root, result_path.name)) + return json_obj diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 287c74d821c3..99c3c5671ea4 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -11,11 +11,13 @@ from unittest import mock import tempfile, shutil # Handling test_tmpdir +import json import os import kunit_config import kunit_parser import kunit_kernel +import kunit_json import kunit test_tmpdir = '' @@ -230,6 +232,37 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests(file.readlines()) self.assertEqual('kunit-resource-test', result.suites[0].name) +class KUnitJsonTest(unittest.TestCase): + + def _json_for(self, log_file): + with(open(get_absolute_path(log_file))) as file: + test_result = kunit_parser.parse_run_tests(file) + json_obj = kunit_json.get_json_result( + test_result=test_result, + def_config='kunit_defconfig', + build_dir=None, + json_path='stdout') + return json.loads(json_obj) + + def test_failed_test_json(self): + result = self._json_for( + 'test_data/test_is_test_passed-failure.log') + self.assertEqual( + {'name': 'example_simple_test', 'status': 'FAIL'}, + result["sub_groups"][1]["test_cases"][0]) + + def test_crashed_test_json(self): + result = self._json_for( + 'test_data/test_is_test_passed-crash.log') + self.assertEqual( + {'name': 'example_simple_test', 'status': 'ERROR'}, + result["sub_groups"][1]["test_cases"][0]) + + def test_no_tests_json(self): + result = self._json_for( + 'test_data/test_is_test_passed-no_tests_run.log') + self.assertEqual(0, len(result['sub_groups'])) + class StrContains(str): def __eq__(self, other): return self in other From 67e2fae3b767fd4444f3f161f6420d0d0338a10d Mon Sep 17 00:00:00 2001 From: Brendan Higgins Date: Wed, 23 Sep 2020 14:19:38 -0700 Subject: [PATCH 3/5] kunit: tool: fix --alltests flag Alltests flag evidently stopped working when run from outside of the root of the source tree, so fix that. Also add an additional broken config to the broken_on_uml config. Signed-off-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/configs/broken_on_uml.config | 1 + tools/testing/kunit/kunit_kernel.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/testing/kunit/configs/broken_on_uml.config b/tools/testing/kunit/configs/broken_on_uml.config index 239b9f03da2c..a7f0603d33f6 100644 --- a/tools/testing/kunit/configs/broken_on_uml.config +++ b/tools/testing/kunit/configs/broken_on_uml.config @@ -39,3 +39,4 @@ # CONFIG_QCOM_CPR is not set # CONFIG_RESET_BRCMSTB_RESCAL is not set # CONFIG_RESET_INTEL_GW is not set +# CONFIG_ADI_AXI_ADC is not set diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index e20e2056cb38..1b1826500f61 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -53,18 +53,23 @@ class LinuxSourceTreeOperations(object): except subprocess.CalledProcessError as e: raise ConfigError(e.output) - def make_allyesconfig(self): + def make_allyesconfig(self, build_dir, make_options): kunit_parser.print_with_timestamp( 'Enabling all CONFIGs for UML...') + command = ['make', 'ARCH=um', 'allyesconfig'] + if make_options: + command.extend(make_options) + if build_dir: + command += ['O=' + build_dir] process = subprocess.Popen( - ['make', 'ARCH=um', 'allyesconfig'], + command, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) process.wait() kunit_parser.print_with_timestamp( 'Disabling broken configs to run KUnit tests...') with ExitStack() as es: - config = open(KCONFIG_PATH, 'a') + config = open(get_kconfig_path(build_dir), 'a') disable = open(BROKEN_ALLCONFIG_PATH, 'r').read() config.write(disable) kunit_parser.print_with_timestamp( @@ -161,9 +166,9 @@ class LinuxSourceTree(object): return self.build_config(build_dir, make_options) def build_um_kernel(self, alltests, jobs, build_dir, make_options): - if alltests: - self._ops.make_allyesconfig() try: + if alltests: + self._ops.make_allyesconfig(build_dir, make_options) self._ops.make_olddefconfig(build_dir, make_options) self._ops.make(jobs, build_dir, make_options) except (ConfigError, BuildError) as e: From 82206a0c06ccbd4a80cd623b9d52fcda130d6c7d Mon Sep 17 00:00:00 2001 From: Brendan Higgins Date: Mon, 28 Sep 2020 13:02:27 -0700 Subject: [PATCH 4/5] kunit: tool: handle when .kunit exists but .kunitconfig does not Right now .kunitconfig and the build dir are automatically created if the build dir does not exists; however, if the build dir is present and .kunitconfig is not, kunit_tool will crash. Fix this by checking for both the build dir as well as the .kunitconfig. NOTE: This depends on commit 5578d008d9e0 ("kunit: tool: fix running kunit_tool from outside kernel tree") Link: https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git/commit/?id=5578d008d9e06bb531fb3e62dd17096d9fd9c853 Signed-off-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 3c95a0eb0d04..ebf5f5763dee 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -261,6 +261,8 @@ def main(argv, linux=None): if cli_args.subcommand == 'run': if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir) + + if not os.path.exists(kunit_kernel.kunitconfig_path): create_default_kunitconfig() if not linux: @@ -277,10 +279,12 @@ def main(argv, linux=None): if result.status != KunitStatus.SUCCESS: sys.exit(1) elif cli_args.subcommand == 'config': - if cli_args.build_dir: - if not os.path.exists(cli_args.build_dir): - os.mkdir(cli_args.build_dir) - create_default_kunitconfig() + if cli_args.build_dir and ( + not os.path.exists(cli_args.build_dir)): + os.mkdir(cli_args.build_dir) + + if not os.path.exists(kunit_kernel.kunitconfig_path): + create_default_kunitconfig() if not linux: linux = kunit_kernel.LinuxSourceTree() From 1abdd39f14b25dd2d69096b624a4f86f158a9feb Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Wed, 30 Sep 2020 11:31:51 -0700 Subject: [PATCH 5/5] kunit: tool: fix display of make errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CalledProcessError stores the output of the failed process as `bytes`, not a `str`. So when we log it on build error, the make output is all crammed into one line with "\n" instead of actually printing new lines. After this change, we get readable output with new lines, e.g. > CC lib/kunit/kunit-example-test.o > In file included from ../lib/kunit/test.c:9: > ../include/kunit/test.h:22:1: error: unknown type name ‘invalid_type_that_causes_compile’ > 22 | invalid_type_that_causes_compile errors; > | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ > make[3]: *** [../scripts/Makefile.build:283: lib/kunit/test.o] Error 1 Secondly, trying to concat exceptions to strings will fail with > TypeError: can only concatenate str (not "OSError") to str so fix this with an explicit cast to str. Signed-off-by: Daniel Latypov Reviewed-by: Brendan Higgins Tested-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit_kernel.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index 1b1826500f61..b557b1e93f98 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -36,9 +36,9 @@ class LinuxSourceTreeOperations(object): try: subprocess.check_output(['make', 'mrproper'], stderr=subprocess.STDOUT) except OSError as e: - raise ConfigError('Could not call make command: ' + e) + raise ConfigError('Could not call make command: ' + str(e)) except subprocess.CalledProcessError as e: - raise ConfigError(e.output) + raise ConfigError(e.output.decode()) def make_olddefconfig(self, build_dir, make_options): command = ['make', 'ARCH=um', 'olddefconfig'] @@ -49,9 +49,9 @@ class LinuxSourceTreeOperations(object): try: subprocess.check_output(command, stderr=subprocess.STDOUT) except OSError as e: - raise ConfigError('Could not call make command: ' + e) + raise ConfigError('Could not call make command: ' + str(e)) except subprocess.CalledProcessError as e: - raise ConfigError(e.output) + raise ConfigError(e.output.decode()) def make_allyesconfig(self, build_dir, make_options): kunit_parser.print_with_timestamp( @@ -84,9 +84,9 @@ class LinuxSourceTreeOperations(object): try: subprocess.check_output(command, stderr=subprocess.STDOUT) except OSError as e: - raise BuildError('Could not call execute make: ' + e) + raise BuildError('Could not call execute make: ' + str(e)) except subprocess.CalledProcessError as e: - raise BuildError(e.output) + raise BuildError(e.output.decode()) def linux_bin(self, params, timeout, build_dir, outfile): """Runs the Linux UML binary. Must be named 'linux'."""