From b6d2212f5276bcf0e5ef6b5dc0c985b72664ae34 Mon Sep 17 00:00:00 2001 From: Tom Weaver Date: Tue, 31 Mar 2020 10:18:12 +0100 Subject: [PATCH] [Dexter] Add support for Windows to regression test suite. This patch addresses the issue of the regression suite not running on windows hardware. It changes the following things: * add new dexter regression suite command to lit.cfg.py that makes use of the clang-cl_vs2015 and dbgend builder and debuggers. * sprinkle the new regressionsuite command through the feature and tool tests that require them. * mark certain problem tests on windows * [revert fix] fixed darwin regression test failures by adding unsupported line for system-darwin to command tests. There's a couple of tests that fail (or pass) in unexpected ways on Windows. Problem tests are both the penalty and perfect expect_watch_type.cpp tests. Type information reporting parity is not possible a this time in dexter due to the nature of how different debuggers report type information back to their users. reviewers: Orlando Differential Revision: https://reviews.llvm.org/D76609 --- .../commands/penalty/expect_program_state.cpp | 6 ++-- .../commands/penalty/expect_step_kinds.cpp | 6 ++-- .../commands/penalty/expect_step_order.cpp | 6 ++-- .../commands/penalty/expect_watch_type.cpp | 11 ++++--- .../commands/penalty/expect_watch_value.cpp | 7 ++--- .../commands/penalty/unreachable.cpp | 7 ++--- .../commands/perfect/expect_program_state.cpp | 7 ++--- .../perfect/expect_step_kind/direction.cpp | 9 +++--- .../perfect/expect_step_kind/func.cpp | 6 ++-- .../expect_step_kind/func_external.cpp | 12 ++++---- .../perfect/expect_step_kind/recursive.cpp | 6 ++-- .../perfect/expect_step_kind/small_loop.cpp | 6 ++-- .../commands/perfect/expect_step_order.cpp | 6 ++-- .../commands/perfect/expect_watch_type.cpp | 10 ++++--- .../commands/perfect/expect_watch_value.cpp | 6 ++-- .../commands/perfect/unreachable.cpp | 6 ++-- .../dexter/feature_tests/subtools/view.cpp | 6 +--- debuginfo-tests/lit.cfg.py | 29 +++++++++++++++++++ 18 files changed, 81 insertions(+), 71 deletions(-) diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_program_state.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_program_state.cpp index 476906846793..46ae81872fa9 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_program_state.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_program_state.cpp @@ -2,11 +2,9 @@ // Check that \DexExpectProgramState correctly applies a penalty when // an expected program state is never found. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: not %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -glldb" -- %s \ -// RUN: | FileCheck %s +// RUN: not %dexter_regression_test -- %s | FileCheck %s // CHECK: expect_program_state.cpp: int GCD(int lhs, int rhs) diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp index 9612c068ad61..2075cd22fa67 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp @@ -2,11 +2,9 @@ // Check that \DexExpectStepKind correctly applies a penalty when // unexpected step kinds are encountered. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: not %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// RUN: not %dexter_regression_test -- %s | FileCheck %s // CHECK: expect_step_kinds.cpp: int abs(int i){ diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_order.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_order.cpp index e52a7fffe203..dc5f8c6b0c92 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_order.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_order.cpp @@ -2,11 +2,9 @@ // Check that \DexExpectStepOrder correctly applies a penalty for steps // found out of expected order. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: not %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// RUN: not %dexter_regression_test -- %s | FileCheck %s // CHECK: expect_step_order.cpp: int main() diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_type.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_type.cpp index 145eeed7ef86..838e1f87e303 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_type.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_type.cpp @@ -2,11 +2,14 @@ // Check that \DexExpectWatchType applies penalties when expected // types are not found and unexpected types are. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: not %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// +// NOTE: This test passes but not in the expected way on Windows. +// TODO: Reduce this test's coverage and be more specific about +// expected behaviour. +// +// RUN: not %dexter_regression_test -- %s | FileCheck %s // CHECK: expect_watch_type.cpp: template diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_value.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_value.cpp index b4c60fa32179..fa0ed119980e 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_value.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_value.cpp @@ -2,11 +2,10 @@ // Check that \DexExpectWatchValue correctly applies a penalty when // expected values are not found. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: not %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// +// RUN: not %dexter_regression_test -- %s | FileCheck %s // CHECK: expect_watch_value.cpp: int main() diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/unreachable.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/unreachable.cpp index 5cea1c4257d0..48f56b7d0452 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/penalty/unreachable.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/unreachable.cpp @@ -2,11 +2,10 @@ // Check that \DexUnreachable correctly applies a penalty if the command // line is stepped on. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: not %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// +// RUN: not %dexter_regression_test -- %s | FileCheck %s // CHECK: unreachable.cpp: int diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_program_state.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_program_state.cpp index ebb4766f089a..3e77938446b0 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_program_state.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_program_state.cpp @@ -2,11 +2,10 @@ // Check that \DexExpectWatchValue applies no penalties when expected // program states are found. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -glldb" -- %s \ -// RUN: | FileCheck %s +// +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: expect_program_state.cpp: int GCD(int lhs, int rhs) diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp index 3ff0e9fecbc6..1f7a96bafffa 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp @@ -5,11 +5,12 @@ // 'VERTICAL_FORWARD' for every step onto a greater source line number in // the same function. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// TODO: The dbgeng debugger does not support column step reporting at present. +// XFAIL: system-windows +// +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: direction.cpp: int func(int i) { diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp index d5ab3ffbd036..b386dc4aa48c 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp @@ -3,11 +3,9 @@ // trivial test. Expect one 'FUNC' per call to a function which is defined // in one of the source files in the test directory. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: func.cpp: int func(int i) { diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp index 6dff5c802968..cbefcb969082 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp @@ -2,13 +2,13 @@ // Check that \DexExpectStepKind correctly counts 'FUNC_EXTERNAL' steps // for a trivial test. Expect one 'FUNC_EXTERNAL' per external call. // -// REQUIRES: system-linux, lldb -// XFAIL: system-linux -// This fails right now on my linux machine, needs examining as to why. +// UNSUPPORTED: system-darwin // -// RUN: %dexter --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// XFAIL:* +// This fails right now on my linux and windows machine, needs examining as to +// why. +// +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: func_external.cpp: #include diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp index f2b54e442679..0264420a46be 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp @@ -3,11 +3,9 @@ // Specifically, ensure recursive calls count towards 'FUNC' and not // 'VERTICAL_BACKWARD'. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: recursive.cpp: int func(int i) { diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp index 1f9060747ec2..fb19869f9182 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp @@ -3,11 +3,9 @@ // where the last source line in the loop is a call. Expect steps out // of a function to a line before the call to count as 'VERTICAL_BACKWARD'. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: small_loop.cpp: int func(int i){ diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_order.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_order.cpp index a89a0cee3cc5..cbcb5063c10f 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_order.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_order.cpp @@ -2,11 +2,9 @@ // Check that \DexExpectStepOrder applies no penalty when the expected // order is found. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: expect_step_order.cpp: int main() diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_type.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_type.cpp index 41593837a9c8..67077bebcef8 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_type.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_type.cpp @@ -2,11 +2,13 @@ // Check that \DexExpectWatchType applies no penalties when expected // types are found. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// TODO: On Windows WITH dbgeng, This test takes a long time to run and doesn't evaluate type values +// in the same manner as LLDB. +// XFAIL: system-windows +// +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: expect_watch_type.cpp: template diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_value.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_value.cpp index 035e41136980..f22431a0900d 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_value.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_value.cpp @@ -2,11 +2,9 @@ // Check that \DexExpectWatchValue applies no penalties when expected // values are found. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: expect_watch_value.cpp: unsigned long Factorial(int n) { diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/unreachable.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/unreachable.cpp index b158b4851be4..bcc1b835a648 100644 --- a/debuginfo-tests/dexter/feature_tests/commands/perfect/unreachable.cpp +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/unreachable.cpp @@ -2,11 +2,9 @@ // Check that \DexUnreachable has no effect if the command line is never // stepped on. // -// REQUIRES: system-linux, lldb +// UNSUPPORTED: system-darwin // -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \ -// RUN: | FileCheck %s +// RUN: %dexter_regression_test -- %s | FileCheck %s // CHECK: unreachable.cpp: int main() diff --git a/debuginfo-tests/dexter/feature_tests/subtools/view.cpp b/debuginfo-tests/dexter/feature_tests/subtools/view.cpp index 43598dbdc9b9..03e0b9420fb1 100644 --- a/debuginfo-tests/dexter/feature_tests/subtools/view.cpp +++ b/debuginfo-tests/dexter/feature_tests/subtools/view.cpp @@ -1,11 +1,7 @@ // Purpose: // Check the `view` subtool works with typical inputs. // -// REQUIRES: system-linux, lldb -// -// RUN: %dexter_base test --fail-lt 1.0 -w \ -// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" \ -// RUN: --results %t -- %s +// RUN: %dexter_regression_test --results %t -- %s // // RUN: %dexter_base view %t/view.cpp.dextIR | FileCheck %s // CHECK: ## BEGIN diff --git a/debuginfo-tests/lit.cfg.py b/debuginfo-tests/lit.cfg.py index 2d93b8da9bf6..fb2f72357b26 100644 --- a/debuginfo-tests/lit.cfg.py +++ b/debuginfo-tests/lit.cfg.py @@ -112,6 +112,35 @@ tools.append(ToolSubst('%dexter', dexter_test_cmd)) dexter_base_cmd = '"{}" "{}"'.format(config.python3_executable, dexter_path) tools.append(ToolSubst('%dexter_base', dexter_base_cmd)) +# Set up commands for DexTer regression tests. +# Builder, debugger, optimisation level and several other flags differ +# depending on whether we're running a unix like or windows os. +if platform.system() == 'Windows': + dexter_regression_test_builder = '--builder clang-cl_vs2015' + dexter_regression_test_debugger = '--debugger dbgeng' + dexter_regression_test_cflags = '--cflags "/Zi /Od"' + dexter_regression_test_ldflags = '--ldflags "/Zi"' +else: + dexter_regression_test_builder = '--builder clang' + dexter_regression_test_debugger = "--debugger lldb" + dexter_regression_test_cflags = '--cflags "-O0 -glldb"' + dexter_regression_test_ldflags = '' + +# Typical command would take the form: +# ./path_to_py/python.exe ./path_to_dex/dexter.py test --fail-lt 1.0 -w --builder clang --debugger lldb --cflags '-O0 -g' +dexter_regression_test_command = ' '.join( + # "python3", "dexter.py", test, fail_mode, builder, debugger, cflags, ldflags + ["{}".format(config.python3_executable), + "{}".format(dexter_path), + 'test', + '--fail-lt 1.0 -w', + dexter_regression_test_builder, + dexter_regression_test_debugger, + dexter_regression_test_cflags, + dexter_regression_test_ldflags]) + +tools.append(ToolSubst('%dexter_regression_test', dexter_regression_test_command)) + tool_dirs = [config.llvm_tools_dir] llvm_config.add_tool_substitutions(tools, tool_dirs)