mirror of https://github.com/pwndbg/pwndbg
feat: use pytest for qemu-system tests (#1679)
* feat: use pytest for qemu-system tests * CI: update qemu workflow * feat: make tests aware of ARCH and KERNEL_TYPE
This commit is contained in:
theguy147
GitHub
parent
424c21a6be
commit
91c72a001e
|
|
@ -65,6 +65,7 @@ jobs:
|
|||
run: |
|
||||
./setup.sh --user
|
||||
./setup-dev.sh --user
|
||||
mkdir .cov
|
||||
|
||||
- name: Download images
|
||||
run: |
|
||||
|
|
@ -72,6 +73,15 @@ jobs:
|
|||
|
||||
# We set `kernel.yama.ptrace_scope=0` for `gdb-pt-dump`
|
||||
- name: Run tests
|
||||
working-directory: ./tests/qemu-tests
|
||||
run: |
|
||||
sudo sysctl -w kernel.yama.ptrace_scope=0
|
||||
./tests/qemu-tests/tests.sh
|
||||
./tests.sh --cov
|
||||
|
||||
- name: Process coverage data
|
||||
run: |
|
||||
coverage combine
|
||||
coverage xml
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
ARCH="$1"
|
||||
shift
|
||||
|
||||
KERNEL_TYPE="$1"
|
||||
shift
|
||||
|
||||
CWD=$(dirname -- "$0")
|
||||
IMAGE_DIR="${CWD}/images"
|
||||
|
||||
if [[ -z "$ARCH" || -z "$KERNEL_TYPE" ]]; then
|
||||
echo "usage: $0 ARCH [ack | linux]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ptrace_scope=$(cat /proc/sys/kernel/yama/ptrace_scope)
|
||||
if [[ $ptrace_scope -ne 0 && $(id -u) -ne 0 ]]; then
|
||||
cat << EOF
|
||||
WARNING: You are not running as root and ptrace_scope is not set to zero. If you
|
||||
run into issues when using pwndbg or gdb-pt-dump, rerun this script as root, or
|
||||
alternatively run the following command:
|
||||
|
||||
echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
|
||||
|
||||
EOF
|
||||
fi
|
||||
|
||||
if [[ "$ARCH" == x86_64 ]]; then
|
||||
GDB=gdb
|
||||
else
|
||||
GDB=gdb-multiarch
|
||||
fi
|
||||
|
||||
VMLINUX="${IMAGE_DIR}/vmlinux-${KERNEL_TYPE}-${ARCH}"
|
||||
|
||||
exec "${GDB}" -q \
|
||||
-ex "file ${VMLINUX}" \
|
||||
-ex "target remote :1234" \
|
||||
-ex "source ${CWD}/tests/test_qemu_system.py" \
|
||||
-ex "quit" \
|
||||
"$@"
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
TESTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests/system")
|
||||
|
||||
|
||||
class CollectTestFunctionNames:
|
||||
"""See https://github.com/pytest-dev/pytest/issues/2039#issuecomment-257753269"""
|
||||
|
||||
def __init__(self):
|
||||
self.collected = []
|
||||
|
||||
def pytest_collection_modifyitems(self, items):
|
||||
for item in items:
|
||||
self.collected.append(item.nodeid)
|
||||
|
||||
|
||||
collector = CollectTestFunctionNames()
|
||||
rv = pytest.main(["--collect-only", TESTS_PATH], plugins=[collector])
|
||||
|
||||
if rv == pytest.ExitCode.INTERRUPTED:
|
||||
print("Failed to collect all tests, perhaps there is a syntax error in one of test files?")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
print("Listing collected tests:")
|
||||
for nodeid in collector.collected:
|
||||
print("Test:", nodeid)
|
||||
|
||||
# easy way to exit GDB session
|
||||
sys.exit(0)
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
use_pdb = os.environ.get("USE_PDB") == "1"
|
||||
|
||||
sys._pwndbg_unittest_run = True
|
||||
|
||||
CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
test = os.environ["PWNDBG_LAUNCH_TEST"]
|
||||
|
||||
test = os.path.join(CURRENT_DIR, test)
|
||||
|
||||
args = [test, "-vvv", "-s", "--showlocals", "--color=yes"]
|
||||
|
||||
if use_pdb:
|
||||
args.append("--pdb")
|
||||
|
||||
print("Launching pytest with args: %s" % args)
|
||||
|
||||
return_code = pytest.main(args)
|
||||
|
||||
if return_code != 0:
|
||||
print("-" * 80)
|
||||
print("If you want to debug tests locally, run ./tests.sh with the --pdb flag")
|
||||
print("-" * 80)
|
||||
|
||||
sys.exit(return_code)
|
||||
|
|
@ -10,5 +10,5 @@ qemu-aarch64 \
|
|||
gdb-multiarch \
|
||||
-ex "file ./binaries/reference-binary.aarch64.out" \
|
||||
-ex "target remote :1234" \
|
||||
-ex "source ./tests/test_qemu_user_aarch64.py" \
|
||||
-ex "source ./tests/user/test_aarch64.py" \
|
||||
-ex "quit"
|
||||
|
|
|
|||
|
|
@ -1,19 +1,221 @@
|
|||
#!/bin/bash
|
||||
|
||||
#set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
ROOT_DIR="$(readlink -f ../../)"
|
||||
GDB_INIT_PATH="$ROOT_DIR/gdbinit.py"
|
||||
COVERAGERC_PATH="$ROOT_DIR/pyproject.toml"
|
||||
|
||||
ARCH=""
|
||||
KERNEL_TYPE=""
|
||||
VMLINUX=""
|
||||
|
||||
PLATFORMS=(
|
||||
# ARCH KERNEL_TYPE
|
||||
"x86_64 linux"
|
||||
"x86_64 ack"
|
||||
"arm64 linux"
|
||||
"arm64 ack"
|
||||
)
|
||||
|
||||
CWD=$(dirname -- "$0")
|
||||
IMAGE_DIR="${CWD}/images"
|
||||
|
||||
set -x
|
||||
ptrace_scope=$(cat /proc/sys/kernel/yama/ptrace_scope)
|
||||
if [[ $ptrace_scope -ne 0 && $(id -u) -ne 0 ]]; then
|
||||
cat << EOF
|
||||
WARNING: You are not running as root and ptrace_scope is not set to zero. If you
|
||||
run into issues when using pwndbg or gdb-pt-dump, rerun this script as root, or
|
||||
alternatively run the following command:
|
||||
|
||||
for kernel_type in linux ack; do
|
||||
for arch in x86_64 arm64; do
|
||||
"${CWD}/run_qemu_system.sh" $arch $kernel_type > /dev/null &
|
||||
echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
|
||||
|
||||
"${CWD}/gdb.sh" $arch $kernel_type
|
||||
exit_code=$?
|
||||
EOF
|
||||
fi
|
||||
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
exit $exit_code
|
||||
fi
|
||||
pkill qemu
|
||||
done
|
||||
help_and_exit() {
|
||||
echo "Usage: ./tests.sh [-p|--pdb] [-c|--cov] [<test-name-filter>]"
|
||||
echo " -p, --pdb enable pdb (Python debugger) post mortem debugger on failed tests"
|
||||
echo " -c, --cov enable codecov"
|
||||
echo " -v, --verbose display all test output instead of just failing test output"
|
||||
echo " --collect-only only show the output of test collection, don't run any tests"
|
||||
echo " <test-name-filter> run only tests that match the regex"
|
||||
exit 1
|
||||
}
|
||||
|
||||
handle_sigint() {
|
||||
echo "Exiting..." >&2
|
||||
pkill qemu-system
|
||||
exit 1
|
||||
}
|
||||
trap handle_sigint SIGINT
|
||||
|
||||
if [[ $# -gt 3 ]]; then
|
||||
help_and_exit
|
||||
fi
|
||||
|
||||
USE_PDB=0
|
||||
TEST_NAME_FILTER=""
|
||||
RUN_CODECOV=0
|
||||
VERBOSE=0
|
||||
COLLECT_ONLY=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-p | --pdb)
|
||||
USE_PDB=1
|
||||
echo "Will run tests with Python debugger"
|
||||
shift
|
||||
;;
|
||||
-c | --cov)
|
||||
echo "Will run codecov"
|
||||
RUN_CODECOV=1
|
||||
shift
|
||||
;;
|
||||
-v | --verbose)
|
||||
VERBOSE=1
|
||||
shift
|
||||
;;
|
||||
--collect-only)
|
||||
COLLECT_ONLY=1
|
||||
shift
|
||||
;;
|
||||
-h | --help)
|
||||
help_and_exit
|
||||
;;
|
||||
*)
|
||||
if [[ ! -z "${TEST_NAME_FILTER}" ]]; then
|
||||
help_and_exit
|
||||
fi
|
||||
TEST_NAME_FILTER="$1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
gdb_load_pwndbg=(--command "$GDB_INIT_PATH" -ex "set exception-verbose on")
|
||||
run_gdb() {
|
||||
if [[ "$ARCH" == x86_64 ]]; then
|
||||
GDB=gdb
|
||||
else
|
||||
GDB=gdb-multiarch
|
||||
fi
|
||||
|
||||
$GDB --silent --nx --nh "${gdb_load_pwndbg[@]}" "$@" -ex "quit" 2> /dev/null
|
||||
return $?
|
||||
}
|
||||
|
||||
# NOTE: We run tests under GDB sessions and because of some cleanup/tests dependencies problems
|
||||
# we decided to run each test in a separate GDB session
|
||||
gdb_args=(--command pytests_collect.py)
|
||||
TESTS_COLLECT_OUTPUT=$(run_gdb "${gdb_args[@]}")
|
||||
|
||||
if [ $? -eq 1 ]; then
|
||||
echo -E "$TESTS_COLLECT_OUTPUT"
|
||||
exit 1
|
||||
elif [ $COLLECT_ONLY -eq 1 ]; then
|
||||
echo "$TESTS_COLLECT_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TESTS_LIST=($(echo -E "$TESTS_COLLECT_OUTPUT" | grep -o "tests/.*::.*" | grep "${TEST_NAME_FILTER}"))
|
||||
|
||||
init_gdb() {
|
||||
gdb_connect_qemu=(-ex "file ${VMLINUX}" -ex "target remote :1234")
|
||||
gdb_args=("${gdb_connect_qemu[@]}" -ex 'break start_kernel' -ex 'continue')
|
||||
run_gdb "${gdb_args[@]}" > /dev/null 2>&1
|
||||
}
|
||||
|
||||
run_test() {
|
||||
test_case="$1"
|
||||
|
||||
gdb_connect_qemu=(-ex "file ${VMLINUX}" -ex "target remote :1234")
|
||||
gdb_args=("${gdb_connect_qemu[@]}" --command pytests_launcher.py)
|
||||
if [ ${RUN_CODECOV} -ne 0 ]; then
|
||||
gdb_args=(-ex 'py import coverage;coverage.process_startup()' "${gdb_args[@]}")
|
||||
fi
|
||||
SRC_DIR=$ROOT_DIR \
|
||||
COVERAGE_FILE=$ROOT_DIR/.cov/coverage \
|
||||
COVERAGE_PROCESS_START=$COVERAGERC_PATH \
|
||||
USE_PDB="${USE_PDB}" \
|
||||
PWNDBG_LAUNCH_TEST="${test_case}" \
|
||||
PWNDBG_DISABLE_COLORS=1 \
|
||||
PWNDBG_ARCH="$ARCH" \
|
||||
PWNDBG_KERNEL_TYPE="$KERNEL_TYPE" \
|
||||
run_gdb "${gdb_args[@]}"
|
||||
return $?
|
||||
}
|
||||
|
||||
process_output() {
|
||||
output="$1"
|
||||
|
||||
read -r testname result < <(
|
||||
echo "$output" | grep -Po '(^tests/[^ ]+)|(\x1b\[3.m(PASSED|FAILED|SKIPPED|XPASS|XFAIL)\x1b\[0m)' \
|
||||
| tr '\n' ' ' \
|
||||
| cut -d ' ' -f 1,2
|
||||
)
|
||||
testfile=${testname%::*}
|
||||
testname=${testname#*::}
|
||||
|
||||
printf '%-70s %s\n' $testname $result
|
||||
|
||||
if [[ "$result" =~ FAIL ]]; then
|
||||
FAILED_TESTS+=("$testname")
|
||||
fi
|
||||
|
||||
# Only show the output of failed tests unless the verbose flag was used
|
||||
if [[ $VERBOSE -eq 1 || "$result" =~ FAIL ]]; then
|
||||
echo ""
|
||||
echo "$output"
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
test_system() {
|
||||
FAILED_TESTS=()
|
||||
echo "============================ Testing $KERNEL_TYPE-$ARCH ============================"
|
||||
|
||||
"${CWD}/run_qemu_system.sh" "$ARCH" "$KERNEL_TYPE" > /dev/null 2>&1 &
|
||||
|
||||
init_gdb
|
||||
start=$(date +%s)
|
||||
|
||||
for t in "${TESTS_LIST[@]}"; do
|
||||
output=$(run_test "$t")
|
||||
process_output "$output"
|
||||
done
|
||||
|
||||
end=$(date +%s)
|
||||
seconds=$((end - start))
|
||||
echo "Tests completed in ${seconds} seconds"
|
||||
|
||||
num_tests_failed=${#FAILED_TESTS[@]}
|
||||
num_tests_passed_or_skipped=$((${#TESTS_LIST[@]} - $num_tests_failed))
|
||||
|
||||
echo ""
|
||||
echo "*********************************"
|
||||
echo "********* TESTS SUMMARY *********"
|
||||
echo "*********************************"
|
||||
echo "Tests passed or skipped: ${num_tests_passed_or_skipped}"
|
||||
echo "Tests failed: ${num_tests_failed}"
|
||||
|
||||
if [ "${num_tests_failed}" -ne 0 ]; then
|
||||
echo ""
|
||||
echo "Failing tests: ${FAILED_TESTS[@]}"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pkill qemu-system
|
||||
}
|
||||
|
||||
for platform in "${PLATFORMS[@]}"; do
|
||||
read -r arch kernel_type <<< "$platform"
|
||||
|
||||
ARCH="$arch"
|
||||
KERNEL_TYPE="$kernel_type"
|
||||
VMLINUX="${IMAGE_DIR}/vmlinux-${KERNEL_TYPE}-${ARCH}"
|
||||
|
||||
test_system
|
||||
done
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
import gdb
|
||||
|
||||
|
||||
def test_command_kbase():
|
||||
pass # TODO
|
||||
|
||||
|
||||
def test_command_kchecksec():
|
||||
res = gdb.execute("kchecksec", to_string=True)
|
||||
# TODO: do something with res
|
||||
|
||||
|
||||
def test_command_kcmdline():
|
||||
res = gdb.execute("kcmdline", to_string=True)
|
||||
# TODO: do something with res
|
||||
|
||||
|
||||
def test_command_kconfig():
|
||||
res = gdb.execute("kconfig", to_string=True)
|
||||
assert "CONFIG_IKCONFIG = y" in res
|
||||
|
||||
res = gdb.execute("kconfig IKCONFIG", to_string=True)
|
||||
assert "CONFIG_IKCONFIG = y" in res
|
||||
|
||||
|
||||
def test_command_kversion():
|
||||
res = gdb.execute("kversion", to_string=True)
|
||||
assert "Linux version" in res
|
||||
|
||||
|
||||
def test_command_slab_list():
|
||||
res = gdb.execute("slab list", to_string=True)
|
||||
assert "kmalloc" in res
|
||||
|
||||
|
||||
def test_command_slab_info():
|
||||
pass # TODO
|
||||
|
|
@ -1,27 +1,13 @@
|
|||
import traceback
|
||||
|
||||
import gdb
|
||||
|
||||
import pwndbg
|
||||
import pwndbg.commands.kconfig
|
||||
|
||||
gdb.execute("break start_kernel")
|
||||
gdb.execute("continue")
|
||||
|
||||
try:
|
||||
pwndbg.commands.kconfig.kconfig()
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
exit(1)
|
||||
|
||||
|
||||
try:
|
||||
def test_gdblib_kernel_krelease():
|
||||
release_ver = pwndbg.gdblib.kernel.krelease()
|
||||
# release should be int tuple of form (major, minor, patch) or (major, minor)
|
||||
assert len(release_ver) >= 2
|
||||
release_str = "Linux version " + ".".join([str(x) for x in release_ver])
|
||||
assert release_str in pwndbg.gdblib.kernel.kversion()
|
||||
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
exit(1)
|
||||
|
||||
def test_gdblib_kernel_is_kaslr_enabled():
|
||||
pwndbg.gdblib.kernel.is_kaslr_enabled()
|
||||
Loading…
Reference in New Issue