From 1438fc06169cb775ce434e5183d84cc98911a179 Mon Sep 17 00:00:00 2001 From: OBarronCS <55004530+OBarronCS@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:49:45 -0700 Subject: [PATCH] Qemu user test structure (#2275) * Initial version of qemu-user tests * Refactor testing files to reduce file duplication, introduce qemu-user-tests * lint and edit github actions workflow file. Move old qemu-user tests to seperate directory * Add iproute2 so ss command is available * test ubuntu 24 * funkiness with current working directory... * Further remote old test_qemu.sh and integrate into a Pytest fixture * lint * Disable ASLR, add test for aarch64 jumps * Use Popen.kill() function to make sure it closes. Co-authored-by: Disconnect3d * qemu.kill() on the other fixture as well * comment * comment * lint * system test path stuff * remove old try-catch block * revert * revert path change * Use os._exit to pass return code, and move qemu-user tests above system tests because they run significantly faster * lint * Flush stdout before os._exit * Comment out flaky check for the address of main in old qemu tests * rename qemu-user to cross-arch * rename qemu-user to cross-arch and hotfix to not run pytest when cross-arch is used * remove todo comment * another comment * Test pwndbg.gdblib.symbol.address is not None and revert setarch -R * Revert os.exit change * Revert os.exit change * Revert os.exit change * readd os.exit in new exit places * lint * rebase * delete file introduced in rebase * break up tests into 3 files to invoke separately. Update GitHub workflow, remove code duplication in existing test * code coverage * fix code coverage * lint * test difference between Ubuntu 22 and 24 in Kernel tests * lint --------- Co-authored-by: Disconnect3d --- .github/workflows/tests.yml | 38 ++++- docker-compose.yml | 9 ++ pwndbg/gdblib/disasm/__init__.py | 30 ++++ qemu-tests.sh | 5 + setup-dev.sh | 6 +- tests.sh | 20 +-- tests/gdb-tests/tests/utils.py | 2 +- tests/{gdb-tests => }/pytests_collect.py | 7 +- tests/{gdb-tests => }/pytests_launcher.py | 7 +- tests/qemu-tests/binaries/Makefile | 15 -- tests/qemu-tests/conftest.py | 105 ++++++++++++++ tests/qemu-tests/pytests_collect.py | 36 ----- tests/qemu-tests/pytests_launcher.py | 32 ----- tests/qemu-tests/test_qemu.sh | 60 -------- tests/qemu-tests/tests.sh | 8 +- tests/qemu-tests/tests/user/__init__.py | 3 + tests/qemu-tests/tests/user/binaries/Makefile | 19 +++ .../tests/user/binaries/__init__.py | 9 ++ .../user}/binaries/reference-binary.aarch64.c | 0 .../user}/binaries/reference-binary.riscv64.c | 0 .../tests/user/test_aarch64_example.py | 136 ++++++++++++++++++ ...t_aarch64.py => test_aarch64_reference.py} | 17 ++- ...t_riscv64.py => test_riscv64_reference.py} | 18 ++- tests/{gdb-tests => }/tests.py | 113 ++++++++++++--- unit-tests.sh | 20 +++ 25 files changed, 501 insertions(+), 214 deletions(-) create mode 100755 qemu-tests.sh rename tests/{gdb-tests => }/pytests_collect.py (81%) rename tests/{gdb-tests => }/pytests_launcher.py (75%) delete mode 100644 tests/qemu-tests/binaries/Makefile create mode 100644 tests/qemu-tests/conftest.py delete mode 100644 tests/qemu-tests/pytests_collect.py delete mode 100644 tests/qemu-tests/pytests_launcher.py delete mode 100755 tests/qemu-tests/test_qemu.sh create mode 100644 tests/qemu-tests/tests/user/__init__.py create mode 100644 tests/qemu-tests/tests/user/binaries/Makefile create mode 100644 tests/qemu-tests/tests/user/binaries/__init__.py rename tests/qemu-tests/{ => tests/user}/binaries/reference-binary.aarch64.c (100%) rename tests/qemu-tests/{ => tests/user}/binaries/reference-binary.riscv64.c (100%) create mode 100644 tests/qemu-tests/tests/user/test_aarch64_example.py rename tests/qemu-tests/tests/user/{test_aarch64.py => test_aarch64_reference.py} (75%) rename tests/qemu-tests/tests/user/{test_riscv64.py => test_riscv64_reference.py} (69%) rename tests/{gdb-tests => }/tests.py (63%) create mode 100755 unit-tests.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6aa15507..984e3321 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,6 +51,7 @@ jobs: mkdir .cov sudo sysctl -w kernel.yama.ptrace_scope=0 ./tests.sh --cov + ./unit-tests.sh --cov - name: Process coverage data if: matrix.os == 'ubuntu-22.04' @@ -62,6 +63,36 @@ jobs: if: matrix.os == 'ubuntu-22.04' uses: codecov/codecov-action@v3 + qemu-user-tests: + runs-on: [ubuntu-24.04] + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Cache for pip + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ matrix.os }}-cache-pip + + - name: Install dependencies + run: | + ./setup.sh + ./setup-dev.sh + mkdir .cov + + - name: Run cross-architecture tests + run: | + ./qemu-tests.sh --cov + + - name: Process coverage data + run: | + ./.venv/bin/coverage combine + ./.venv/bin/coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + qemu-tests: runs-on: [ubuntu-22.04] timeout-minutes: 30 @@ -79,7 +110,7 @@ jobs: ./setup.sh ./setup-dev.sh mkdir .cov - + - name: Set up cache for QEMU images id: qemu-cache uses: actions/cache@v3 @@ -98,11 +129,6 @@ jobs: sudo sysctl -w kernel.yama.ptrace_scope=0 ./tests.sh --cov - - name: Run qemu-user tests - working-directory: ./tests/qemu-tests - run: | - ./test_qemu.sh - - name: Process coverage data run: | ./.venv/bin/coverage combine diff --git a/docker-compose.yml b/docker-compose.yml index c429e621..c41d860a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,15 @@ services: dockerfile: Dockerfile args: image: ubuntu:22.04 + + ubuntu24.04: + <<: *base-spec + build: + context: . + dockerfile: Dockerfile + args: + image: ubuntu:24.04 + debian11: <<: *base-spec diff --git a/pwndbg/gdblib/disasm/__init__.py b/pwndbg/gdblib/disasm/__init__.py index faabdd00..59d3cf9b 100644 --- a/pwndbg/gdblib/disasm/__init__.py +++ b/pwndbg/gdblib/disasm/__init__.py @@ -332,6 +332,36 @@ def can_run_first_emulate() -> bool: first_time_emulate = True +def no_emulate_one(): + result = near(pwndbg.gdblib.regs.pc, emulate=False, show_prev_insns=False) + if result: + return result[0][0] + return None + + +def emulate_one(): + result = near(pwndbg.gdblib.regs.pc, emulate=True, show_prev_insns=False) + if result: + return result[0][0] + return None + + +def one_with_config(): + """ + Returns a single Pwndbg Instruction at the current PC. + + Emulation determined by the `pwndbg.config.emulate` setting. + """ + result = near( + pwndbg.gdblib.regs.pc, + emulate=bool(not pwndbg.config.emulate == "off"), + show_prev_insns=False, + ) + if result: + return result[0][0] + return None + + # Return (list of PwndbgInstructions, index in list where instruction.address = passed in address) def near( address, instructions=1, emulate=False, show_prev_insns=True, use_cache=False, linear=False diff --git a/qemu-tests.sh b/qemu-tests.sh new file mode 100755 index 00000000..ae816d65 --- /dev/null +++ b/qemu-tests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +(cd tests && python3 tests.py -t cross-arch $@) +exit_code=$? +exit $exit_code diff --git a/setup-dev.sh b/setup-dev.sh index 8007ba0f..af3d8755 100755 --- a/setup-dev.sh +++ b/setup-dev.sh @@ -97,11 +97,15 @@ install_apt() { gdb-multiarch \ parallel \ netcat-openbsd \ + iproute2 \ qemu-system-x86 \ qemu-system-arm \ qemu-user \ gcc-aarch64-linux-gnu \ - gcc-riscv64-linux-gnu + gcc-riscv64-linux-gnu \ + gcc-arm-linux-gnueabihf \ + gcc-mips-linux-gnu \ + gcc-mips64-linux-gnuabi64 if [[ "$1" != "" && "$1" != "20.04" ]]; then sudo apt install shfmt diff --git a/tests.sh b/tests.sh index 5db5d874..13e18f1b 100755 --- a/tests.sh +++ b/tests.sh @@ -1,24 +1,6 @@ #!/usr/bin/env bash # Run integration tests -(cd tests/gdb-tests && python3 tests.py $@) +(cd tests && python3 tests.py $@) exit_code=$? - -COV=0 -# Run unit tests -for arg in "$@"; do - if [ "$arg" == "--cov" ]; then - COV=1 - break - fi -done - -if [ $COV -eq 1 ]; then - coverage run -m pytest tests/unit-tests -else - pytest tests/unit-tests -fi - -exit_code=$((exit_code + $?)) - exit $exit_code diff --git a/tests/gdb-tests/tests/utils.py b/tests/gdb-tests/tests/utils.py index 9d2a4f7a..b569e982 100644 --- a/tests/gdb-tests/tests/utils.py +++ b/tests/gdb-tests/tests/utils.py @@ -5,7 +5,7 @@ import os import re import subprocess -gdb_init_path = os.environ.get("GDB_INIT_PATH", "../../gdbinit.py") +gdb_init_path = os.environ.get("GDB_INIT_PATH", "../gdbinit.py") def run_gdb_with_script( diff --git a/tests/gdb-tests/pytests_collect.py b/tests/pytests_collect.py similarity index 81% rename from tests/gdb-tests/pytests_collect.py rename to tests/pytests_collect.py index d4ce7bb2..61f4ce16 100644 --- a/tests/gdb-tests/pytests_collect.py +++ b/tests/pytests_collect.py @@ -5,7 +5,12 @@ import sys import pytest -TESTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests") +TESTS_PATH = os.environ.get("TESTS_PATH") + +if TESTS_PATH is None: + print("'TESTS_PATH' environment variable not set. Failed to collect tests.") + sys.stdout.flush() + os._exit(1) class CollectTestFunctionNames: diff --git a/tests/gdb-tests/pytests_launcher.py b/tests/pytests_launcher.py similarity index 75% rename from tests/gdb-tests/pytests_launcher.py rename to tests/pytests_launcher.py index 0aa81401..23e5a3df 100644 --- a/tests/gdb-tests/pytests_launcher.py +++ b/tests/pytests_launcher.py @@ -3,6 +3,7 @@ from __future__ import annotations import os import sys +import coverage import pytest use_pdb = os.environ.get("USE_PDB") == "1" @@ -29,4 +30,8 @@ if return_code != 0: print("If you want to debug tests locally, run ./tests.sh with the --pdb flag") print("-" * 80) -sys.exit(return_code) +# We must call this to ensure the code coverage file writes get flushed +# https://github.com/nedbat/coveragepy/issues/310 +coverage._atexit() +sys.stdout.flush() +os._exit(return_code) diff --git a/tests/qemu-tests/binaries/Makefile b/tests/qemu-tests/binaries/Makefile deleted file mode 100644 index c2508d39..00000000 --- a/tests/qemu-tests/binaries/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -.PHONY: all -all: reference-binary.aarch64.out reference-binary.riscv64.out - -reference-binary.aarch64.out : reference-binary.aarch64.c - @echo "[+] Building '$@'" - @aarch64-linux-gnu-gcc $(CFLAGS) $(EXTRA_FLAGS) -w -o $@ $? $(LDFLAGS) - -# apt install crossbuild-essential-riscv64 -reference-binary.riscv64.out : reference-binary.riscv64.c - @echo "[+] Building '$@'" - @riscv64-linux-gnu-gcc -march=rv64gc -mabi=lp64d -g $(CFLAGS) $(EXTRA_FLAGS) -w -o $@ $? $(LDFLAGS) - -clean: - rm reference-binary.aarch64.out - rm reference-binary.riscv64.out \ No newline at end of file diff --git a/tests/qemu-tests/conftest.py b/tests/qemu-tests/conftest.py new file mode 100644 index 00000000..4be40867 --- /dev/null +++ b/tests/qemu-tests/conftest.py @@ -0,0 +1,105 @@ +""" +This file should consist of global test fixtures. +""" + +from __future__ import annotations + +import os +import subprocess +import sys + +import gdb +import pytest +from pwn import context +from pwn import make_elf_from_assembly + +_start_binary_called = False + +QEMU_PORT = os.environ.get("QEMU_PORT") + + +@pytest.fixture +def qemu_assembly_run(): + """ + Returns function that launches given binary with 'starti' command + + The `path` is returned from `make_elf_from_assembly` (provided by pwntools) + """ + + qemu: subprocess.Popen = None + + if QEMU_PORT is None: + print("'QEMU_PORT' environment variable not set") + sys.stdout.flush() + os._exit(1) + + def _start_binary(asm: str, arch: str, *args): + nonlocal qemu + + context.arch = arch + binary_tmp_path = make_elf_from_assembly(asm) + + qemu = subprocess.Popen( + [ + f"qemu-{arch}", + "-g", + f"{QEMU_PORT}", + f"{binary_tmp_path}", + ] + ) + + gdb.execute(f"target remote :{QEMU_PORT}") + gdb.execute("set exception-verbose on") + + global _start_binary_called + # if _start_binary_called: + # raise Exception('Starting more than one binary is not supported in pwndbg tests.') + + _start_binary_called = True + + yield _start_binary + + qemu.kill() + + +@pytest.fixture +def qemu_start_binary(): + """ + Returns function that launches given binary with 'starti' command + + Argument `path` is the path to the binary + """ + + qemu: subprocess.Popen = None + + if QEMU_PORT is None: + print("'QEMU_PORT' environment variable not set") + sys.stdout.flush() + os._exit(1) + + def _start_binary(path: str, arch: str, *args): + nonlocal qemu + + qemu = subprocess.Popen( + [ + f"qemu-{arch}", + "-L", + f"/usr/{arch}-linux-gnu/", + "-g", + f"{QEMU_PORT}", + f"{path}", + ] + ) + + gdb.execute(f"target remote :{QEMU_PORT}") + gdb.execute("set exception-verbose on") + + global _start_binary_called + # if _start_binary_called: + # raise Exception('Starting more than one binary is not supported in pwndbg tests.') + + _start_binary_called = True + + yield _start_binary + + qemu.kill() diff --git a/tests/qemu-tests/pytests_collect.py b/tests/qemu-tests/pytests_collect.py deleted file mode 100644 index d3839b5b..00000000 --- a/tests/qemu-tests/pytests_collect.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -import os -import sys - -import pytest - -TESTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests/system") - - -class CollectTestFunctionNames: - """See https://github.com/pytest-dev/pytest/issues/2039#issuecomment-257753269""" - - def __init__(self): - self.collected = [] - - def pytest_collection_modifyitems(self, items): - for item in items: - self.collected.append(item.nodeid) - - -collector = CollectTestFunctionNames() -rv = pytest.main(["--collect-only", TESTS_PATH], plugins=[collector]) - -if rv == pytest.ExitCode.INTERRUPTED: - print("Failed to collect all tests, perhaps there is a syntax error in one of test files?") - sys.stdout.flush() - os._exit(1) - - -print("Listing collected tests:") -for nodeid in collector.collected: - print("Test:", nodeid) - -# easy way to exit GDB session -sys.exit(0) diff --git a/tests/qemu-tests/pytests_launcher.py b/tests/qemu-tests/pytests_launcher.py deleted file mode 100644 index 0aa81401..00000000 --- a/tests/qemu-tests/pytests_launcher.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import os -import sys - -import pytest - -use_pdb = os.environ.get("USE_PDB") == "1" - -sys._pwndbg_unittest_run = True - -CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) - -test = os.environ["PWNDBG_LAUNCH_TEST"] - -test = os.path.join(CURRENT_DIR, test) - -args = [test, "-vvv", "-s", "--showlocals", "--color=yes"] - -if use_pdb: - args.append("--pdb") - -print(f"Launching pytest with args: {args}") - -return_code = pytest.main(args) - -if return_code != 0: - print("-" * 80) - print("If you want to debug tests locally, run ./tests.sh with the --pdb flag") - print("-" * 80) - -sys.exit(return_code) diff --git a/tests/qemu-tests/test_qemu.sh b/tests/qemu-tests/test_qemu.sh deleted file mode 100755 index 3a8411fa..00000000 --- a/tests/qemu-tests/test_qemu.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -make -C binaries - -ROOT_DIR="$(readlink -f ../../)" -GDB_INIT_PATH="$ROOT_DIR/gdbinit.py" -COVERAGERC_PATH="$ROOT_DIR/pyproject.toml" - -handle_sigint() { - echo "Exiting..." >&2 - pkill qemu-aarch64 - pkill qemu-riscv64 - exit 1 -} -trap handle_sigint SIGINT - -gdb_load_pwndbg=(--command "$GDB_INIT_PATH" -ex "set exception-verbose on") -run_gdb() { - COVERAGE_FILE=$ROOT_DIR/.cov/coverage \ - COVERAGE_PROCESS_START=$COVERAGERC_PATH \ - PWNDBG_DISABLE_COLORS=1 \ - gdb-multiarch --silent --nx --nh "${gdb_load_pwndbg[@]}" "$@" -ex "quit" 2> /dev/null - return $? -} - -test_arch() { - local arch="$1" - - qemu-${arch} \ - -g 1234 \ - -L /usr/${arch}-linux-gnu/ \ - ./binaries/reference-binary.${arch}.out & - - run_gdb \ - -ex "set sysroot /usr/${arch}-linux-gnu/" \ - -ex "file ./binaries/reference-binary.${arch}.out" \ - -ex 'py import coverage;coverage.process_startup()' \ - -ex "target remote :1234" \ - -ex "source ./tests/user/test_${arch}.py" - local result=$? - pkill qemu-${arch} - return $result -} - -ARCHS=("aarch64" "riscv64") - -FAILED_TESTS=() -for arch in "${ARCHS[@]}"; do - test_arch "$arch" - if [ $? -ne 0 ]; then - FAILED_TESTS+=("$arch") - fi -done - -if [ "${#FAILED_TESTS[@]}" -ne 0 ]; then - echo "" - echo "Failing tests: ${FAILED_TESTS[@]}" - echo "" - exit 1 -fi diff --git a/tests/qemu-tests/tests.sh b/tests/qemu-tests/tests.sh index d882b2d5..123ddbfd 100755 --- a/tests/qemu-tests/tests.sh +++ b/tests/qemu-tests/tests.sh @@ -123,8 +123,8 @@ run_gdb() { # NOTE: We run tests under GDB sessions and because of some cleanup/tests dependencies problems # we decided to run each test in a separate GDB session -gdb_args=(--command pytests_collect.py) -TESTS_COLLECT_OUTPUT=$(run_gdb "x86_64" "${gdb_args[@]}") +gdb_args=(--command ../pytests_collect.py) +TESTS_COLLECT_OUTPUT=$(TESTS_PATH="$ROOT_DIR/tests/qemu-tests/tests/system" run_gdb "x86_64" "${gdb_args[@]}") if [ $? -eq 1 ]; then echo -E "$TESTS_COLLECT_OUTPUT" @@ -155,7 +155,7 @@ run_test() { local arch="$4" gdb_connect_qemu=(-ex "file ${IMAGE_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}") - gdb_args=("${gdb_connect_qemu[@]}" --command pytests_launcher.py) + gdb_args=("${gdb_connect_qemu[@]}" --command ../pytests_launcher.py) if [ ${RUN_CODECOV} -ne 0 ]; then gdb_args=(-ex 'py import coverage;coverage.process_startup()' "${gdb_args[@]}") fi @@ -164,7 +164,7 @@ run_test() { COVERAGE_FILE=$ROOT_DIR/.cov/coverage \ COVERAGE_PROCESS_START=$COVERAGERC_PATH \ USE_PDB="${USE_PDB}" \ - PWNDBG_LAUNCH_TEST="${test_case}" \ + PWNDBG_LAUNCH_TEST="qemu-tests/${test_case}" \ PWNDBG_DISABLE_COLORS=1 \ PWNDBG_ARCH="${arch}" \ PWNDBG_KERNEL_TYPE="${kernel_type}" \ diff --git a/tests/qemu-tests/tests/user/__init__.py b/tests/qemu-tests/tests/user/__init__.py new file mode 100644 index 00000000..92db9c3a --- /dev/null +++ b/tests/qemu-tests/tests/user/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from . import binaries diff --git a/tests/qemu-tests/tests/user/binaries/Makefile b/tests/qemu-tests/tests/user/binaries/Makefile new file mode 100644 index 00000000..c125165d --- /dev/null +++ b/tests/qemu-tests/tests/user/binaries/Makefile @@ -0,0 +1,19 @@ +.PHONY: all +all: reference-binary.aarch64.out reference-binary.riscv64.out + +%.aarch64.out : %.aarch64.c + @echo "[+] Building '$@'" + @aarch64-linux-gnu-gcc $(CFLAGS) $(EXTRA_FLAGS) -w -o $@ $< $(LDFLAGS) + +%.riscv64.out : %.riscv64.c + @echo "[+] Building '$@'" + @riscv64-linux-gnu-gcc -march=rv64gc -mabi=lp64d -g $(CFLAGS) $(EXTRA_FLAGS) -w -o $@ $? $(LDFLAGS) + +AARCH64_SOURCES := $(wildcard *.aarch64.c) +AARCH64_TARGETS := $(AARCH64_SOURCES:.aarch64.c=.aarch64.out) + +RISCV64_SOURCES := $(wildcard *.riscv64.c) +RISCV64_TARGETS := $(RISCV64_SOURCES:.riscv64.c=.riscv64.out) + +clean: + rm -f *.aarch64.out *.x86_64.out *.arm.out diff --git a/tests/qemu-tests/tests/user/binaries/__init__.py b/tests/qemu-tests/tests/user/binaries/__init__.py new file mode 100644 index 00000000..5054f3ac --- /dev/null +++ b/tests/qemu-tests/tests/user/binaries/__init__.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import os + +path = os.path.dirname(__file__) + + +def get(x): + return os.path.join(path, x) diff --git a/tests/qemu-tests/binaries/reference-binary.aarch64.c b/tests/qemu-tests/tests/user/binaries/reference-binary.aarch64.c similarity index 100% rename from tests/qemu-tests/binaries/reference-binary.aarch64.c rename to tests/qemu-tests/tests/user/binaries/reference-binary.aarch64.c diff --git a/tests/qemu-tests/binaries/reference-binary.riscv64.c b/tests/qemu-tests/tests/user/binaries/reference-binary.riscv64.c similarity index 100% rename from tests/qemu-tests/binaries/reference-binary.riscv64.c rename to tests/qemu-tests/tests/user/binaries/reference-binary.riscv64.c diff --git a/tests/qemu-tests/tests/user/test_aarch64_example.py b/tests/qemu-tests/tests/user/test_aarch64_example.py new file mode 100644 index 00000000..14ab4e21 --- /dev/null +++ b/tests/qemu-tests/tests/user/test_aarch64_example.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import gdb +from capstone.arm64_const import ARM64_INS_BL + +import pwndbg.gdblib.disasm +import pwndbg.gdblib.nearpc +from pwndbg.gdblib.disasm.instruction import InstructionCondition + +AARCH64_GRACEFUL_EXIT = """ +mov x0, 0 +mov x8, 93 +svc 0 +""" + +SIMPLE_FUNCTION = f""" + +bl my_function +b end + +my_function: + ret + +end: +{AARCH64_GRACEFUL_EXIT} +""" + + +def test_syscall_annotation(qemu_assembly_run): + """ """ + qemu_assembly_run(AARCH64_GRACEFUL_EXIT, "aarch64") + + instructions = pwndbg.gdblib.disasm.near( + address=pwndbg.gdblib.regs.pc, instructions=3, emulate=True + )[0] + future_syscall_ins = instructions[2] + + assert future_syscall_ins.syscall == 93 + assert future_syscall_ins.syscall_name == "exit" + + gdb.execute("stepuntilasm svc") + + # Both for emulation and non-emulation, ensure a syscall at current PC gets enriched + instructions = pwndbg.gdblib.disasm.emulate_one(), pwndbg.gdblib.disasm.no_emulate_one() + + for i in instructions: + assert i.syscall == 93 + assert i.syscall_name == "exit" + + +def test_branch_enhancement(qemu_assembly_run): + qemu_assembly_run(SIMPLE_FUNCTION, "aarch64") + + instruction = pwndbg.gdblib.disasm.one_with_config() + + assert instruction.id == ARM64_INS_BL + assert instruction.call_like + assert not instruction.is_conditional_jump + assert instruction.is_unconditional_jump + assert instruction.target_string == "my_function" + + +CONDITIONAL_JUMPS = f""" +mov x2, 0b1010 +mov x3, 0 + +cbz x3, A +nop + +A: +cbnz x2, B +nop + +B: +tbz x2, #0, C +nop + +C: +tbnz x2, #3, D +nop + +D: +cmp x2, x3 +b.eq E +nop + +E: +b.ne F +nop + +F: +{AARCH64_GRACEFUL_EXIT} +""" + + +def test_conditional_jumps(qemu_assembly_run): + qemu_assembly_run(CONDITIONAL_JUMPS, "aarch64") + + gdb.execute("stepuntilasm cbz") + ins = pwndbg.gdblib.disasm.one_with_config() + + assert ins.condition == InstructionCondition.TRUE + + gdb.execute("si") + ins = pwndbg.gdblib.disasm.one_with_config() + + assert ins.condition == InstructionCondition.TRUE + + gdb.execute("si") + ins = pwndbg.gdblib.disasm.one_with_config() + + assert ins.condition == InstructionCondition.TRUE + + gdb.execute("si") + ins = pwndbg.gdblib.disasm.one_with_config() + + assert ins.condition == InstructionCondition.TRUE + + gdb.execute("si") + gdb.execute("si") + + ins = pwndbg.gdblib.disasm.one_with_config() + + assert ins.condition == InstructionCondition.FALSE + + gdb.execute("si") + gdb.execute("si") + + ins = pwndbg.gdblib.disasm.one_with_config() + + assert ins.condition == InstructionCondition.TRUE + + +def test_conditional_jumps_no_emulate(qemu_assembly_run): + gdb.execute("set emulate off") + test_conditional_jumps(qemu_assembly_run) diff --git a/tests/qemu-tests/tests/user/test_aarch64.py b/tests/qemu-tests/tests/user/test_aarch64_reference.py similarity index 75% rename from tests/qemu-tests/tests/user/test_aarch64.py rename to tests/qemu-tests/tests/user/test_aarch64_reference.py index 29a33433..c8468f1e 100644 --- a/tests/qemu-tests/tests/user/test_aarch64.py +++ b/tests/qemu-tests/tests/user/test_aarch64_reference.py @@ -1,15 +1,17 @@ from __future__ import annotations -import sys -import traceback - import gdb +import user -import pwndbg +import pwndbg.gdblib.symbol -try: +REFERENCE_BINARY = user.binaries.get("reference-binary.aarch64.out") + + +def test_aarch64_reference(qemu_start_binary): + qemu_start_binary(REFERENCE_BINARY, "aarch64") gdb.execute("break break_here") - assert pwndbg.gdblib.symbol.address("main") == 0x5500000A1C + assert pwndbg.gdblib.symbol.address("main") is not None gdb.execute("continue") gdb.execute("argv", to_string=True) @@ -35,6 +37,3 @@ try: gdb.execute("piebase", to_string=True) gdb.execute("nextret", to_string=True) -except AssertionError: - traceback.print_exc(file=sys.stdout) - sys.exit(1) diff --git a/tests/qemu-tests/tests/user/test_riscv64.py b/tests/qemu-tests/tests/user/test_riscv64_reference.py similarity index 69% rename from tests/qemu-tests/tests/user/test_riscv64.py rename to tests/qemu-tests/tests/user/test_riscv64_reference.py index 48961fbd..d23b78c0 100644 --- a/tests/qemu-tests/tests/user/test_riscv64.py +++ b/tests/qemu-tests/tests/user/test_riscv64_reference.py @@ -1,15 +1,17 @@ from __future__ import annotations -import sys -import traceback - import gdb +import user -import pwndbg +import pwndbg.gdblib.symbol -try: +REFERENCE_BINARY = user.binaries.get("reference-binary.riscv64.out") + + +def test_riscv64_reference(qemu_start_binary): + qemu_start_binary(REFERENCE_BINARY, "riscv64") gdb.execute("break 4") - assert pwndbg.gdblib.symbol.address("main") == 0x4000000668 + assert pwndbg.gdblib.symbol.address("main") is not None gdb.execute("continue") gdb.execute("stepuntilasm jalr") @@ -26,7 +28,3 @@ try: gdb.execute("stepi") assembly = gdb.execute("nearpc 0", to_string=True) assert assembly.split()[2] == target, (assembly.split()[2], target) - -except AssertionError: - traceback.print_exc(file=sys.stdout) - sys.exit(1) diff --git a/tests/gdb-tests/tests.py b/tests/tests.py similarity index 63% rename from tests/gdb-tests/tests.py rename to tests/tests.py index 10851f7f..995cbd23 100644 --- a/tests/gdb-tests/tests.py +++ b/tests/tests.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import concurrent.futures import os +import random import re import subprocess import sys @@ -11,7 +12,7 @@ from subprocess import CompletedProcess from typing import List from typing import Tuple -root_dir = os.path.realpath("../../") +root_dir = os.path.realpath("../") def ensureZigPath(): @@ -24,27 +25,66 @@ def ensureZigPath(): def makeBinaries(): try: - subprocess.check_call(["make", "all"], cwd="./tests/binaries") + subprocess.check_call(["make", "all"], cwd="./gdb-tests/tests/binaries") except subprocess.CalledProcessError: exit(1) -def run_gdb(gdb_args: List[str], env=None, capture_output=True) -> CompletedProcess[str]: +def makeCrossArchBinaries(): + try: + subprocess.check_call(["make", "all"], cwd="./qemu-tests/tests/user/binaries") + except subprocess.CalledProcessError: + exit(1) + + +def open_ports(n: int) -> List[int]: + """ + Returns a list of `n` open ports + """ + try: + result = subprocess.run( + ["netstat", "-tuln"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if result.returncode != 0: + # If netstat not found, try ss + raise FileNotFoundError + except FileNotFoundError: + result = subprocess.run(["ss", "-tuln"], stdout=subprocess.PIPE) + + used_ports = set(re.findall(r":(\d+)", result.stdout.decode())) + used_ports = set(map(int, used_ports)) + + available_ports = [port for port in range(1024, 65536) if port not in used_ports] + return random.sample(available_ports, n) + + +def run_gdb( + gdb_binary: str, gdb_args: List[str], env=None, capture_output=True +) -> CompletedProcess[str]: env = os.environ if env is None else env return subprocess.run( - ["gdb", "--silent", "--nx", "--nh"] + gdb_args + ["--eval-command", "quit"], + [gdb_binary, "--silent", "--nx", "--nh"] + gdb_args + ["--eval-command", "quit"], env=env, capture_output=capture_output, text=True, ) -def getTestsList(collect_only: bool, test_name_filter: str, gdbinit_path: str) -> List[str]: +def getTestsList( + collect_only: bool, + test_name_filter: str, + gdb_binary: str, + gdbinit_path: str, + test_dir_path: str, +) -> List[str]: # NOTE: We run tests under GDB sessions and because of some cleanup/tests dependencies problems # we decided to run each test in a separate GDB session gdb_args = ["--init-command", gdbinit_path, "--command", "pytests_collect.py"] - result = run_gdb(gdb_args) + env = os.environ.copy() + env["TESTS_PATH"] = os.path.join(os.path.dirname(os.path.realpath(__file__)), test_dir_path) + + result = run_gdb(gdb_binary, gdb_args, env=env) tests_collect_output = result.stdout if result.returncode == 1: @@ -55,14 +95,14 @@ def getTestsList(collect_only: bool, test_name_filter: str, gdbinit_path: str) - exit(0) # Extract the test names from the output using regex - pattern = re.compile(r"tests/.*::.*") + pattern = re.compile(rf"{test_dir_path}.*::.*") matches = pattern.findall(tests_collect_output) tests_list = [match for match in matches if re.search(test_name_filter, match)] return tests_list def run_test( - test_case: str, args: argparse.Namespace, gdbinit_path: str + test_case: str, args: argparse.Namespace, gdb_binary: str, gdbinit_path: str, port: int = None ) -> Tuple[CompletedProcess[str], str]: gdb_args = ["--init-command", gdbinit_path, "--command", "pytests_launcher.py"] if args.cov: @@ -82,11 +122,20 @@ def run_test( env["USE_PDB"] = "1" env["PWNDBG_LAUNCH_TEST"] = test_case env["PWNDBG_DISABLE_COLORS"] = "1" - result = run_gdb(gdb_args, env=env, capture_output=not args.serial) + if port is not None: + env["QEMU_PORT"] = str(port) + result = run_gdb(gdb_binary, gdb_args, env=env, capture_output=not args.serial) return (result, test_case) -def run_tests_and_print_stats(tests_list: List[str], args: argparse.Namespace, gdbinit_path: str): +def run_tests_and_print_stats( + tests_list: List[str], + args: argparse.Namespace, + gdb_binary: str, + gdbinit_path: str, + test_dir_path: str, + ports: List[int] = [], +): start = time.time() test_results: List[Tuple[CompletedProcess[str], str]] = [] @@ -96,7 +145,7 @@ def run_tests_and_print_stats(tests_list: List[str], args: argparse.Namespace, g content = process.stdout # Extract the test name and result using regex - testname = re.search(r"^(tests/[^ ]+)", content, re.MULTILINE)[0] + testname = re.search(rf"^({test_dir_path}/[^ ]+)", content, re.MULTILINE)[0] result = re.search( r"(\x1b\[3.m(PASSED|FAILED|SKIPPED|XPASS|XFAIL)\x1b\[0m)", content, re.MULTILINE )[0] @@ -109,16 +158,21 @@ def run_tests_and_print_stats(tests_list: List[str], args: argparse.Namespace, g print("") print(content) + port_iterator = iter(ports) + if args.serial: - test_results = [run_test(test, args, gdbinit_path) for test in tests_list] + test_results = [ + run_test(test, args, gdb_binary, gdbinit_path, next(port_iterator, None)) + for test in tests_list + ] else: print("") print("Running tests in parallel") with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: for test in tests_list: - executor.submit(run_test, test, args, gdbinit_path).add_done_callback( - lambda future: handle_parallel_test_result(future.result()) - ) + executor.submit( + run_test, test, args, gdb_binary, gdbinit_path, next(port_iterator, None) + ).add_done_callback(lambda future: handle_parallel_test_result(future.result())) end = time.time() seconds = int(end - start) @@ -145,6 +199,8 @@ def run_tests_and_print_stats(tests_list: List[str], args: argparse.Namespace, g def parse_args(): parser = argparse.ArgumentParser(description="Run tests.") + parser.add_argument("-t", "--type", dest="type", choices=["gdb", "cross-arch"], default="gdb") + parser.add_argument( "-p", "--pdb", @@ -177,6 +233,8 @@ def parse_args(): return parser.parse_args() +TEST_FOLDER_NAME = {"gdb": "gdb-tests/tests", "cross-arch": "qemu-tests/tests/user"} + if __name__ == "__main__": args = parse_args() if args.cov: @@ -192,7 +250,24 @@ if __name__ == "__main__": os.environ["GDB_INIT_PATH"] = gdbinit_path else: gdbinit_path = os.path.join(root_dir, "gdbinit.py") - ensureZigPath() - makeBinaries() - tests: List[str] = getTestsList(args.collect_only, args.test_name_filter, gdbinit_path) - run_tests_and_print_stats(tests, args, gdbinit_path) + + gdb_binary = "gdb" + + if args.type == "gdb": + ensureZigPath() + makeBinaries() + else: + makeCrossArchBinaries() + gdb_binary = "gdb-multiarch" + + test_dir_path = TEST_FOLDER_NAME[args.type] + + tests: List[str] = getTestsList( + args.collect_only, args.test_name_filter, gdb_binary, gdbinit_path, test_dir_path + ) + + ports = [] + if args.type == "cross-arch": + ports = open_ports(len(tests)) + + run_tests_and_print_stats(tests, args, gdb_binary, gdbinit_path, test_dir_path, ports) diff --git a/unit-tests.sh b/unit-tests.sh new file mode 100755 index 00000000..a002377f --- /dev/null +++ b/unit-tests.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +COV=0 +# Run unit tests +for arg in "$@"; do + if [ "$arg" == "--cov" ]; then + COV=1 + break + fi +done + +if [ $COV -eq 1 ]; then + coverage run -m pytest tests/unit-tests +else + pytest tests/unit-tests +fi + +exit_code=$((exit_code + $?)) + +exit $exit_code