Qemu user test structure (#2275)

* Initial version of qemu-user tests

* Refactor testing files to reduce file duplication, introduce qemu-user-tests

* lint and edit github actions workflow file. Move old qemu-user tests to seperate directory

* Add iproute2 so ss command is available

* test ubuntu 24

* funkiness with current working directory...

* Further remote old test_qemu.sh and integrate into a Pytest fixture

* lint

* Disable ASLR, add test for aarch64 jumps

* Use Popen.kill() function to make sure it closes.

Co-authored-by: Disconnect3d <dominik.b.czarnota@gmail.com>

* qemu.kill() on the other fixture as well

* comment

* comment

* lint

* system test path stuff

* remove old try-catch block

* revert

* revert path change

* Use os._exit to pass return code, and move qemu-user tests above system tests because they run significantly faster

* lint

* Flush stdout before os._exit

* Comment out flaky check for the address of main in old qemu tests

* rename qemu-user to cross-arch

* rename qemu-user to cross-arch and hotfix to not run pytest when
cross-arch is used

* remove todo comment

* another comment

* Test pwndbg.gdblib.symbol.address is not None and revert setarch -R

* Revert os.exit change

* Revert os.exit change

* Revert os.exit change

* readd os.exit in new exit places

* lint

* rebase

* delete file introduced in rebase

* break up tests into 3 files to invoke separately. Update GitHub workflow, remove code duplication in existing test

* code coverage

* fix code coverage

* lint

* test difference between Ubuntu 22 and 24 in Kernel tests

* lint

---------

Co-authored-by: Disconnect3d <dominik.b.czarnota@gmail.com>

.github/workflows/tests.yml

@@ -51,6 +51,7 @@ jobs:
         mkdir .cov
         sudo sysctl -w kernel.yama.ptrace_scope=0
         ./tests.sh --cov
+        ./unit-tests.sh --cov
 
     - name: Process coverage data
       if: matrix.os == 'ubuntu-22.04'
@@ -62,6 +63,36 @@ jobs:
       if: matrix.os == 'ubuntu-22.04'
       uses: codecov/codecov-action@v3
 
+  qemu-user-tests:
+    runs-on: [ubuntu-24.04]
+    timeout-minutes: 20
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Cache for pip
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+    
+    - name: Install dependencies
+      run: |
+        ./setup.sh
+        ./setup-dev.sh
+        mkdir .cov
+
+    - name: Run cross-architecture tests
+      run: |
+        ./qemu-tests.sh --cov
+
+    - name: Process coverage data
+      run: |
+        ./.venv/bin/coverage combine
+        ./.venv/bin/coverage xml
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+
   qemu-tests:
     runs-on: [ubuntu-22.04]
     timeout-minutes: 30
@@ -98,11 +129,6 @@ jobs:
         sudo sysctl -w kernel.yama.ptrace_scope=0
         ./tests.sh --cov
 
-    - name: Run qemu-user tests
-      working-directory: ./tests/qemu-tests
-      run: |
-        ./test_qemu.sh
-
     - name: Process coverage data
       run: |
         ./.venv/bin/coverage combine

docker-compose.yml

@@ -28,6 +28,15 @@ services:
       args:
         image: ubuntu:22.04
   
+  ubuntu24.04:
+    <<: *base-spec
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        image: ubuntu:24.04
+
+
   debian11:
     <<: *base-spec
     build:

pwndbg/gdblib/disasm/__init__.py

@@ -332,6 +332,36 @@ def can_run_first_emulate() -> bool:
 first_time_emulate = True
 
 
+def no_emulate_one():
+    result = near(pwndbg.gdblib.regs.pc, emulate=False, show_prev_insns=False)
+    if result:
+        return result[0][0]
+    return None
+
+
+def emulate_one():
+    result = near(pwndbg.gdblib.regs.pc, emulate=True, show_prev_insns=False)
+    if result:
+        return result[0][0]
+    return None
+
+
+def one_with_config():
+    """
+    Returns a single Pwndbg Instruction at the current PC.
+
+    Emulation determined by the `pwndbg.config.emulate` setting.
+    """
+    result = near(
+        pwndbg.gdblib.regs.pc,
+        emulate=bool(not pwndbg.config.emulate == "off"),
+        show_prev_insns=False,
+    )
+    if result:
+        return result[0][0]
+    return None
+
+
 # Return (list of PwndbgInstructions, index in list where instruction.address = passed in address)
 def near(
     address, instructions=1, emulate=False, show_prev_insns=True, use_cache=False, linear=False

qemu-tests.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+(cd tests && python3 tests.py -t cross-arch $@)
+exit_code=$?
+exit $exit_code

setup-dev.sh

@@ -97,11 +97,15 @@ install_apt() {
         gdb-multiarch \
         parallel \
         netcat-openbsd \
+        iproute2 \
         qemu-system-x86 \
         qemu-system-arm \
         qemu-user \
         gcc-aarch64-linux-gnu \
-        gcc-riscv64-linux-gnu
+        gcc-riscv64-linux-gnu \
+        gcc-arm-linux-gnueabihf \
+        gcc-mips-linux-gnu \
+        gcc-mips64-linux-gnuabi64
 
     if [[ "$1" != "" && "$1" != "20.04" ]]; then
         sudo apt install shfmt

tests.sh

@@ -1,24 +1,6 @@
 #!/usr/bin/env bash
 
 # Run integration tests
-(cd tests/gdb-tests && python3 tests.py $@)
+(cd tests && python3 tests.py $@)
 exit_code=$?
-
-COV=0
-# Run unit tests
-for arg in "$@"; do
-    if [ "$arg" == "--cov" ]; then
-        COV=1
-        break
-    fi
-done
-
-if [ $COV -eq 1 ]; then
-    coverage run -m pytest tests/unit-tests
-else
-    pytest tests/unit-tests
-fi
-
-exit_code=$((exit_code + $?))
-
 exit $exit_code

tests/gdb-tests/tests/utils.py

@@ -5,7 +5,7 @@ import os
 import re
 import subprocess
 
-gdb_init_path = os.environ.get("GDB_INIT_PATH", "../../gdbinit.py")
+gdb_init_path = os.environ.get("GDB_INIT_PATH", "../gdbinit.py")
 
 
 def run_gdb_with_script(

tests/pytests_collect.py

@@ -5,7 +5,12 @@ import sys
 
 import pytest
 
-TESTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests")
+TESTS_PATH = os.environ.get("TESTS_PATH")
+
+if TESTS_PATH is None:
+    print("'TESTS_PATH' environment variable not set. Failed to collect tests.")
+    sys.stdout.flush()
+    os._exit(1)
 
 
 class CollectTestFunctionNames:

tests/pytests_launcher.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 import sys
 
+import coverage
 import pytest
 
 use_pdb = os.environ.get("USE_PDB") == "1"
@@ -29,4 +30,8 @@ if return_code != 0:
     print("If you want to debug tests locally, run ./tests.sh with the --pdb flag")
     print("-" * 80)
 
-sys.exit(return_code)
+# We must call this to ensure the code coverage file writes get flushed
+# https://github.com/nedbat/coveragepy/issues/310
+coverage._atexit()
+sys.stdout.flush()
+os._exit(return_code)

tests/qemu-tests/binaries/Makefile

@@ -1,15 +0,0 @@
-.PHONY: all
-all: reference-binary.aarch64.out reference-binary.riscv64.out
-
-reference-binary.aarch64.out : reference-binary.aarch64.c
-	@echo "[+] Building '$@'"
-	@aarch64-linux-gnu-gcc $(CFLAGS) $(EXTRA_FLAGS) -w -o $@ $? $(LDFLAGS)
-
-# apt install crossbuild-essential-riscv64
-reference-binary.riscv64.out : reference-binary.riscv64.c
-	@echo "[+] Building '$@'"
-	@riscv64-linux-gnu-gcc -march=rv64gc -mabi=lp64d -g $(CFLAGS) $(EXTRA_FLAGS) -w -o $@ $? $(LDFLAGS)
-
-clean:
-	rm reference-binary.aarch64.out
-	rm reference-binary.riscv64.out

tests/qemu-tests/conftest.py

@@ -0,0 +1,105 @@
+"""
+This file should consist of global test fixtures.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+
+import gdb
+import pytest
+from pwn import context
+from pwn import make_elf_from_assembly
+
+_start_binary_called = False
+
+QEMU_PORT = os.environ.get("QEMU_PORT")
+
+
+@pytest.fixture
+def qemu_assembly_run():
+    """
+    Returns function that launches given binary with 'starti' command
+
+    The `path` is returned from `make_elf_from_assembly` (provided by pwntools)
+    """
+
+    qemu: subprocess.Popen = None
+
+    if QEMU_PORT is None:
+        print("'QEMU_PORT' environment variable not set")
+        sys.stdout.flush()
+        os._exit(1)
+
+    def _start_binary(asm: str, arch: str, *args):
+        nonlocal qemu
+
+        context.arch = arch
+        binary_tmp_path = make_elf_from_assembly(asm)
+
+        qemu = subprocess.Popen(
+            [
+                f"qemu-{arch}",
+                "-g",
+                f"{QEMU_PORT}",
+                f"{binary_tmp_path}",
+            ]
+        )
+
+        gdb.execute(f"target remote :{QEMU_PORT}")
+        gdb.execute("set exception-verbose on")
+
+        global _start_binary_called
+        # if _start_binary_called:
+        #     raise Exception('Starting more than one binary is not supported in pwndbg tests.')
+
+        _start_binary_called = True
+
+    yield _start_binary
+
+    qemu.kill()
+
+
+@pytest.fixture
+def qemu_start_binary():
+    """
+    Returns function that launches given binary with 'starti' command
+
+    Argument `path` is the path to the binary
+    """
+
+    qemu: subprocess.Popen = None
+
+    if QEMU_PORT is None:
+        print("'QEMU_PORT' environment variable not set")
+        sys.stdout.flush()
+        os._exit(1)
+
+    def _start_binary(path: str, arch: str, *args):
+        nonlocal qemu
+
+        qemu = subprocess.Popen(
+            [
+                f"qemu-{arch}",
+                "-L",
+                f"/usr/{arch}-linux-gnu/",
+                "-g",
+                f"{QEMU_PORT}",
+                f"{path}",
+            ]
+        )
+
+        gdb.execute(f"target remote :{QEMU_PORT}")
+        gdb.execute("set exception-verbose on")
+
+        global _start_binary_called
+        # if _start_binary_called:
+        #     raise Exception('Starting more than one binary is not supported in pwndbg tests.')
+
+        _start_binary_called = True
+
+    yield _start_binary
+
+    qemu.kill()

tests/qemu-tests/pytests_collect.py

@@ -1,36 +0,0 @@
-from __future__ import annotations
-
-import os
-import sys
-
-import pytest
-
-TESTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests/system")
-
-
-class CollectTestFunctionNames:
-    """See https://github.com/pytest-dev/pytest/issues/2039#issuecomment-257753269"""
-
-    def __init__(self):
-        self.collected = []
-
-    def pytest_collection_modifyitems(self, items):
-        for item in items:
-            self.collected.append(item.nodeid)
-
-
-collector = CollectTestFunctionNames()
-rv = pytest.main(["--collect-only", TESTS_PATH], plugins=[collector])
-
-if rv == pytest.ExitCode.INTERRUPTED:
-    print("Failed to collect all tests, perhaps there is a syntax error in one of test files?")
-    sys.stdout.flush()
-    os._exit(1)
-
-
-print("Listing collected tests:")
-for nodeid in collector.collected:
-    print("Test:", nodeid)
-
-# easy way to exit GDB session
-sys.exit(0)

tests/qemu-tests/pytests_launcher.py

@@ -1,32 +0,0 @@
-from __future__ import annotations
-
-import os
-import sys
-
-import pytest
-
-use_pdb = os.environ.get("USE_PDB") == "1"
-
-sys._pwndbg_unittest_run = True
-
-CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
-
-test = os.environ["PWNDBG_LAUNCH_TEST"]
-
-test = os.path.join(CURRENT_DIR, test)
-
-args = [test, "-vvv", "-s", "--showlocals", "--color=yes"]
-
-if use_pdb:
-    args.append("--pdb")
-
-print(f"Launching pytest with args: {args}")
-
-return_code = pytest.main(args)
-
-if return_code != 0:
-    print("-" * 80)
-    print("If you want to debug tests locally, run ./tests.sh with the --pdb flag")
-    print("-" * 80)
-
-sys.exit(return_code)

tests/qemu-tests/test_qemu.sh

@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-
-make -C binaries
-
-ROOT_DIR="$(readlink -f ../../)"
-GDB_INIT_PATH="$ROOT_DIR/gdbinit.py"
-COVERAGERC_PATH="$ROOT_DIR/pyproject.toml"
-
-handle_sigint() {
-    echo "Exiting..." >&2
-    pkill qemu-aarch64
-    pkill qemu-riscv64
-    exit 1
-}
-trap handle_sigint SIGINT
-
-gdb_load_pwndbg=(--command "$GDB_INIT_PATH" -ex "set exception-verbose on")
-run_gdb() {
-    COVERAGE_FILE=$ROOT_DIR/.cov/coverage \
-        COVERAGE_PROCESS_START=$COVERAGERC_PATH \
-        PWNDBG_DISABLE_COLORS=1 \
-        gdb-multiarch --silent --nx --nh "${gdb_load_pwndbg[@]}" "$@" -ex "quit" 2> /dev/null
-    return $?
-}
-
-test_arch() {
-    local arch="$1"
-
-    qemu-${arch} \
-        -g 1234 \
-        -L /usr/${arch}-linux-gnu/ \
-        ./binaries/reference-binary.${arch}.out &
-
-    run_gdb \
-        -ex "set sysroot /usr/${arch}-linux-gnu/" \
-        -ex "file ./binaries/reference-binary.${arch}.out" \
-        -ex 'py import coverage;coverage.process_startup()' \
-        -ex "target remote :1234" \
-        -ex "source ./tests/user/test_${arch}.py"
-    local result=$?
-    pkill qemu-${arch}
-    return $result
-}
-
-ARCHS=("aarch64" "riscv64")
-
-FAILED_TESTS=()
-for arch in "${ARCHS[@]}"; do
-    test_arch "$arch"
-    if [ $? -ne 0 ]; then
-        FAILED_TESTS+=("$arch")
-    fi
-done
-
-if [ "${#FAILED_TESTS[@]}" -ne 0 ]; then
-    echo ""
-    echo "Failing tests: ${FAILED_TESTS[@]}"
-    echo ""
-    exit 1
-fi

tests/qemu-tests/tests.sh

@@ -123,8 +123,8 @@ run_gdb() {
 
 # NOTE: We run tests under GDB sessions and because of some cleanup/tests dependencies problems
 # we decided to run each test in a separate GDB session
-gdb_args=(--command pytests_collect.py)
-TESTS_COLLECT_OUTPUT=$(run_gdb "x86_64" "${gdb_args[@]}")
+gdb_args=(--command ../pytests_collect.py)
+TESTS_COLLECT_OUTPUT=$(TESTS_PATH="$ROOT_DIR/tests/qemu-tests/tests/system" run_gdb "x86_64" "${gdb_args[@]}")
 
 if [ $? -eq 1 ]; then
     echo -E "$TESTS_COLLECT_OUTPUT"
@@ -155,7 +155,7 @@ run_test() {
     local arch="$4"
 
     gdb_connect_qemu=(-ex "file ${IMAGE_DIR}/vmlinux-${kernel_type}-${kernel_version}-${arch}" -ex "target remote :${GDB_PORT}")
-    gdb_args=("${gdb_connect_qemu[@]}" --command pytests_launcher.py)
+    gdb_args=("${gdb_connect_qemu[@]}" --command ../pytests_launcher.py)
     if [ ${RUN_CODECOV} -ne 0 ]; then
         gdb_args=(-ex 'py import coverage;coverage.process_startup()' "${gdb_args[@]}")
     fi
@@ -164,7 +164,7 @@ run_test() {
         COVERAGE_FILE=$ROOT_DIR/.cov/coverage \
         COVERAGE_PROCESS_START=$COVERAGERC_PATH \
         USE_PDB="${USE_PDB}" \
-        PWNDBG_LAUNCH_TEST="${test_case}" \
+        PWNDBG_LAUNCH_TEST="qemu-tests/${test_case}" \
         PWNDBG_DISABLE_COLORS=1 \
         PWNDBG_ARCH="${arch}" \
         PWNDBG_KERNEL_TYPE="${kernel_type}" \

tests/qemu-tests/tests/user/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+from . import binaries

tests/qemu-tests/tests/user/binaries/Makefile

@@ -0,0 +1,19 @@
+.PHONY: all
+all: reference-binary.aarch64.out reference-binary.riscv64.out
+
+%.aarch64.out : %.aarch64.c
+	@echo "[+] Building '$@'"
+	@aarch64-linux-gnu-gcc $(CFLAGS) $(EXTRA_FLAGS) -w -o $@ $< $(LDFLAGS)
+
+%.riscv64.out : %.riscv64.c
+	@echo "[+] Building '$@'"
+	@riscv64-linux-gnu-gcc -march=rv64gc -mabi=lp64d -g $(CFLAGS) $(EXTRA_FLAGS) -w -o $@ $? $(LDFLAGS)
+
+AARCH64_SOURCES := $(wildcard *.aarch64.c)
+AARCH64_TARGETS := $(AARCH64_SOURCES:.aarch64.c=.aarch64.out)
+
+RISCV64_SOURCES := $(wildcard *.riscv64.c)
+RISCV64_TARGETS := $(RISCV64_SOURCES:.riscv64.c=.riscv64.out)
+
+clean:
+	rm -f *.aarch64.out *.x86_64.out *.arm.out

tests/qemu-tests/tests/user/binaries/__init__.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import os
+
+path = os.path.dirname(__file__)
+
+
+def get(x):
+    return os.path.join(path, x)

tests/qemu-tests/tests/user/test_aarch64_example.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import gdb
+from capstone.arm64_const import ARM64_INS_BL
+
+import pwndbg.gdblib.disasm
+import pwndbg.gdblib.nearpc
+from pwndbg.gdblib.disasm.instruction import InstructionCondition
+
+AARCH64_GRACEFUL_EXIT = """
+mov x0, 0
+mov x8, 93
+svc 0
+"""
+
+SIMPLE_FUNCTION = f"""
+
+bl my_function
+b end
+
+my_function:
+    ret
+
+end:
+{AARCH64_GRACEFUL_EXIT}
+"""
+
+
+def test_syscall_annotation(qemu_assembly_run):
+    """ """
+    qemu_assembly_run(AARCH64_GRACEFUL_EXIT, "aarch64")
+
+    instructions = pwndbg.gdblib.disasm.near(
+        address=pwndbg.gdblib.regs.pc, instructions=3, emulate=True
+    )[0]
+    future_syscall_ins = instructions[2]
+
+    assert future_syscall_ins.syscall == 93
+    assert future_syscall_ins.syscall_name == "exit"
+
+    gdb.execute("stepuntilasm svc")
+
+    # Both for emulation and non-emulation, ensure a syscall at current PC gets enriched
+    instructions = pwndbg.gdblib.disasm.emulate_one(), pwndbg.gdblib.disasm.no_emulate_one()
+
+    for i in instructions:
+        assert i.syscall == 93
+        assert i.syscall_name == "exit"
+
+
+def test_branch_enhancement(qemu_assembly_run):
+    qemu_assembly_run(SIMPLE_FUNCTION, "aarch64")
+
+    instruction = pwndbg.gdblib.disasm.one_with_config()
+
+    assert instruction.id == ARM64_INS_BL
+    assert instruction.call_like
+    assert not instruction.is_conditional_jump
+    assert instruction.is_unconditional_jump
+    assert instruction.target_string == "my_function"
+
+
+CONDITIONAL_JUMPS = f"""
+mov x2, 0b1010
+mov x3, 0
+
+cbz x3, A
+nop
+
+A:
+cbnz x2, B
+nop
+
+B:
+tbz x2, #0, C
+nop
+
+C:
+tbnz x2, #3, D
+nop
+
+D:
+cmp x2, x3
+b.eq E
+nop
+
+E:
+b.ne F
+nop
+
+F:
+{AARCH64_GRACEFUL_EXIT}
+"""
+
+
+def test_conditional_jumps(qemu_assembly_run):
+    qemu_assembly_run(CONDITIONAL_JUMPS, "aarch64")
+
+    gdb.execute("stepuntilasm cbz")
+    ins = pwndbg.gdblib.disasm.one_with_config()
+
+    assert ins.condition == InstructionCondition.TRUE
+
+    gdb.execute("si")
+    ins = pwndbg.gdblib.disasm.one_with_config()
+
+    assert ins.condition == InstructionCondition.TRUE
+
+    gdb.execute("si")
+    ins = pwndbg.gdblib.disasm.one_with_config()
+
+    assert ins.condition == InstructionCondition.TRUE
+
+    gdb.execute("si")
+    ins = pwndbg.gdblib.disasm.one_with_config()
+
+    assert ins.condition == InstructionCondition.TRUE
+
+    gdb.execute("si")
+    gdb.execute("si")
+
+    ins = pwndbg.gdblib.disasm.one_with_config()
+
+    assert ins.condition == InstructionCondition.FALSE
+
+    gdb.execute("si")
+    gdb.execute("si")
+
+    ins = pwndbg.gdblib.disasm.one_with_config()
+
+    assert ins.condition == InstructionCondition.TRUE
+
+
+def test_conditional_jumps_no_emulate(qemu_assembly_run):
+    gdb.execute("set emulate off")
+    test_conditional_jumps(qemu_assembly_run)

tests/qemu-tests/tests/user/test_aarch64_reference.py

@@ -1,15 +1,17 @@
 from __future__ import annotations
 
-import sys
-import traceback
-
 import gdb
+import user
 
-import pwndbg
+import pwndbg.gdblib.symbol
 
-try:
+REFERENCE_BINARY = user.binaries.get("reference-binary.aarch64.out")
+
+
+def test_aarch64_reference(qemu_start_binary):
+    qemu_start_binary(REFERENCE_BINARY, "aarch64")
     gdb.execute("break break_here")
-    assert pwndbg.gdblib.symbol.address("main") == 0x5500000A1C
+    assert pwndbg.gdblib.symbol.address("main") is not None
     gdb.execute("continue")
 
     gdb.execute("argv", to_string=True)
@@ -35,6 +37,3 @@ try:
     gdb.execute("piebase", to_string=True)
 
     gdb.execute("nextret", to_string=True)
-except AssertionError:
-    traceback.print_exc(file=sys.stdout)
-    sys.exit(1)

tests/qemu-tests/tests/user/test_riscv64_reference.py

@@ -1,15 +1,17 @@
 from __future__ import annotations
 
-import sys
-import traceback
-
 import gdb
+import user
 
-import pwndbg
+import pwndbg.gdblib.symbol
 
-try:
+REFERENCE_BINARY = user.binaries.get("reference-binary.riscv64.out")
+
+
+def test_riscv64_reference(qemu_start_binary):
+    qemu_start_binary(REFERENCE_BINARY, "riscv64")
     gdb.execute("break 4")
-    assert pwndbg.gdblib.symbol.address("main") == 0x4000000668
+    assert pwndbg.gdblib.symbol.address("main") is not None
     gdb.execute("continue")
 
     gdb.execute("stepuntilasm jalr")
@@ -26,7 +28,3 @@ try:
     gdb.execute("stepi")
     assembly = gdb.execute("nearpc 0", to_string=True)
     assert assembly.split()[2] == target, (assembly.split()[2], target)
-
-except AssertionError:
-    traceback.print_exc(file=sys.stdout)
-    sys.exit(1)

tests/tests.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import argparse
 import concurrent.futures
 import os
+import random
 import re
 import subprocess
 import sys
@@ -11,7 +12,7 @@ from subprocess import CompletedProcess
 from typing import List
 from typing import Tuple
 
-root_dir = os.path.realpath("../../")
+root_dir = os.path.realpath("../")
 
 
 def ensureZigPath():
@@ -24,27 +25,66 @@ def ensureZigPath():
 
 def makeBinaries():
     try:
-        subprocess.check_call(["make", "all"], cwd="./tests/binaries")
+        subprocess.check_call(["make", "all"], cwd="./gdb-tests/tests/binaries")
     except subprocess.CalledProcessError:
         exit(1)
 
 
-def run_gdb(gdb_args: List[str], env=None, capture_output=True) -> CompletedProcess[str]:
+def makeCrossArchBinaries():
+    try:
+        subprocess.check_call(["make", "all"], cwd="./qemu-tests/tests/user/binaries")
+    except subprocess.CalledProcessError:
+        exit(1)
+
+
+def open_ports(n: int) -> List[int]:
+    """
+    Returns a list of `n` open ports
+    """
+    try:
+        result = subprocess.run(
+            ["netstat", "-tuln"], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+        if result.returncode != 0:
+            # If netstat not found, try ss
+            raise FileNotFoundError
+    except FileNotFoundError:
+        result = subprocess.run(["ss", "-tuln"], stdout=subprocess.PIPE)
+
+    used_ports = set(re.findall(r":(\d+)", result.stdout.decode()))
+    used_ports = set(map(int, used_ports))
+
+    available_ports = [port for port in range(1024, 65536) if port not in used_ports]
+    return random.sample(available_ports, n)
+
+
+def run_gdb(
+    gdb_binary: str, gdb_args: List[str], env=None, capture_output=True
+) -> CompletedProcess[str]:
     env = os.environ if env is None else env
     return subprocess.run(
-        ["gdb", "--silent", "--nx", "--nh"] + gdb_args + ["--eval-command", "quit"],
+        [gdb_binary, "--silent", "--nx", "--nh"] + gdb_args + ["--eval-command", "quit"],
         env=env,
         capture_output=capture_output,
         text=True,
     )
 
 
-def getTestsList(collect_only: bool, test_name_filter: str, gdbinit_path: str) -> List[str]:
+def getTestsList(
+    collect_only: bool,
+    test_name_filter: str,
+    gdb_binary: str,
+    gdbinit_path: str,
+    test_dir_path: str,
+) -> List[str]:
     # NOTE: We run tests under GDB sessions and because of some cleanup/tests dependencies problems
     # we decided to run each test in a separate GDB session
     gdb_args = ["--init-command", gdbinit_path, "--command", "pytests_collect.py"]
 
-    result = run_gdb(gdb_args)
+    env = os.environ.copy()
+    env["TESTS_PATH"] = os.path.join(os.path.dirname(os.path.realpath(__file__)), test_dir_path)
+
+    result = run_gdb(gdb_binary, gdb_args, env=env)
     tests_collect_output = result.stdout
 
     if result.returncode == 1:
@@ -55,14 +95,14 @@ def getTestsList(collect_only: bool, test_name_filter: str, gdbinit_path: str) -
         exit(0)
 
     # Extract the test names from the output using regex
-    pattern = re.compile(r"tests/.*::.*")
+    pattern = re.compile(rf"{test_dir_path}.*::.*")
     matches = pattern.findall(tests_collect_output)
     tests_list = [match for match in matches if re.search(test_name_filter, match)]
     return tests_list
 
 
 def run_test(
-    test_case: str, args: argparse.Namespace, gdbinit_path: str
+    test_case: str, args: argparse.Namespace, gdb_binary: str, gdbinit_path: str, port: int = None
 ) -> Tuple[CompletedProcess[str], str]:
     gdb_args = ["--init-command", gdbinit_path, "--command", "pytests_launcher.py"]
     if args.cov:
@@ -82,11 +122,20 @@ def run_test(
         env["USE_PDB"] = "1"
     env["PWNDBG_LAUNCH_TEST"] = test_case
     env["PWNDBG_DISABLE_COLORS"] = "1"
-    result = run_gdb(gdb_args, env=env, capture_output=not args.serial)
+    if port is not None:
+        env["QEMU_PORT"] = str(port)
+    result = run_gdb(gdb_binary, gdb_args, env=env, capture_output=not args.serial)
     return (result, test_case)
 
 
-def run_tests_and_print_stats(tests_list: List[str], args: argparse.Namespace, gdbinit_path: str):
+def run_tests_and_print_stats(
+    tests_list: List[str],
+    args: argparse.Namespace,
+    gdb_binary: str,
+    gdbinit_path: str,
+    test_dir_path: str,
+    ports: List[int] = [],
+):
     start = time.time()
     test_results: List[Tuple[CompletedProcess[str], str]] = []
 
@@ -96,7 +145,7 @@ def run_tests_and_print_stats(tests_list: List[str], args: argparse.Namespace, g
         content = process.stdout
 
         # Extract the test name and result using regex
-        testname = re.search(r"^(tests/[^ ]+)", content, re.MULTILINE)[0]
+        testname = re.search(rf"^({test_dir_path}/[^ ]+)", content, re.MULTILINE)[0]
         result = re.search(
             r"(\x1b\[3.m(PASSED|FAILED|SKIPPED|XPASS|XFAIL)\x1b\[0m)", content, re.MULTILINE
         )[0]
@@ -109,16 +158,21 @@ def run_tests_and_print_stats(tests_list: List[str], args: argparse.Namespace, g
             print("")
             print(content)
 
+    port_iterator = iter(ports)
+
     if args.serial:
-        test_results = [run_test(test, args, gdbinit_path) for test in tests_list]
+        test_results = [
+            run_test(test, args, gdb_binary, gdbinit_path, next(port_iterator, None))
+            for test in tests_list
+        ]
     else:
         print("")
         print("Running tests in parallel")
         with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
             for test in tests_list:
-                executor.submit(run_test, test, args, gdbinit_path).add_done_callback(
-                    lambda future: handle_parallel_test_result(future.result())
-                )
+                executor.submit(
+                    run_test, test, args, gdb_binary, gdbinit_path, next(port_iterator, None)
+                ).add_done_callback(lambda future: handle_parallel_test_result(future.result()))
 
     end = time.time()
     seconds = int(end - start)
@@ -145,6 +199,8 @@ def run_tests_and_print_stats(tests_list: List[str], args: argparse.Namespace, g
 
 def parse_args():
     parser = argparse.ArgumentParser(description="Run tests.")
+    parser.add_argument("-t", "--type", dest="type", choices=["gdb", "cross-arch"], default="gdb")
+
     parser.add_argument(
         "-p",
         "--pdb",
@@ -177,6 +233,8 @@ def parse_args():
     return parser.parse_args()
 
 
+TEST_FOLDER_NAME = {"gdb": "gdb-tests/tests", "cross-arch": "qemu-tests/tests/user"}
+
 if __name__ == "__main__":
     args = parse_args()
     if args.cov:
@@ -192,7 +250,24 @@ if __name__ == "__main__":
         os.environ["GDB_INIT_PATH"] = gdbinit_path
     else:
         gdbinit_path = os.path.join(root_dir, "gdbinit.py")
+
+    gdb_binary = "gdb"
+
+    if args.type == "gdb":
         ensureZigPath()
         makeBinaries()
-    tests: List[str] = getTestsList(args.collect_only, args.test_name_filter, gdbinit_path)
-    run_tests_and_print_stats(tests, args, gdbinit_path)
+    else:
+        makeCrossArchBinaries()
+        gdb_binary = "gdb-multiarch"
+
+    test_dir_path = TEST_FOLDER_NAME[args.type]
+
+    tests: List[str] = getTestsList(
+        args.collect_only, args.test_name_filter, gdb_binary, gdbinit_path, test_dir_path
+    )
+
+    ports = []
+    if args.type == "cross-arch":
+        ports = open_ports(len(tests))
+
+    run_tests_and_print_stats(tests, args, gdb_binary, gdbinit_path, test_dir_path, ports)

unit-tests.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+COV=0
+# Run unit tests
+for arg in "$@"; do
+    if [ "$arg" == "--cov" ]; then
+        COV=1
+        break
+    fi
+done
+
+if [ $COV -eq 1 ]; then
+    coverage run -m pytest tests/unit-tests
+else
+    pytest tests/unit-tests
+fi
+
+exit_code=$((exit_code + $?))
+
+exit $exit_code