diff --git a/src/server/server.ts b/src/server/server.ts
index 92af40af..8a2b55bc 100644
--- a/src/server/server.ts
+++ b/src/server/server.ts
@@ -1,4 +1,5 @@
 import fs from "node:fs";
+import { isIPv6 } from "node:net";
 import { Server as HttpServer } from "node:http";
 import { join, dirname } from "node:path";
 import type { AddressInfo } from "node:net";
@@ -159,11 +160,15 @@ export class Server {
     );
     const port = this.state.port ?? 3000;
     const { host, webhookPath, webhookProxy } = this.state;
-    const printableHost = host ?? "localhost";
 
     this.state.httpServer = await new Promise((resolve, reject) => {
       const server = this.state.httpServer!.listen(port, host, async () => {
         this.state.port = (server.address() as AddressInfo).port;
+        this.state.host = (server.address() as AddressInfo).address;
+
+        if (isIPv6(this.state.host)) {
+          this.state.host = `[${this.state.host}]`;
+        }
 
         if (webhookProxy) {
           this.state.eventSource = await createWebhookProxy({
@@ -173,7 +178,7 @@ export class Server {
             url: webhookProxy,
           });
         }
-        this.log.info(`Listening on http://${printableHost}:${this.port}`);
+        this.log.info(`Listening on http://${this.host}:${this.port}`);
         resolve(server);
       });
 
diff --git a/test/apps/default.test.ts b/test/apps/default.test.ts
index c7480e14..84b24901 100644
--- a/test/apps/default.test.ts
+++ b/test/apps/default.test.ts
@@ -40,7 +40,7 @@ describe("default app", async () => {
       const server = await instantiateServer();
       await server.start();
       expect(
-        (await fetch(`http://localhost:${server.port}/probot`)).status,
+        (await fetch(`http://${server.host}:${server.port}/probot`)).status,
       ).toBe(200);
       await server.stop();
     });
@@ -49,7 +49,9 @@ describe("default app", async () => {
       it("returns the correct HTML with values", async () => {
         const server = await instantiateServer();
         await server.start();
-        const response = await fetch(`http://localhost:${server.port}/probot`);
+        const response = await fetch(
+          `http://${server.host}:${server.port}/probot`,
+        );
         expect(response.status).toBe(200);
 
         const actualText = await response.text();
@@ -63,7 +65,9 @@ describe("default app", async () => {
       it("returns the correct HTML without values", async () => {
         const server = await instantiateServer(__dirname);
         await server.start();
-        const response = await fetch(`http://localhost:${server.port}/probot`);
+        const response = await fetch(
+          `http://${server.host}:${server.port}/probot`,
+        );
         expect(response.status).toBe(200);
 
         const actualText = await response.text();
@@ -79,7 +83,7 @@ describe("default app", async () => {
     it("redirects to /probot", async () => {
       const server = await instantiateServer(__dirname);
       await server.start();
-      const response = await fetch(`http://localhost:${server.port}/`, {
+      const response = await fetch(`http://${server.host}:${server.port}/`, {
         redirect: "manual",
       });
 
diff --git a/test/apps/setup.test.ts b/test/apps/setup.test.ts
index dcfd0aae..157a61ae 100644
--- a/test/apps/setup.test.ts
+++ b/test/apps/setup.test.ts
@@ -47,9 +47,7 @@ describe("Setup app", async () => {
 
     await server.start();
 
-    port = server.port!;
-
-    await server.loadHandler(setupAppFactory(undefined, port));
+    await server.loadHandler(setupAppFactory(server.host, server.port));
   });
 
   afterEach(async () => {
@@ -65,11 +63,11 @@ describe("Setup app", async () => {
         "Probot is in setup mode, webhooks cannot be received and",
         "custom routes will not work until APP_ID and PRIVATE_KEY",
         "are configured in .env.",
-        `Please follow the instructions at http://localhost:${port} to configure .env.`,
+        `Please follow the instructions at http://${server.host}:${server.port} to configure .env.`,
         "Once you are done, restart the server.",
         "",
         `Running Probot v0.0.0-development (Node.js: ${process.version})`,
-        `Listening on http://localhost:${port}`,
+        `Listening on http://${server.host}:${server.port}`,
       ];
 
       const infoLogs = logOutput
@@ -92,9 +90,9 @@ describe("Setup app", async () => {
         port,
       });
 
-      await server2.loadHandler(setupAppFactory("localhost", port));
+      await server2.loadHandler(setupAppFactory(server.host, port));
 
-      const expMsg = `Please follow the instructions at http://localhost:${port} to configure .env.`;
+      const expMsg = `Please follow the instructions at http://${server.host}:${server.port} to configure .env.`;
 
       const infoLogs = logOutput
         .filter((output: any) => output.level === pino.levels.values.info)
@@ -105,7 +103,9 @@ describe("Setup app", async () => {
 
   describe("GET /probot", () => {
     it("returns a 200 response", async () => {
-      const response = await fetch(`http://localhost:${port}/probot`);
+      const response = await fetch(
+        `http://${server.host}:${server.port}/probot`,
+      );
       expect(response.status).toBe(200);
     });
   });
@@ -141,12 +141,12 @@ describe("Setup app", async () => {
         port: 0,
       });
 
-      await server.loadHandler(setupAppFactory("localhost", server.port));
-
       await server.start();
 
+      await server.loadHandler(setupAppFactory(server.host, server.port));
+
       const setupResponse = await fetch(
-        `http://localhost:${server.port}/probot/setup?code=123`,
+        `http://${server.host}:${server.port}/probot/setup?code=123`,
         { redirect: "manual" },
       );
 
@@ -172,16 +172,21 @@ describe("Setup app", async () => {
           privateKey: "dummy value for setup, see #1512",
         }),
         log: pino(streamLogsToOutput),
+        port: 0,
       });
 
-      await server.loadHandler(setupAppFactory("localhost", port));
+      await server.start();
+
+      await server.loadHandler(setupAppFactory(server.host, server.port));
 
       const setupResponse = await fetch(
-        `http://localhost:${port}/probot/setup`,
+        `http://${server.host}:${server.port}/probot/setup`,
       );
 
       expect(setupResponse.status).toBe(400);
       expect(await setupResponse.text()).toMatchSnapshot();
+
+      await server.stop();
     });
 
     it("throws a 400 Error if code is an empty string", async () => {
@@ -193,23 +198,30 @@ describe("Setup app", async () => {
           privateKey: "dummy value for setup, see #1512",
         }),
         log: pino(streamLogsToOutput),
+        port: 0,
       });
 
-      await server.loadHandler(setupAppFactory(undefined, port));
+      await server.start();
+
+      await server.loadHandler(setupAppFactory(server.host, server.port));
 
       const setupResponse = await fetch(
-        `http://localhost:${port}/probot/setup?code=`,
+        `http://${server.host}:${server.port}/probot/setup?code=`,
       );
 
       expect(setupResponse.status).toBe(400);
 
       expect(await setupResponse.text()).toMatchSnapshot();
+
+      await server.stop();
     });
   });
 
   describe("GET /probot/import", () => {
     it("renders importView", async () => {
-      const importView = await fetch(`http://localhost:${port}/probot/import`);
+      const importView = await fetch(
+        `http://${server.host}:${server.port}/probot/import`,
+      );
 
       expect(importView.status).toBe(200);
       expect(await importView.text()).toMatchSnapshot();
@@ -224,13 +236,16 @@ describe("Setup app", async () => {
         webhook_secret: "baz",
       });
 
-      const response = await fetch(`http://localhost:${port}/probot/import`, {
-        body,
-        method: "POST",
-        headers: {
-          "content-type": "application/json",
+      const response = await fetch(
+        `http://${server.host}:${server.port}/probot/import`,
+        {
+          body,
+          method: "POST",
+          headers: {
+            "content-type": "application/json",
+          },
         },
-      });
+      );
 
       expect(response.status).toBe(200);
       expect(await response.text()).toBe("");
@@ -246,7 +261,7 @@ describe("Setup app", async () => {
       });
 
       const importResponse = await fetch(
-        `http://localhost:${port}/probot/import`,
+        `http://${server.host}:${server.port}/probot/import`,
         {
           body,
           method: "POST",
@@ -264,7 +279,7 @@ describe("Setup app", async () => {
   describe("GET /probot/success", () => {
     it("returns a 200 response", async () => {
       const successResponse = await fetch(
-        `http://localhost:${port}/probot/success`,
+        `http://${server.host}:${server.port}/probot/success`,
       );
 
       expect(successResponse.status).toBe(200);
diff --git a/test/integration/integration-smee.test.ts b/test/integration/integration-smee.test.ts
index ba35607f..0766a69a 100644
--- a/test/integration/integration-smee.test.ts
+++ b/test/integration/integration-smee.test.ts
@@ -1,12 +1,14 @@
 import { Writable } from "node:stream";
-import { ManifestCreation } from "../../src/manifest-creation.js";
-import { describe, test, expect, afterEach } from "vitest";
-import { ApplicationFunction, Probot, Server } from "../../src/index.js";
+
+import { sign } from "@octokit/webhooks-methods";
 import { pino } from "pino";
+import { describe, test, expect, afterEach } from "vitest";
+
+import { ManifestCreation } from "../../src/manifest-creation.js";
+import { ApplicationFunction, Probot, Server } from "../../src/index.js";
 import WebhookExamples, {
   type WebhookDefinition,
 } from "@octokit/webhooks-examples";
-import { sign } from "@octokit/webhooks-methods";
 
 describe("smee-client", () => {
   afterEach(async () => {
diff --git a/test/run.test.ts b/test/run.test.ts
index f7a03001..cd7b4e99 100644
--- a/test/run.test.ts
+++ b/test/run.test.ts
@@ -105,7 +105,7 @@ describe("run", async () => {
       const dataString = JSON.stringify(pushEvent);
 
       const response = await fetch(
-        `http://localhost:${server.port}/api/github/webhooks`,
+        `http://${server.host}:${server.port}/api/github/webhooks`,
         {
           method: "POST",
           body: dataString,
@@ -136,7 +136,7 @@ describe("run", async () => {
 
       try {
         const response = await fetch(
-          `http://localhost:${server.port}/custom-webhook`,
+          `http://${server.host}:${server.port}/custom-webhook`,
           {
             method: "POST",
             body: dataString,
diff --git a/test/server.test.ts b/test/server.test.ts
index 5dc86df1..60b1f8ea 100644
--- a/test/server.test.ts
+++ b/test/server.test.ts
@@ -1,6 +1,6 @@
+import http from "node:http";
 import { Writable } from "node:stream";
 
-import http from "http";
 import { pino } from "pino";
 import { sign } from "@octokit/webhooks-methods";
 import WebhookExamples, {
@@ -80,7 +80,7 @@ describe("Server", async () => {
 
   describe("GET /ping", () => {
     it("returns a 200 response", async () => {
-      const response = await fetch(`http://localhost:${server.port}/ping`);
+      const response = await fetch(`http://${server.host}:${server.port}/ping`);
       expect(response.status).toBe(200);
       expect(await response.text()).toBe("PONG");
 
@@ -116,7 +116,7 @@ describe("Server", async () => {
 
       const dataString = JSON.stringify(pushEvent);
 
-      const response = await fetch(`http://localhost:${server.port}/`, {
+      const response = await fetch(`http://${server.host}:${server.port}/`, {
         body: dataString,
         method: "POST",
         headers: {
@@ -140,7 +140,7 @@ describe("Server", async () => {
       await server.load(() => {});
 
       const response = await fetch(
-        `http://localhost:${server.port}/api/github/webhooks`,
+        `http://${server.host}:${server.port}/api/github/webhooks`,
         {
           body: JSON.stringify(pushEvent),
           method: "POST",
@@ -186,7 +186,7 @@ describe("Server", async () => {
       const dataString = JSON.stringify(pushEvent);
 
       const response = await fetch(
-        `http://localhost:${server.port}/api/github/webhooks`,
+        `http://${server.host}:${server.port}/api/github/webhooks`,
         {
           method: "POST",
           body: dataString,
@@ -212,7 +212,7 @@ describe("Server", async () => {
         expect.assertions(3);
 
         const response = await fetch(
-          `http://localhost:${server.port}/notfound`,
+          `http://${server.host}:${server.port}/notfound`,
         );
 
         expect(response.status).toBe(404);
@@ -257,7 +257,9 @@ describe("Server", async () => {
         await testApp.start();
 
         expect(output.length).toEqual(2);
-        expect(output[1].msg).toEqual("Listening on http://localhost:3001");
+        expect(output[1].msg).toEqual(
+          `Listening on http://${server.host}:3001`,
+        );
 
         await testApp.stop();
       });