Add check that all pages render (#572)

Closes https://github.com/Qiskit/documentation/issues/511. Example:

```
$ npm run check-pages-render

Checked 10 / 71 pages
Checked 20 / 71 pages
Checked 30 / 71 pages
Checked 40 / 71 pages
Checked 50 / 71 pages
Checked 60 / 71 pages
Checked 70 / 71 pages
✅ All pages render without crashing
```

## Only checks non-API docs by default

This script is quite slow, at least on my M1 because the Docker images
is built with x86.

So, to avoid CI slowing down too much, we only check non-API pages in PR
builds. Our nightly cron job checks everything else.

## Does not auto-start Docker

We no longer have the file `webServer.ts` thanks to
https://github.com/Qiskit/documentation/pull/578, which was a great
improvement.

Rather than adding back somewhat complex code for us to auto-start the
server—and then to periodically ping if it's ready or time out—we expect
the user to start up the server. That's acceptable since usually people
will rely on CI to run this check. It's too slow for people to be
frequently running locally.

---------

Co-authored-by: Frank Harkins <frankharkins@hotmail.co.uk>

.github/workflows/main.yml

@@ -41,5 +41,12 @@ jobs:
         run: npm run check:fmt
       - name: Typecheck
         run: npm run typecheck
-      - name: Run infrastructure tests
+      - name: Infrastructure tests
         run: npm test
+
+      - name: Start local Docker preview
+        run: |
+          ./start &
+          sleep 20
+      - name: Check that pages render
+        run: npm run check-pages-render

.github/workflows/nightly-extended-checks.yml

@@ -10,16 +10,16 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-# This Action validates all the links in the documentation, including
-# external links. This Action will be run at 00:00 (UTC) every day.
+# This Action runs every day at 00:00 UTC to check things that
+# we care about but would be too slow to check in every PR.
 
-name: Check external links
+name: Nightly extended checks
 on:
   schedule:
     - cron: "0 0 * * *"
 
 jobs:
-  external-link-checker:
+  nightly-extended-checks:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'Qiskit'
     steps:
@@ -34,8 +34,20 @@ jobs:
       - name: Check external links
         run: >
           npm run check:links --
-          --current-apis
           --qiskit-release-notes
+          --current-apis
           --historical-apis
           --skip-broken-historical
           --external
+
+      - name: Start local Docker preview
+        run: |
+          ./start &
+          sleep 20
+      - name: Check API pages render
+        run: >
+          npm run check-pages-render --
+          --qiskit-release-notes
+          --current-apis
+          --historical-apis
+          --translations

README.md

@@ -218,6 +218,20 @@ Ayyyyy, this is a fake description.
 
 If the word appears in multiple files, prefer the second approach to add it to `cSpell.json`.
 
+## Check that pages render
+
+It's possible to write broken pages that crash when loaded. This is usually due to syntax errors.
+
+To check that all the non-API docs render:
+
+1. Start up the local preview with `./start` by following the instructions at [Preview the docs locally](#preview-the-docs-locally)
+2. In a new tab, `npm run check-pages-render`
+
+You can also check that API docs and translations render by using any of these arguments: `npm run check-pages-render -- --qiskit-release-notes --current-apis --historical-apis --translations`. Warning that this is exponentially slower.
+
+CI will check on every PR that non-API docs correctly render. We also run a nightly cron job to check the API docs and
+translations.
+
 ## Format TypeScript files
 
 If you're working on our support code in `scripts/`, run `npm run fmt` to automatically format the files.

package.json

@@ -10,6 +10,7 @@
     "check:links": "node -r esbuild-register scripts/commands/checkLinks.ts",
     "check:spelling": "cspell --relative --no-progress docs/**/*.md* docs/api/**/*.md*",
     "check:fmt": "prettier --check .",
+    "check-pages-render": "node -r esbuild-register scripts/commands/checkPagesRender.ts",
     "fmt": "prettier --write .",
     "test": "jest",
     "typecheck": "tsc",

scripts/commands/checkPagesRender.ts

@@ -0,0 +1,149 @@
+// This code is a Qiskit project.
+//
+// (C) Copyright IBM 2024.
+//
+// This code is licensed under the Apache License, Version 2.0. You may
+// obtain a copy of this license in the LICENSE file in the root directory
+// of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+//
+// Any modifications or derivative works of this code must retain this
+// copyright notice, and modified files need to carry a notice indicating
+// that they have been altered from the originals.
+
+import { globby } from "globby";
+import yargs from "yargs/yargs";
+import { hideBin } from "yargs/helpers";
+
+import { zxMain } from "../lib/zx";
+
+const PORT = 3000;
+
+interface Arguments {
+  [x: string]: unknown;
+  currentApis: boolean;
+  historicalApis: boolean;
+  qiskitReleaseNotes: boolean;
+  translations: boolean;
+}
+
+const readArgs = (): Arguments => {
+  return yargs(hideBin(process.argv))
+    .version(false)
+    .option("current-apis", {
+      type: "boolean",
+      default: false,
+      description: "Check the pages in the current API docs.",
+    })
+    .option("historical-apis", {
+      type: "boolean",
+      default: false,
+      description:
+        "Check the pages in the historical API docs, e.g. `api/qiskit/0.44`. " +
+        "Warning: this is slow.",
+    })
+    .option("qiskit-release-notes", {
+      type: "boolean",
+      default: false,
+      description: "Check the pages in the `api/qiskit/release-notes` folder.",
+    })
+    .option("translations", {
+      type: "boolean",
+      default: false,
+      description: "Check the pages in the `translations/` subfolders.",
+    })
+    .parseSync();
+};
+
+zxMain(async () => {
+  const args = readArgs();
+  await validateDockerRunning();
+  const files = await determineFilePaths(args);
+
+  let allGood = true;
+  let numFilesChecked = 1;
+  for (const fp of files) {
+    const rendered = await canRender(fp);
+    if (!rendered) {
+      console.error(`❌ Failed to render: ${fp}`);
+      allGood = false;
+    }
+
+    // This script can be slow, so log progress every 10 files.
+    if (numFilesChecked % 10 == 0) {
+      console.log(`Checked ${numFilesChecked} / ${files.length} pages`);
+    }
+    numFilesChecked++;
+  }
+
+  if (allGood) {
+    console.info("✅ All pages render without crashing");
+  } else {
+    console.error(
+      "💔 Some pages crash when rendering. This is usually due to invalid syntax, such as forgetting " +
+        "the closing component tag, like `</Admonition>`. You can sometimes get a helpful error message " +
+        "by previewing the docs locally or in CI. See the README for instructions.",
+    );
+    process.exit(1);
+  }
+});
+
+async function canRender(fp: string): Promise<boolean> {
+  const url = pathToUrl(fp);
+  try {
+    const response = await fetch(url);
+    if (response.status >= 300) {
+      return false;
+    }
+  } catch (error) {
+    return false;
+  }
+
+  return true;
+}
+
+function pathToUrl(path: string): string {
+  const strippedPath = path
+    .replace("docs/", "")
+    .replace("translations/", "")
+    .replace(/\.(?:md|mdx|ipynb)$/g, "");
+  return `http://localhost:${PORT}/${strippedPath}`;
+}
+
+async function validateDockerRunning(): Promise<void> {
+  try {
+    const response = await fetch(`http://localhost:${PORT}`);
+    if (response.status !== 404) {
+      console.error(
+        "Failed to access http://localhost:3000. Have you started the Docker server with `./start`? " +
+          "Refer to the README for instructions.",
+      );
+      process.exit(1);
+    }
+  } catch (error) {
+    console.error(
+      "Error when accessing http://localhost:3000. Make sure that you've started the Docker server " +
+        "with `./start`. Refer to the README for instructions.\n\n" +
+        `${error}`,
+    );
+    process.exit(1);
+  }
+}
+
+async function determineFilePaths(args: Arguments): Promise<string[]> {
+  const globs = ["docs/**/*.{ipynb,md,mdx}"];
+  if (!args.currentApis) {
+    globs.push("!docs/api/{qiskit,qiskit-ibm-provider,qiskit-ibm-runtime}/*");
+  }
+  if (!args.historicalApis) {
+    globs.push(
+      "!docs/api/{qiskit,qiskit-ibm-provider,qiskit-ibm-runtime}/[0-9]*/*",
+    );
+  }
+  if (!args.qiskitReleaseNotes) {
+    globs.push("!docs/api/qiskit/release-notes/*");
+  }
+  if (args.translations) {
+    globs.push("translations/**/*.{ipynb,md,mdx}");
+  }
+  return globby(globs);
+}

scripts/lib/links/LinkChecker.ts

@@ -89,7 +89,7 @@ export class Link {
   async checkExternalLink(): Promise<string | null> {
     try {
       const response = await fetch(this.value, {
-        headers: { "User-Agent": "prn-broken-links-finder" },
+        headers: { "User-Agent": "qiskit-documentation-broken-links-finder" },
       });
 
       if (response.status >= 300) {