Make notebook tester into package (#1208)

Makes the notebook tester into an installable package so we can re-use
it in other repos.

Main changes are:
* Adding `pyproject.toml` and moving script to
`qiskit_docs_notebook_tester/__init__.py`
* Moving the lists of notebooks to a config file
* Splitting the requirements needed for running the code in the
notebooks form those neeed by the testing script itself

The interface via `tox` should be unaffected.

---------

Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com>

.gitignore

@@ -17,4 +17,3 @@ tsconfig.tsbuildinfo
 /.out/
 /.sphinx-artifacts/
 poetry.lock
-pyproject.toml

README.md

@@ -117,8 +117,9 @@ You also need to install a few system dependencies: TeX, Poppler, and graphviz.
 
 > [!NOTE]
 > If your notebook submits hardware jobs to IBM Quantum, you must add it to the
-> ignore list in `scripts/nb-tester/test-notebooks.py`. This is not needed if
-> you only retrieve information.
+> list `notebooks-that-submit-jobs` in
+> [`scripts/nb-tester/notebooks.toml`](scripts/nb-tester/notebooks.toml). This
+> is not needed if the notebook only retrieves information.
 >
 > If your notebook uses the latex circuit drawer (`qc.draw("latex")`), you must
 > add it to the "Check for notebooks that require LaTeX" step in

scripts/nb-tester/notebooks.toml

@@ -0,0 +1,20 @@
+# Used to find all notebooks in the repo
+all_notebooks = "[!.]*/**/*.ipynb"
+
+# Always exclude notebooks matching the following patterns
+notebooks_exclude = [
+    "scripts/ibm-quantum-learning-uploader/test/template.ipynb",
+    "**/.ipynb_checkpoints/**",
+]
+
+# The following notebooks submit jobs to IBM Quantum
+notebooks_that_submit_jobs = [
+    "docs/start/hello-world.ipynb",
+    "tutorials/build-repitition-codes/build-repitition-codes.ipynb",
+    "tutorials/chsh-inequality/chsh-inequality.ipynb",
+    "tutorials/grovers-algorithm/grovers.ipynb",
+    "tutorials/quantum-approximate-optimization-algorithm/qaoa.ipynb",
+    "tutorials/repeat-until-success/repeat-until-success.ipynb",
+    "tutorials/submitting-transpiled-circuits/submitting-transpiled-circuits.ipynb",
+    "tutorials/variational-quantum-eigensolver/vqe.ipynb",
+]

scripts/nb-tester/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = [ "hatchling",]
+build-backend = "hatchling.build"
+
+[project]
+name = "qiskit-docs-notebook-tester"
+version = "0.0.1"
+description = "Tool for the Qiskit docs team to test their notebooks"
+requires-python = ">=3.9"
+license = "Apache-2.0"
+dependencies = [
+  "nbconvert~=7.16.0",
+  "nbformat~=5.9.2",
+  "ipykernel~=6.29.2",
+  "squeaky==0.7.0",
+  "tomli==2.0.1",
+]
+
+[[project.authors]]
+name = "Qiskit docs team"
+
+[project.scripts]
+test-docs-notebooks = "qiskit_docs_notebook_tester:main"

scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py

@@ -10,6 +10,9 @@
 # notice, and modified files need to carry a notice indicating that they have
 # been altered from the originals.
 
+from __future__ import annotations
+
+
 import argparse
 import asyncio
 import sys
@@ -22,30 +25,33 @@ from typing import Iterator
 import nbclient
 import nbconvert
 import nbformat
+import tomli
 from qiskit_ibm_runtime import QiskitRuntimeService
 from squeaky import clean_notebook
 
-NOTEBOOKS_GLOB = "[!.]*/**/*.ipynb"
-NOTEBOOKS_EXCLUDE = [
-    "docs/api/**",
-    "**/.ipynb_checkpoints/**",
-]
-NOTEBOOKS_THAT_SUBMIT_JOBS = [
-    "docs/start/hello-world.ipynb",
-    "tutorials/build-repitition-codes/build-repitition-codes.ipynb",
-    "tutorials/chsh-inequality/chsh-inequality.ipynb",
-    "tutorials/grovers-algorithm/grovers.ipynb",
-    "tutorials/quantum-approximate-optimization-algorithm/qaoa.ipynb",
-    "tutorials/repeat-until-success/repeat-until-success.ipynb",
-    "tutorials/submitting-transpiled-circuits/submitting-transpiled-circuits.ipynb",
-    "tutorials/variational-quantum-eigensolver/vqe.ipynb",
-]
+@dataclass
+class Config:
+    all_notebooks: str
+    notebooks_exclude: list[str]
+    notebooks_that_submit_jobs: list[str]
 
+    @classmethod
+    def read(cls, path: str) -> Config:
+        """
+        Load the globs from the TOML file
+        """
+        try:
+            return cls(**tomli.loads(Path(path).read_text()))
+        except TypeError as err:
+            raise ValueError(
+                f"Couldn't read config from {path}; check it exists and the"
+                " entries are correct."
+            ) from err
 
 def matches(path: Path, glob_list: list[str]) -> bool:
     return any(path.match(glob) for glob in glob_list)
 
-def filter_paths(paths: list[Path], args: argparse.Namespace) -> Iterator[Path]:
+def filter_paths(paths: list[Path], args: argparse.Namespace, config: Config) -> Iterator[Path]:
     """
     Filter out any paths we don't want to run, printing messages.
     """
@@ -55,20 +61,19 @@ def filter_paths(paths: list[Path], args: argparse.Namespace) -> Iterator[Path]:
             print(f"ℹ️ Skipping {path}; file is not `.ipynb` format.")
             continue
 
-        if matches(path, NOTEBOOKS_EXCLUDE):
-            this_file = Path(__file__).resolve()
+        if matches(path, config.notebooks_exclude):
             print(
-                f"ℹ️ Skipping {path}; to run it, edit `NOTEBOOKS_EXCLUDE` in {this_file}."
+                f"ℹ️ Skipping {path}; to run it, edit `notebooks-exclude` in {args.config_path}."
             )
             continue
 
-        if not submit_jobs and matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS):
+        if not submit_jobs and matches(path, config.notebooks_that_submit_jobs):
             print(
                 f"ℹ️ Skipping {path} as it submits jobs; use the --submit-jobs flag to run it."
             )
             continue
 
-        if args.only_submit_jobs and not matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS):
+        if args.only_submit_jobs and not matches(path, config.notebooks_that_submit_jobs):
             print(
                 f"ℹ️ Skipping {path} as it does not submit jobs and the --only-submit-jobs flag is set."
             )
@@ -171,20 +176,20 @@ async def _execute_notebook(filepath: Path, args: argparse.Namespace) -> nbforma
     return nb
 
 
-def find_notebooks() -> list[Path]:
+def find_notebooks(config: Config) -> list[Path]:
     """
-    Get paths to all notebooks in NOTEBOOKS_GLOB that are not excluded by
-    NOTEBOOKS_EXCLUDE
+    Get paths to notebooks in glob `all-notebooks` that are not excluded by
+    glob `notebooks-exclude`.
     """
-    all_notebooks = Path(".").glob(NOTEBOOKS_GLOB)
+    all_notebooks = Path(".").glob(config.all_notebooks)
     return [
         path
         for path in all_notebooks
-        if not matches(path, NOTEBOOKS_EXCLUDE)
+        if not matches(path, config.notebooks_exclude)
     ]
 
 
-def cancel_trailing_jobs(start_time: datetime) -> bool:
+def cancel_trailing_jobs(start_time: datetime, config_path: str) -> bool:
     """
     Cancel any runtime jobs created after `start_time`.
 
@@ -207,8 +212,8 @@ def cancel_trailing_jobs(start_time: datetime) -> bool:
 
     print(
         f"⚠️ Cancelling {len(jobs)} job(s) created after {start_time}.\n"
-        "Add any notebooks that submit jobs to NOTEBOOKS_EXCLUDE in "
-        "`scripts/nb-tester/test-notebook.py`."
+        "Add any notebooks that submit jobs to `notebooks-that-submit-jobs` in "
+        f"`{config_path}`."
     )
     for job in jobs:
         job.cancel()
@@ -225,7 +230,7 @@ def create_argument_parser() -> argparse.ArgumentParser:
         help=(
             "Paths to notebooks. If not provided, the script will search for "
             "notebooks in `docs/`. To exclude a notebook from this process, add it "
-            "to `NOTEBOOKS_EXCLUDE` in the script."
+            "to `notebooks-exclude` in the config file."
         ),
         nargs="*",
     )
@@ -249,24 +254,28 @@ def create_argument_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Same as --submit-jobs, but also skips notebooks that do not submit jobs to IBM Quantum",
     )
+    parser.add_argument(
+        "--config-path",
+        help="Path to a TOML file containing the globs for detecting and sorting notebooks",
+    )
     return parser
 
 
-async def main() -> None:
+async def _main() -> None:
     args = create_argument_parser().parse_args()
-    paths = map(Path, args.filenames or find_notebooks())
-    filtered_paths = filter_paths(paths, args)
+    config = Config.read(args.config_path)
+    paths = map(Path, args.filenames or find_notebooks(config))
+    filtered_paths = filter_paths(paths, args, config)
 
     # Execute notebooks
     start_time = datetime.now()
     print("Executing notebooks:")
     results = await asyncio.gather(*(execute_notebook(path, args) for path in filtered_paths))
     print("Checking for trailing jobs...")
-    results.append(cancel_trailing_jobs(start_time))
+    results.append(cancel_trailing_jobs(start_time, args.config_path))
     if not all(results):
         sys.exit(1)
     sys.exit(0)
 
-
-if __name__ == "__main__":
-    asyncio.run(main())
+def main():
+    asyncio.run(_main())

scripts/nb-tester/requirements.txt

@@ -1,10 +1,6 @@
 # We pin to exact versions for a more reproducible and
 # stable build.
-nbconvert~=7.16.0
-nbformat~=5.9.2
-ipykernel~=6.29.2
 qiskit[all]~=1.0
 qiskit-aer~=0.14.0.1
 qiskit-ibm-runtime~=0.23.0
 qiskit-ibm-provider~=0.11.0
-squeaky==0.7.0

tox.ini

@@ -4,12 +4,14 @@ env_list = py3
 skipsdist = true
 
 [testenv]
-deps = -r scripts/nb-tester/requirements.txt
+deps =
+  -e scripts/nb-tester
+  -r scripts/nb-tester/requirements.txt
 setenv = PYDEVD_DISABLE_FILE_VALIDATION=1
-commands = python scripts/nb-tester/test-notebook.py {posargs}
+commands = test-docs-notebooks {posargs} --config-path scripts/nb-tester/notebooks.toml
 
-[testenv:lint]
-commands = squeaky --check --no-advice {posargs:docs tutorials}
-
-[testenv:fix]
-commands = squeaky {posargs:docs tutorials}
+[testenv:{lint,fix}]
+deps = squeaky==0.7.0
+commands = 
+  lint: squeaky --check --no-advice {posargs:docs tutorials}
+  fix:  squeaky {posargs:docs tutorials}