Make notebook tester into package (#1208)
Makes the notebook tester into an installable package so we can re-use it in other repos. Main changes are: * Adding `pyproject.toml` and moving script to `qiskit_docs_notebook_tester/__init__.py` * Moving the lists of notebooks to a config file * Splitting the requirements needed for running the code in the notebooks form those neeed by the testing script itself The interface via `tox` should be unaffected. --------- Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com>
This commit is contained in:
parent
ee69a31c99
commit
0290bfb588
|
@ -17,4 +17,3 @@ tsconfig.tsbuildinfo
|
||||||
/.out/
|
/.out/
|
||||||
/.sphinx-artifacts/
|
/.sphinx-artifacts/
|
||||||
poetry.lock
|
poetry.lock
|
||||||
pyproject.toml
|
|
||||||
|
|
|
@ -117,8 +117,9 @@ You also need to install a few system dependencies: TeX, Poppler, and graphviz.
|
||||||
|
|
||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
> If your notebook submits hardware jobs to IBM Quantum, you must add it to the
|
> If your notebook submits hardware jobs to IBM Quantum, you must add it to the
|
||||||
> ignore list in `scripts/nb-tester/test-notebooks.py`. This is not needed if
|
> list `notebooks-that-submit-jobs` in
|
||||||
> you only retrieve information.
|
> [`scripts/nb-tester/notebooks.toml`](scripts/nb-tester/notebooks.toml). This
|
||||||
|
> is not needed if the notebook only retrieves information.
|
||||||
>
|
>
|
||||||
> If your notebook uses the latex circuit drawer (`qc.draw("latex")`), you must
|
> If your notebook uses the latex circuit drawer (`qc.draw("latex")`), you must
|
||||||
> add it to the "Check for notebooks that require LaTeX" step in
|
> add it to the "Check for notebooks that require LaTeX" step in
|
||||||
|
|
|
@ -0,0 +1,20 @@
|
||||||
|
# Used to find all notebooks in the repo
|
||||||
|
all_notebooks = "[!.]*/**/*.ipynb"
|
||||||
|
|
||||||
|
# Always exclude notebooks matching the following patterns
|
||||||
|
notebooks_exclude = [
|
||||||
|
"scripts/ibm-quantum-learning-uploader/test/template.ipynb",
|
||||||
|
"**/.ipynb_checkpoints/**",
|
||||||
|
]
|
||||||
|
|
||||||
|
# The following notebooks submit jobs to IBM Quantum
|
||||||
|
notebooks_that_submit_jobs = [
|
||||||
|
"docs/start/hello-world.ipynb",
|
||||||
|
"tutorials/build-repitition-codes/build-repitition-codes.ipynb",
|
||||||
|
"tutorials/chsh-inequality/chsh-inequality.ipynb",
|
||||||
|
"tutorials/grovers-algorithm/grovers.ipynb",
|
||||||
|
"tutorials/quantum-approximate-optimization-algorithm/qaoa.ipynb",
|
||||||
|
"tutorials/repeat-until-success/repeat-until-success.ipynb",
|
||||||
|
"tutorials/submitting-transpiled-circuits/submitting-transpiled-circuits.ipynb",
|
||||||
|
"tutorials/variational-quantum-eigensolver/vqe.ipynb",
|
||||||
|
]
|
|
@ -0,0 +1,23 @@
|
||||||
|
[build-system]
|
||||||
|
requires = [ "hatchling",]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "qiskit-docs-notebook-tester"
|
||||||
|
version = "0.0.1"
|
||||||
|
description = "Tool for the Qiskit docs team to test their notebooks"
|
||||||
|
requires-python = ">=3.9"
|
||||||
|
license = "Apache-2.0"
|
||||||
|
dependencies = [
|
||||||
|
"nbconvert~=7.16.0",
|
||||||
|
"nbformat~=5.9.2",
|
||||||
|
"ipykernel~=6.29.2",
|
||||||
|
"squeaky==0.7.0",
|
||||||
|
"tomli==2.0.1",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[project.authors]]
|
||||||
|
name = "Qiskit docs team"
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
|
test-docs-notebooks = "qiskit_docs_notebook_tester:main"
|
|
@ -10,6 +10,9 @@
|
||||||
# notice, and modified files need to carry a notice indicating that they have
|
# notice, and modified files need to carry a notice indicating that they have
|
||||||
# been altered from the originals.
|
# been altered from the originals.
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
import sys
|
import sys
|
||||||
|
@ -22,30 +25,33 @@ from typing import Iterator
|
||||||
import nbclient
|
import nbclient
|
||||||
import nbconvert
|
import nbconvert
|
||||||
import nbformat
|
import nbformat
|
||||||
|
import tomli
|
||||||
from qiskit_ibm_runtime import QiskitRuntimeService
|
from qiskit_ibm_runtime import QiskitRuntimeService
|
||||||
from squeaky import clean_notebook
|
from squeaky import clean_notebook
|
||||||
|
|
||||||
NOTEBOOKS_GLOB = "[!.]*/**/*.ipynb"
|
@dataclass
|
||||||
NOTEBOOKS_EXCLUDE = [
|
class Config:
|
||||||
"docs/api/**",
|
all_notebooks: str
|
||||||
"**/.ipynb_checkpoints/**",
|
notebooks_exclude: list[str]
|
||||||
]
|
notebooks_that_submit_jobs: list[str]
|
||||||
NOTEBOOKS_THAT_SUBMIT_JOBS = [
|
|
||||||
"docs/start/hello-world.ipynb",
|
|
||||||
"tutorials/build-repitition-codes/build-repitition-codes.ipynb",
|
|
||||||
"tutorials/chsh-inequality/chsh-inequality.ipynb",
|
|
||||||
"tutorials/grovers-algorithm/grovers.ipynb",
|
|
||||||
"tutorials/quantum-approximate-optimization-algorithm/qaoa.ipynb",
|
|
||||||
"tutorials/repeat-until-success/repeat-until-success.ipynb",
|
|
||||||
"tutorials/submitting-transpiled-circuits/submitting-transpiled-circuits.ipynb",
|
|
||||||
"tutorials/variational-quantum-eigensolver/vqe.ipynb",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def read(cls, path: str) -> Config:
|
||||||
|
"""
|
||||||
|
Load the globs from the TOML file
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return cls(**tomli.loads(Path(path).read_text()))
|
||||||
|
except TypeError as err:
|
||||||
|
raise ValueError(
|
||||||
|
f"Couldn't read config from {path}; check it exists and the"
|
||||||
|
" entries are correct."
|
||||||
|
) from err
|
||||||
|
|
||||||
def matches(path: Path, glob_list: list[str]) -> bool:
|
def matches(path: Path, glob_list: list[str]) -> bool:
|
||||||
return any(path.match(glob) for glob in glob_list)
|
return any(path.match(glob) for glob in glob_list)
|
||||||
|
|
||||||
def filter_paths(paths: list[Path], args: argparse.Namespace) -> Iterator[Path]:
|
def filter_paths(paths: list[Path], args: argparse.Namespace, config: Config) -> Iterator[Path]:
|
||||||
"""
|
"""
|
||||||
Filter out any paths we don't want to run, printing messages.
|
Filter out any paths we don't want to run, printing messages.
|
||||||
"""
|
"""
|
||||||
|
@ -55,20 +61,19 @@ def filter_paths(paths: list[Path], args: argparse.Namespace) -> Iterator[Path]:
|
||||||
print(f"ℹ️ Skipping {path}; file is not `.ipynb` format.")
|
print(f"ℹ️ Skipping {path}; file is not `.ipynb` format.")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if matches(path, NOTEBOOKS_EXCLUDE):
|
if matches(path, config.notebooks_exclude):
|
||||||
this_file = Path(__file__).resolve()
|
|
||||||
print(
|
print(
|
||||||
f"ℹ️ Skipping {path}; to run it, edit `NOTEBOOKS_EXCLUDE` in {this_file}."
|
f"ℹ️ Skipping {path}; to run it, edit `notebooks-exclude` in {args.config_path}."
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not submit_jobs and matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS):
|
if not submit_jobs and matches(path, config.notebooks_that_submit_jobs):
|
||||||
print(
|
print(
|
||||||
f"ℹ️ Skipping {path} as it submits jobs; use the --submit-jobs flag to run it."
|
f"ℹ️ Skipping {path} as it submits jobs; use the --submit-jobs flag to run it."
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if args.only_submit_jobs and not matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS):
|
if args.only_submit_jobs and not matches(path, config.notebooks_that_submit_jobs):
|
||||||
print(
|
print(
|
||||||
f"ℹ️ Skipping {path} as it does not submit jobs and the --only-submit-jobs flag is set."
|
f"ℹ️ Skipping {path} as it does not submit jobs and the --only-submit-jobs flag is set."
|
||||||
)
|
)
|
||||||
|
@ -171,20 +176,20 @@ async def _execute_notebook(filepath: Path, args: argparse.Namespace) -> nbforma
|
||||||
return nb
|
return nb
|
||||||
|
|
||||||
|
|
||||||
def find_notebooks() -> list[Path]:
|
def find_notebooks(config: Config) -> list[Path]:
|
||||||
"""
|
"""
|
||||||
Get paths to all notebooks in NOTEBOOKS_GLOB that are not excluded by
|
Get paths to notebooks in glob `all-notebooks` that are not excluded by
|
||||||
NOTEBOOKS_EXCLUDE
|
glob `notebooks-exclude`.
|
||||||
"""
|
"""
|
||||||
all_notebooks = Path(".").glob(NOTEBOOKS_GLOB)
|
all_notebooks = Path(".").glob(config.all_notebooks)
|
||||||
return [
|
return [
|
||||||
path
|
path
|
||||||
for path in all_notebooks
|
for path in all_notebooks
|
||||||
if not matches(path, NOTEBOOKS_EXCLUDE)
|
if not matches(path, config.notebooks_exclude)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def cancel_trailing_jobs(start_time: datetime) -> bool:
|
def cancel_trailing_jobs(start_time: datetime, config_path: str) -> bool:
|
||||||
"""
|
"""
|
||||||
Cancel any runtime jobs created after `start_time`.
|
Cancel any runtime jobs created after `start_time`.
|
||||||
|
|
||||||
|
@ -207,8 +212,8 @@ def cancel_trailing_jobs(start_time: datetime) -> bool:
|
||||||
|
|
||||||
print(
|
print(
|
||||||
f"⚠️ Cancelling {len(jobs)} job(s) created after {start_time}.\n"
|
f"⚠️ Cancelling {len(jobs)} job(s) created after {start_time}.\n"
|
||||||
"Add any notebooks that submit jobs to NOTEBOOKS_EXCLUDE in "
|
"Add any notebooks that submit jobs to `notebooks-that-submit-jobs` in "
|
||||||
"`scripts/nb-tester/test-notebook.py`."
|
f"`{config_path}`."
|
||||||
)
|
)
|
||||||
for job in jobs:
|
for job in jobs:
|
||||||
job.cancel()
|
job.cancel()
|
||||||
|
@ -225,7 +230,7 @@ def create_argument_parser() -> argparse.ArgumentParser:
|
||||||
help=(
|
help=(
|
||||||
"Paths to notebooks. If not provided, the script will search for "
|
"Paths to notebooks. If not provided, the script will search for "
|
||||||
"notebooks in `docs/`. To exclude a notebook from this process, add it "
|
"notebooks in `docs/`. To exclude a notebook from this process, add it "
|
||||||
"to `NOTEBOOKS_EXCLUDE` in the script."
|
"to `notebooks-exclude` in the config file."
|
||||||
),
|
),
|
||||||
nargs="*",
|
nargs="*",
|
||||||
)
|
)
|
||||||
|
@ -249,24 +254,28 @@ def create_argument_parser() -> argparse.ArgumentParser:
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="Same as --submit-jobs, but also skips notebooks that do not submit jobs to IBM Quantum",
|
help="Same as --submit-jobs, but also skips notebooks that do not submit jobs to IBM Quantum",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--config-path",
|
||||||
|
help="Path to a TOML file containing the globs for detecting and sorting notebooks",
|
||||||
|
)
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|
||||||
async def main() -> None:
|
async def _main() -> None:
|
||||||
args = create_argument_parser().parse_args()
|
args = create_argument_parser().parse_args()
|
||||||
paths = map(Path, args.filenames or find_notebooks())
|
config = Config.read(args.config_path)
|
||||||
filtered_paths = filter_paths(paths, args)
|
paths = map(Path, args.filenames or find_notebooks(config))
|
||||||
|
filtered_paths = filter_paths(paths, args, config)
|
||||||
|
|
||||||
# Execute notebooks
|
# Execute notebooks
|
||||||
start_time = datetime.now()
|
start_time = datetime.now()
|
||||||
print("Executing notebooks:")
|
print("Executing notebooks:")
|
||||||
results = await asyncio.gather(*(execute_notebook(path, args) for path in filtered_paths))
|
results = await asyncio.gather(*(execute_notebook(path, args) for path in filtered_paths))
|
||||||
print("Checking for trailing jobs...")
|
print("Checking for trailing jobs...")
|
||||||
results.append(cancel_trailing_jobs(start_time))
|
results.append(cancel_trailing_jobs(start_time, args.config_path))
|
||||||
if not all(results):
|
if not all(results):
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
def main():
|
||||||
if __name__ == "__main__":
|
asyncio.run(_main())
|
||||||
asyncio.run(main())
|
|
|
@ -1,10 +1,6 @@
|
||||||
# We pin to exact versions for a more reproducible and
|
# We pin to exact versions for a more reproducible and
|
||||||
# stable build.
|
# stable build.
|
||||||
nbconvert~=7.16.0
|
|
||||||
nbformat~=5.9.2
|
|
||||||
ipykernel~=6.29.2
|
|
||||||
qiskit[all]~=1.0
|
qiskit[all]~=1.0
|
||||||
qiskit-aer~=0.14.0.1
|
qiskit-aer~=0.14.0.1
|
||||||
qiskit-ibm-runtime~=0.23.0
|
qiskit-ibm-runtime~=0.23.0
|
||||||
qiskit-ibm-provider~=0.11.0
|
qiskit-ibm-provider~=0.11.0
|
||||||
squeaky==0.7.0
|
|
||||||
|
|
16
tox.ini
16
tox.ini
|
@ -4,12 +4,14 @@ env_list = py3
|
||||||
skipsdist = true
|
skipsdist = true
|
||||||
|
|
||||||
[testenv]
|
[testenv]
|
||||||
deps = -r scripts/nb-tester/requirements.txt
|
deps =
|
||||||
|
-e scripts/nb-tester
|
||||||
|
-r scripts/nb-tester/requirements.txt
|
||||||
setenv = PYDEVD_DISABLE_FILE_VALIDATION=1
|
setenv = PYDEVD_DISABLE_FILE_VALIDATION=1
|
||||||
commands = python scripts/nb-tester/test-notebook.py {posargs}
|
commands = test-docs-notebooks {posargs} --config-path scripts/nb-tester/notebooks.toml
|
||||||
|
|
||||||
[testenv:lint]
|
[testenv:{lint,fix}]
|
||||||
commands = squeaky --check --no-advice {posargs:docs tutorials}
|
deps = squeaky==0.7.0
|
||||||
|
commands =
|
||||||
[testenv:fix]
|
lint: squeaky --check --no-advice {posargs:docs tutorials}
|
||||||
commands = squeaky {posargs:docs tutorials}
|
fix: squeaky {posargs:docs tutorials}
|
||||||
|
|
Loading…
Reference in New Issue