Run notebooks in CI

.forgejo/workflows/test.yml

@@ -27,8 +27,8 @@ jobs:
         run: uv python install 3.13
 
       - name: Install dependencies
-        # --seed gives the venv its own pip, which scripts/check-python.sh shells
-        # out to for its linters; '.[dev]' provides the rest (mypy resolves the
+        # --seed gives the venv its own pip for mypy's automatic type-stub
+        # installation; '.[dev]' provides the rest (mypy resolves the
         # package's own imports against the installed tree).
         run: |
           uv venv --seed --python 3.13
@@ -39,6 +39,35 @@ jobs:
           . .venv/bin/activate
           scripts/check-python.sh great_ai tests
 
+  notebooks:
+    name: Notebooks on Python ${{ matrix.python-version }}
+    runs-on: docker
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install notebook dependencies
+        run: |
+          uv venv --python ${{ matrix.python-version }}
+          uv pip install '.[notebooks]'
+
+      - name: Execute every notebook in a fresh workspace
+        env:
+          HF_HUB_OFFLINE: '1'
+          HF_DATASETS_OFFLINE: '1'
+        run: .venv/bin/python scripts/run-notebooks.py
+
   test:
     name: Test on Python ${{ matrix.python-version }}
     runs-on: docker

Dockerfile

@@ -1,6 +1,8 @@
 # syntax=docker/dockerfile:1.4
 FROM python:3.13-slim-trixie
 
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
 LABEL org.opencontainers.image.title="GreatAI package wrapper image"
 LABEL org.opencontainers.image.vendor="ScoutinScience B.V."
 LABEL org.opencontainers.image.authors="andras@schmelczer.dev"
@@ -18,8 +20,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt update &&\
 
 WORKDIR /dependencies
 COPY . great_ai
-RUN python3 -m pip --no-cache-dir install --upgrade pip &&\
-	pip install --no-cache-dir ./great_ai &&\
+RUN uv pip install --system --no-cache ./great_ai &&\
 	rm -rf great_ai
 
 HEALTHCHECK \

README.md

@@ -13,8 +13,12 @@ Applying AI is becoming increasingly more accessible, but many case studies have
 
 ## Example
 
+With [uv](https://docs.astral.sh/uv/getting-started/installation/) installed, create and activate a Python environment:
+
 ```sh
-pip install great-ai
+uv venv --python 3.13
+source .venv/bin/activate
+uv pip install great-ai
 ```
 
 Create a new file called `demo.py`
@@ -62,9 +66,10 @@ However, [research indicates](https://great-ai.scoutinscience.com) that professi
 ## Find `great-ai` on [PyPI](https://pypi.org/project/great-ai/)
 
 Requires Python 3.12 or later, including Python 3.13.
+With uv installed and your virtual environment active:
 
 ```sh
-pip install great-ai
+uv pip install great-ai
 ```
 
 ## Find `great-ai` on [DockerHub](https://hub.docker.com/repository/docker/schmelczera/great-ai)
@@ -80,10 +85,9 @@ Contributions are welcome.
 ### Install for development
 
 ```sh
-python3 -m venv --copies .env
+uv venv --seed --python 3.13 .env
 source .env/bin/activate
-pip install --upgrade pip
-pip install --editable '.[dev]'
+uv pip install --editable '.[dev]'
 ```
 
 ### Develop
@@ -105,3 +109,15 @@ mkdocs serve
 ```
 
 > Serve documentation.
+
+### Release
+
+With `git`, `python3`, and `uv` installed, run from a clean `main` checkout:
+
+```sh
+scripts/release.sh 0.2.0
+```
+
+This bumps the version, builds, commits, tags, and pushes `main` and the tag
+atomically. The tag triggers the PyPI and DockerHub workflows, which require
+`PYPI_TOKEN`, `DOCKER_USERNAME`, and `DOCKER_PASSWORD` in Forgejo repository secrets.

docs/notebook_support.py

@@ -0,0 +1,61 @@
+"""Shared local paths and small datasets for the documentation notebooks."""
+
+import json
+import os
+from collections import Counter
+from pathlib import Path
+from typing import List, Optional
+
+
+def setup_notebook(root: Path, workflow: str) -> Path:
+    """Keep related notebooks' models and traces together, outside source files."""
+    from great_ai import ParallelTinyDbDriver, configure
+    from great_ai.large_file import LargeFileLocal
+
+    base = Path(os.environ.get("GREAT_AI_NOTEBOOK_WORKDIR", root / ".cache/notebooks"))
+    suffix = "-full" if os.environ.get("GREAT_AI_NOTEBOOK_FULL") == "1" else ""
+    work_dir = (base / (workflow + suffix)).resolve()
+    work_dir.mkdir(parents=True, exist_ok=True)
+    LargeFileLocal.cache_path = work_dir / "models-cache"
+    ParallelTinyDbDriver.path_to_db = work_dir / "tracing_database.json"
+    configure(
+        large_file_implementation=LargeFileLocal,
+        tracing_database_factory=ParallelTinyDbDriver,
+        disable_se4ml_banner=True,
+        seed=42,
+    )
+    print(f"Notebook artifacts: {work_dir}")
+    return work_dir
+
+
+def mag_lines(path: Path, per_label: Optional[int] = None) -> List[str]:
+    """Read the bundled MAG data, optionally limiting each class for a quick run."""
+    counts: Counter = Counter()
+    lines = []
+    with path.open(encoding="utf-8") as stream:
+        for line in stream:
+            label = json.loads(line)["label"]
+            if per_label is None or counts[label] < per_label:
+                counts[label] += 1
+                lines.append(line)
+    return lines
+
+
+def demo_annotations() -> List[dict]:
+    """Synthetic annotations for exercising the pipeline, not measuring quality."""
+    annotations = [{}, {}]
+    for index, topic in enumerate(
+        ["protein", "cancer", "solar", "quantum", "neural", "genome"]
+    ):
+        for sample in range(10):
+            positive = sample % 2
+            sentence = (
+                f"Our {topic} method improves prediction accuracy in experiment {sample}."
+                if positive
+                else f"The {topic} dataset contains observations from experiment {sample}."
+            )
+            annotations[0][sentence] = positive
+            annotations[1][sentence] = (
+                1 - positive if (index * 10 + sample) % 11 == 0 else positive
+            )
+    return annotations

great_ai/utilities/external/pylatexenc/version.py

@@ -35,7 +35,7 @@
 #
 #       - LGTM looks good: https://lgtm.com/projects/g/phfaist/pylatexenc/
 #
-#       - python package creation works: (python setup.py sdist, pip install
+#       - python package creation works: (python setup.py sdist, uv pip install
 #         dist/pylatexenc-xxx.tar.gz)
 #
 #   2) update change log (doc/changes.rst)

pyproject.toml

@@ -53,6 +53,17 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+notebooks = [
+    "jupyterlab >= 4, < 5",
+    "ipykernel >= 6, < 8",
+    "ipywidgets >= 8, < 9",
+    "nbclient >= 0.10, < 1",
+    "transformers >= 5.21, < 6",
+    "datasets >= 5.1, < 6",
+    "torch >= 2.6, < 3",
+    "accelerate >= 1, < 2",
+    "pingouin >= 0.5, < 1",
+]
 dev = [
     "flit >= 4.1.0, < 5",
     "mkdocs >= 1.6.1, < 2",

scripts/check-python.sh

@@ -3,7 +3,7 @@
 set -e
 
 echo "Installing dependencies if necessary"
-python3 -m pip install autoflake isort black[jupyter] mypy flake8
+uv pip install --python "$(command -v python3)" autoflake isort 'black[jupyter]' mypy flake8
 
 for dir in "$@"; do
     echo "Checking $dir"

scripts/run-notebooks.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""Execute documentation notebooks in order, with the current Python interpreter."""
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import nbformat
+from jupyter_client import KernelManager
+from nbclient import NotebookClient
+
+ROOT = Path(__file__).resolve().parents[1]
+NOTEBOOKS = [
+    "docs/tutorial/train.ipynb",
+    "docs/tutorial/deploy.ipynb",
+    "docs/examples/simple/data.ipynb",
+    "docs/examples/simple/train.ipynb",
+    "docs/examples/simple/deploy.ipynb",
+    "docs/examples/simple/stacked-bars.ipynb",
+    "docs/examples/simple-mag/train.ipynb",
+    "docs/examples/scibert/data.ipynb",
+    "docs/examples/scibert/train.ipynb",
+    "docs/examples/scibert/deploy.ipynb",
+    "docs/surveys/surveys.ipynb",
+]
+
+CHECK_DEPLOYMENT = """
+import sys
+from importlib import import_module
+from pathlib import Path
+
+from fastapi.testclient import TestClient
+from great_ai.__main__ import find_app, get_script_name, get_script_name_of_notebook
+
+notebook = sys.argv[1]
+try:
+    module_name = get_script_name(notebook)
+    app_name = find_app(module_name)
+    assert app_name is not None, "The CLI could not import the deployment notebook"
+    module = import_module(module_name)
+    function_name = app_name.split(":")[1].split(".")[0]
+    service = getattr(module, function_name)
+    argument = "text" if sys.argv[2] == "simple" else "sentence"
+    with TestClient(service.app) as client:
+        response = client.post("/predict", json={argument: "Neural networks learn useful features."})
+        response.raise_for_status()
+        assert response.json()["output"] is not None
+finally:
+    Path(get_script_name_of_notebook(notebook)).unlink(missing_ok=True)
+"""
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("notebooks", nargs="*", help="Paths relative to the repository")
+    parser.add_argument("--timeout", type=int, default=600, help="Seconds per cell")
+    args = parser.parse_args()
+    output_dir = (
+        ROOT
+        / ".cache/executed-notebooks"
+        / f"python{sys.version_info.major}.{sys.version_info.minor}"
+    )
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # A fresh workspace detects missing dependencies between notebook stages and
+    # avoids silently passing because an old model or database happens to exist.
+    with TemporaryDirectory(prefix="great-ai-notebooks-") as workspace:
+        env = dict(os.environ, GREAT_AI_NOTEBOOK_WORKDIR=workspace)
+        for name in args.notebooks or NOTEBOOKS:
+            path = (ROOT / name).resolve()
+            relative = path.relative_to(ROOT)
+            print(f"Running {relative}", flush=True)
+            notebook = nbformat.read(path, as_version=4)
+            nbformat.validate(notebook)
+            manager = KernelManager(kernel_name="python3")
+            # Do not depend on a user-installed kernelspec pointing at this venv.
+            manager.kernel_spec.argv = [
+                sys.executable,
+                "-m",
+                "ipykernel_launcher",
+                "-f",
+                "{connection_file}",
+            ]
+            client = NotebookClient(notebook, km=manager, timeout=args.timeout)
+            try:
+                client.execute(cwd=str(path.parent), env=env, cleanup_kc=True)
+            finally:
+                destination = output_dir / relative
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                nbformat.write(notebook, destination)
+            if path.stem == "deploy":
+                subprocess.run(
+                    [
+                        sys.executable,
+                        "-c",
+                        CHECK_DEPLOYMENT,
+                        str(destination),
+                        path.parent.name,
+                    ],
+                    cwd=ROOT,
+                    env=env,
+                    check=True,
+                    timeout=args.timeout,
+                    stdout=subprocess.DEVNULL,
+                )
+            print(f"Passed {relative}", flush=True)
+    print(f"Executed notebooks saved to {output_dir}", flush=True)
+
+
+if __name__ == "__main__":
+    main()