diff --git a/docs/examples/scibert/data.ipynb b/docs/examples/scibert/data.ipynb index 4487b65..f1a6b4a 100644 --- a/docs/examples/scibert/data.ipynb +++ b/docs/examples/scibert/data.ipynb @@ -26,56 +26,8 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "4bf6cce1", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n", - "\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n", - "\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n", - "\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n", - "\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n", - "\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n", - "\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"scibert\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ] }, { @@ -94,7 +46,10 @@ ], "source": [ "import json\n", - "from notebook_support import demo_annotations\n", + "import os\n", + "from pathlib import Path\n", + "\n", + "from docs.notebook_support import demo_annotations\n", "\n", "annotation_dir = os.environ.get(\"GREAT_AI_ANNOTATIONS_DIR\")\n", "if annotation_dir:\n", diff --git a/docs/examples/scibert/deploy.ipynb b/docs/examples/scibert/deploy.ipynb index dd0f0b1..2467494 100644 --- a/docs/examples/scibert/deploy.ipynb +++ b/docs/examples/scibert/deploy.ipynb @@ -22,68 +22,8 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "3775e11d", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[38;5;196mConfiguration has been already initialised, overwriting.\n", - "Make sure to call `configure()` before importing your application code.\u001b[0m\n", - "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n", - "\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n", - "\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n", - "\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n", - "\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n", - "\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n", - "\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"scibert\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "cell-3", - "metadata": {}, - "outputs": [], - "source": [ - "# The setup cell selects local model storage and a local TinyDB tracing database." + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ] }, { @@ -144,6 +84,7 @@ ], "source": [ "from functools import lru_cache\n", + "from pathlib import Path\n", "from great_ai import use_model\n", "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n", "\n", diff --git a/docs/examples/scibert/train.ipynb b/docs/examples/scibert/train.ipynb index a39fad3..21378b0 100644 --- a/docs/examples/scibert/train.ipynb +++ b/docs/examples/scibert/train.ipynb @@ -27,56 +27,18 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "824837fa", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n", - "\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n", - "\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n", - "\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n", - "\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n", - "\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n", - "\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n" - ] - } - ], + "outputs": [], "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"scibert\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" + "from docs.notebook_support import FULL_RUN" ] }, { @@ -244,7 +206,7 @@ " torch.set_num_threads(2)\n", " vocabulary = [\"[PAD]\", \"[UNK]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"]\n", " vocabulary += sorted(set(re.findall(r\"[a-z]+|[0-9]+|[^\\w\\s]\", \" \".join(X).lower())))\n", - " vocab_path = WORK_DIR / \"vocab.txt\"\n", + " vocab_path = Path(\"vocab.txt\")\n", " vocab_path.write_text(\"\\n\".join(vocabulary) + \"\\n\", encoding=\"utf-8\")\n", " tokenizer = BertTokenizer(vocab=str(vocab_path), do_lower_case=True)\n", " model = BertForSequenceClassification(\n", @@ -288,7 +250,7 @@ "\n", "\n", "training_args = TrainingArguments(\n", - " output_dir=WORK_DIR / \"models\",\n", + " output_dir=\"models\",\n", " per_device_train_batch_size=BATCH_SIZE,\n", " per_device_eval_batch_size=BATCH_SIZE,\n", " save_total_limit=5,\n", @@ -321,7 +283,7 @@ "# This evaluates the selected checkpoint on validation, not an untouched test set.\n", "validation_metrics = trainer.evaluate(metric_key_prefix=\"validation\")\n", "trainer.save_metrics(\"validation\", validation_metrics)\n", - "(WORK_DIR / \"models/validation-run.json\").write_text(\n", + "Path(\"models/validation-run.json\").write_text(\n", " json.dumps(\n", " {\n", " \"mode\": \"scibert\" if FULL_RUN else \"tiny-random-bert-demo\",\n", @@ -399,9 +361,9 @@ "\n", "# Trainer has loaded the selected validation checkpoint. Save its tokenizer\n", "# too, so deployment never needs to download a different vocabulary.\n", - "trainer.save_model(str(WORK_DIR / \"pretrained\"))\n", - "tokenizer.save_pretrained(WORK_DIR / \"pretrained\")\n", - "save_model(WORK_DIR / \"pretrained\", key=\"scibert-highlights\", keep_last_n=2)" + "trainer.save_model(\"pretrained\")\n", + "tokenizer.save_pretrained(\"pretrained\")\n", + "save_model(Path(\"pretrained\"), key=\"scibert-highlights\", keep_last_n=2)" ] }, { diff --git a/docs/examples/simple-mag/train.ipynb b/docs/examples/simple-mag/train.ipynb index dcee263..173717a 100644 --- a/docs/examples/simple-mag/train.ipynb +++ b/docs/examples/simple-mag/train.ipynb @@ -10,8 +10,8 @@ "By default this notebook uses 100 training and 30 test samples per class\n", "and a small Naive Bayes parameter search for a quick local run. Set\n", "`GREAT_AI_NOTEBOOK_FULL=1` before launching Jupyter to use all bundled data\n", - "and the larger search. Generated figures are saved under the printed\n", - "artifact directory. Local subset scores do not reproduce the historical\n", + "and the larger search. Generated figures are saved in the working directory.\n", + "Local subset scores do not reproduce the historical\n", "full-dataset comparison with the 2019 SciBERT baseline (macro F1 0.6571)." ] }, @@ -24,56 +24,18 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "4a11c71a", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n", - "\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n", - "\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n", - "\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n", - "\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n", - "\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n", - "\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple-mag\n" - ] - } - ], + "outputs": [], "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"simple-mag\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" + "from docs.notebook_support import DATA_DIR, FULL_RUN, mag_lines" ] }, { @@ -149,7 +111,7 @@ "ax.bar(domains, counts)\n", "ax.set_ylabel(\"Count\")\n", "fig.tight_layout()\n", - "fig.savefig(WORK_DIR / \"mag-distribution.png\", dpi=500)\n", + "fig.savefig(\"mag-distribution.png\", dpi=500)\n", "None" ] }, @@ -2806,7 +2768,7 @@ " values_format=\".2f\",\n", ")\n", "plt.tight_layout()\n", - "plt.savefig(WORK_DIR / \"mag-confusion.png\", dpi=600)" + "plt.savefig(\"mag-confusion.png\", dpi=600)" ] } ], diff --git a/docs/examples/simple/data.ipynb b/docs/examples/simple/data.ipynb index 4d0b397..500ad9a 100644 --- a/docs/examples/simple/data.ipynb +++ b/docs/examples/simple/data.ipynb @@ -29,56 +29,18 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "a800dc4d", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n", - "\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n", - "\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n", - "\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n", - "\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n", - "\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n", - "\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n" - ] - } - ], + "outputs": [], "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"simple\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" + "from docs.notebook_support import DATA_DIR, FULL_RUN, mag_lines" ] }, { @@ -157,10 +119,10 @@ "source": [ "### Load\n", "\n", - "Store the dataset in the local TinyDB tracing database. All notebooks in\n", - "this example share `.cache/notebooks/simple/` (or `simple-full/`). No S3 or\n", - "MongoDB credentials are needed. A dataset tag makes rerunning this cell\n", - "replace just this example's ground truth instead of duplicating it." + "Store the dataset in the default local TinyDB tracing database,\n", + "`tracing_database.json`. Run the data, training and deployment notebooks\n", + "from the same working directory to share it. A dataset tag makes rerunning\n", + "this cell replace just this example's ground truth instead of duplicating it." ] }, { diff --git a/docs/examples/simple/deploy.ipynb b/docs/examples/simple/deploy.ipynb index 5981f57..ab6af0e 100644 --- a/docs/examples/simple/deploy.ipynb +++ b/docs/examples/simple/deploy.ipynb @@ -28,56 +28,8 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "b7a9d250", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n", - "\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n", - "\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n", - "\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n", - "\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n", - "\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n", - "\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"simple\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ] }, { diff --git a/docs/examples/simple/stacked-bars.ipynb b/docs/examples/simple/stacked-bars.ipynb index 2881941..c1a6545 100644 --- a/docs/examples/simple/stacked-bars.ipynb +++ b/docs/examples/simple/stacked-bars.ipynb @@ -9,56 +9,8 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "f0cae36d", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n", - "\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n", - "\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n", - "\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n", - "\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n", - "\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n", - "\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"simple\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ] }, { @@ -126,7 +78,7 @@ "ax.legend()\n", "ax.set_ylabel(\"Ratio of all publications\")\n", "fig.tight_layout()\n", - "fig.savefig(WORK_DIR / \"ss-distribution.png\", dpi=500)\n", + "fig.savefig(\"ss-distribution.png\", dpi=500)\n", "None" ] } diff --git a/docs/examples/simple/train.ipynb b/docs/examples/simple/train.ipynb index bdc62ff..1e883cd 100644 --- a/docs/examples/simple/train.ipynb +++ b/docs/examples/simple/train.ipynb @@ -26,56 +26,18 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "d8ae0655", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n", - "\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n", - "\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n", - "\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n", - "\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n", - "\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n", - "\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n", - "\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n", - "\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n" - ] - } - ], + "outputs": [], "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"simple\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" + "from docs.notebook_support import FULL_RUN" ] }, { diff --git a/docs/notebook_support.py b/docs/notebook_support.py index 1024828..65358f5 100644 --- a/docs/notebook_support.py +++ b/docs/notebook_support.py @@ -12,27 +12,6 @@ SURVEY_DIR = ROOT / "docs/surveys" FULL_RUN = os.environ.get("GREAT_AI_NOTEBOOK_FULL") == "1" -def setup_notebook(workflow: str) -> Path: - """Keep related notebooks' models and traces together, outside source files.""" - from great_ai import ParallelTinyDbDriver, configure - from great_ai.large_file import LargeFileLocal - - base = Path(os.environ.get("GREAT_AI_NOTEBOOK_WORKDIR", ROOT / ".cache/notebooks")) - suffix = "-full" if FULL_RUN else "" - work_dir = (base / (workflow + suffix)).resolve() - work_dir.mkdir(parents=True, exist_ok=True) - LargeFileLocal.cache_path = work_dir / "models-cache" - ParallelTinyDbDriver.path_to_db = work_dir / "tracing_database.json" - configure( - large_file_implementation=LargeFileLocal, - tracing_database_factory=ParallelTinyDbDriver, - disable_se4ml_banner=True, - seed=42, - ) - print(f"Notebook artifacts: {work_dir}") - return work_dir - - def mag_lines(path: Path, per_label: Optional[int] = None) -> List[str]: """Read the bundled MAG data, optionally limiting each class for a quick run.""" counts: Counter = Counter() diff --git a/docs/surveys/surveys.ipynb b/docs/surveys/surveys.ipynb index cb80fe5..f4b7fed 100644 --- a/docs/surveys/surveys.ipynb +++ b/docs/surveys/surveys.ipynb @@ -8,7 +8,8 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ], "metadata": {} }, @@ -18,23 +19,7 @@ "metadata": {}, "execution_count": null, "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"surveys\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" + "from docs.notebook_support import SURVEY_DIR" ], "outputs": [] }, @@ -46,7 +31,6 @@ "source": [ "import pandas as pd\n", "\n", - "SURVEY_DIR = ROOT / \"docs/surveys\"\n", "best_practices = pd.read_csv(SURVEY_DIR / \"Best practices assessment.csv\")\n", "best_practices" ], @@ -161,7 +145,7 @@ ")\n", "plt.tight_layout()\n", "\n", - "plt.savefig(WORK_DIR / \"best-practices.png\", dpi=150)\n", + "plt.savefig(\"best-practices.png\", dpi=150)\n", "\n", "stats.pearsonr(\n", " [x for x, y in best_practice_score_se], [y for x, y in best_practice_score_se]\n", diff --git a/docs/tutorial/deploy.ipynb b/docs/tutorial/deploy.ipynb index 33d1767..1937989 100644 --- a/docs/tutorial/deploy.ipynb +++ b/docs/tutorial/deploy.ipynb @@ -20,36 +20,11 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ], "metadata": {} }, - { - "id": "bf8f9a33", - "cell_type": "code", - "metadata": {}, - "execution_count": null, - "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"tutorial\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" - ], - "outputs": [] - }, { "cell_type": "code", "execution_count": null, diff --git a/docs/tutorial/train.ipynb b/docs/tutorial/train.ipynb index b84320d..2c00097 100644 --- a/docs/tutorial/train.ipynb +++ b/docs/tutorial/train.ipynb @@ -22,7 +22,8 @@ "\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n", - "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" + "and full-data options. Run related notebooks from the same working directory\n", + "to share the default `.cache/` model store and `tracing_database.json` database.\n" ], "metadata": {} }, @@ -32,23 +33,7 @@ "metadata": {}, "execution_count": null, "source": [ - "import os\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "# Works when opened from the repository root or any notebook directory.\n", - "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n", - "ROOT = next(\n", - " p\n", - " for p in [_start, *_start.parents]\n", - " if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n", - ")\n", - "sys.path.insert(0, str(ROOT / \"docs\"))\n", - "from notebook_support import setup_notebook, mag_lines\n", - "\n", - "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n", - "WORK_DIR = setup_notebook(ROOT, \"tutorial\")\n", - "DATA_DIR = ROOT / \"docs/tutorial/data\"" + "from docs.notebook_support import DATA_DIR, FULL_RUN, mag_lines" ], "outputs": [] }, diff --git a/scripts/run-notebooks.py b/scripts/run-notebooks.py index 337cc51..7fbb6b8 100644 --- a/scripts/run-notebooks.py +++ b/scripts/run-notebooks.py @@ -68,10 +68,16 @@ def main() -> None: # A fresh workspace detects missing dependencies between notebook stages and # avoids silently passing because an old model or database happens to exist. with TemporaryDirectory(prefix="great-ai-notebooks-") as workspace: - env = dict(os.environ, GREAT_AI_NOTEBOOK_WORKDIR=workspace) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + filter(None, [str(ROOT), env.get("PYTHONPATH")]) + ) for name in args.notebooks or NOTEBOOKS: path = (ROOT / name).resolve() relative = path.relative_to(ROOT) + # Related notebooks share the library's default local storage. + work_dir = Path(workspace) / relative.parent + work_dir.mkdir(parents=True, exist_ok=True) print(f"Running {relative}", flush=True) notebook = nbformat.read(path, as_version=4) nbformat.validate(notebook) @@ -86,7 +92,7 @@ def main() -> None: ] client = NotebookClient(notebook, km=manager, timeout=args.timeout) try: - client.execute(cwd=str(path.parent), env=env, cleanup_kc=True) + client.execute(cwd=str(work_dir), env=env, cleanup_kc=True) finally: destination = output_dir / relative destination.parent.mkdir(parents=True, exist_ok=True) @@ -100,7 +106,7 @@ def main() -> None: str(destination), path.parent.name, ], - cwd=ROOT, + cwd=work_dir, env=env, check=True, timeout=args.timeout, diff --git a/tests/test_notebook_setup.py b/tests/test_notebook_setup.py deleted file mode 100644 index 2c6af5f..0000000 --- a/tests/test_notebook_setup.py +++ /dev/null @@ -1,42 +0,0 @@ -import json -from pathlib import Path -from typing import Any, Dict - -import pytest -from great_ai import ParallelTinyDbDriver -from great_ai.context import get_context -from great_ai.large_file import LargeFileLocal - -ROOT = Path(__file__).resolve().parents[1] -NOTEBOOKS = sorted((ROOT / "docs").rglob("*.ipynb")) - - -@pytest.mark.parametrize( - "notebook", NOTEBOOKS, ids=lambda path: str(path.relative_to(ROOT)) -) -def test_notebook_setup_uses_checkout_paths_from_another_directory( - isolated_context: None, - notebook: Path, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.syspath_prepend(str(ROOT)) - from docs import notebook_support - - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("GREAT_AI_NOTEBOOK_WORKDIR", str(tmp_path)) - monkeypatch.setattr(notebook_support, "FULL_RUN", False) - monkeypatch.setattr(ParallelTinyDbDriver, "path_to_db", tmp_path / "traces.json") - cells = json.loads(notebook.read_text())["cells"] - setup = next( - "".join(cell["source"]) for cell in cells if cell["cell_type"] == "code" - ) - namespace: Dict[str, Any] = {} - exec(compile(setup, str(notebook), "exec"), namespace) - work_dir = tmp_path / notebook.parent.name - assert work_dir.is_dir() - assert LargeFileLocal.cache_path == work_dir / "models-cache" - database = get_context().tracing_database - assert isinstance(database, ParallelTinyDbDriver) - assert database.path_to_db == work_dir / "tracing_database.json" - assert (notebook_support.DATA_DIR / "train.txt").is_file()