andras/great-ai
andras/great-ai
Watch
1
0
Fork
You've already forked great-ai
 0
Code Issues Pull requests 2 Projects Releases 4 Packages 1 Wiki Activity Actions

Simplify notebook setup

Browse source
This commit is contained in:
Andras Schmelczer 2026-10-28 22:34:22 +00:00
commit 13dadc74ed
14 changed files with 66 additions and 531 deletions
Download patch file Download diff file Expand all files Collapse all files

57
docs/examples/scibert/data.ipynb
View file

@ -26,56 +26,8 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "4bf6cce1",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"scibert\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
]
},
{
@ -94,7 +46,10 @@
],
"source": [
"import json\n",
"from notebook_support import demo_annotations\n",
"import os\n",
"from pathlib import Path\n",
"\n",
"from docs.notebook_support import demo_annotations\n",
"\n",
"annotation_dir = os.environ.get(\"GREAT_AI_ANNOTATIONS_DIR\")\n",
"if annotation_dir:\n",

65
docs/examples/scibert/deploy.ipynb
View file

@ -22,68 +22,8 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "3775e11d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;196mConfiguration has been already initialised, overwriting.\n",
"Make sure to call `configure()` before importing your application code.\u001b[0m\n",
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"scibert\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "cell-3",
"metadata": {},
"outputs": [],
"source": [
"# The setup cell selects local model storage and a local TinyDB tracing database."
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
]
},
{
@ -144,6 +84,7 @@
],
"source": [
"from functools import lru_cache\n",
"from pathlib import Path\n",
"from great_ai import use_model\n",
"from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
"\n",

60
docs/examples/scibert/train.ipynb
View file

@ -27,56 +27,18 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"id": "824837fa",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n"
]
}
],
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"scibert\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
"from docs.notebook_support import FULL_RUN"
]
},
{
@ -244,7 +206,7 @@
" torch.set_num_threads(2)\n",
" vocabulary = [\"[PAD]\", \"[UNK]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"]\n",
" vocabulary += sorted(set(re.findall(r\"[a-z]+|[0-9]+|[^\\w\\s]\", \" \".join(X).lower())))\n",
" vocab_path = WORK_DIR / \"vocab.txt\"\n",
" vocab_path = Path(\"vocab.txt\")\n",
" vocab_path.write_text(\"\\n\".join(vocabulary) + \"\\n\", encoding=\"utf-8\")\n",
" tokenizer = BertTokenizer(vocab=str(vocab_path), do_lower_case=True)\n",
" model = BertForSequenceClassification(\n",
@ -288,7 +250,7 @@
"\n",
"\n",
"training_args = TrainingArguments(\n",
" output_dir=WORK_DIR / \"models\",\n",
" output_dir=\"models\",\n",
" per_device_train_batch_size=BATCH_SIZE,\n",
" per_device_eval_batch_size=BATCH_SIZE,\n",
" save_total_limit=5,\n",
@ -321,7 +283,7 @@
"# This evaluates the selected checkpoint on validation, not an untouched test set.\n",
"validation_metrics = trainer.evaluate(metric_key_prefix=\"validation\")\n",
"trainer.save_metrics(\"validation\", validation_metrics)\n",
"(WORK_DIR / \"models/validation-run.json\").write_text(\n",
"Path(\"models/validation-run.json\").write_text(\n",
" json.dumps(\n",
" {\n",
" \"mode\": \"scibert\" if FULL_RUN else \"tiny-random-bert-demo\",\n",
@ -399,9 +361,9 @@
"\n",
"# Trainer has loaded the selected validation checkpoint. Save its tokenizer\n",
"# too, so deployment never needs to download a different vocabulary.\n",
"trainer.save_model(str(WORK_DIR / \"pretrained\"))\n",
"tokenizer.save_pretrained(WORK_DIR / \"pretrained\")\n",
"save_model(WORK_DIR / \"pretrained\", key=\"scibert-highlights\", keep_last_n=2)"
"trainer.save_model(\"pretrained\")\n",
"tokenizer.save_pretrained(\"pretrained\")\n",
"save_model(Path(\"pretrained\"), key=\"scibert-highlights\", keep_last_n=2)"
]
},
{

56
docs/examples/simple-mag/train.ipynb
View file

@ -10,8 +10,8 @@
"By default this notebook uses 100 training and 30 test samples per class\n",
"and a small Naive Bayes parameter search for a quick local run. Set\n",
"`GREAT_AI_NOTEBOOK_FULL=1` before launching Jupyter to use all bundled data\n",
"and the larger search. Generated figures are saved under the printed\n",
"artifact directory. Local subset scores do not reproduce the historical\n",
"and the larger search. Generated figures are saved in the working directory.\n",
"Local subset scores do not reproduce the historical\n",
"full-dataset comparison with the 2019 SciBERT baseline (macro F1 0.6571)."
]
},
@ -24,56 +24,18 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"id": "4a11c71a",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple-mag\n"
]
}
],
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple-mag\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
"from docs.notebook_support import DATA_DIR, FULL_RUN, mag_lines"
]
},
{
@ -149,7 +111,7 @@
"ax.bar(domains, counts)\n",
"ax.set_ylabel(\"Count\")\n",
"fig.tight_layout()\n",
"fig.savefig(WORK_DIR / \"mag-distribution.png\", dpi=500)\n",
"fig.savefig(\"mag-distribution.png\", dpi=500)\n",
"None"
]
},
@ -2806,7 +2768,7 @@
" values_format=\".2f\",\n",
")\n",
"plt.tight_layout()\n",
"plt.savefig(WORK_DIR / \"mag-confusion.png\", dpi=600)"
"plt.savefig(\"mag-confusion.png\", dpi=600)"
]
}
],

56
docs/examples/simple/data.ipynb
View file

@ -29,56 +29,18 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"id": "a800dc4d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n"
]
}
],
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
"from docs.notebook_support import DATA_DIR, FULL_RUN, mag_lines"
]
},
{
@ -157,10 +119,10 @@
"source": [
"### Load\n",
"\n",
"Store the dataset in the local TinyDB tracing database. All notebooks in\n",
"this example share `.cache/notebooks/simple/` (or `simple-full/`). No S3 or\n",
"MongoDB credentials are needed. A dataset tag makes rerunning this cell\n",
"replace just this example's ground truth instead of duplicating it."
"Store the dataset in the default local TinyDB tracing database,\n",
"`tracing_database.json`. Run the data, training and deployment notebooks\n",
"from the same working directory to share it. A dataset tag makes rerunning\n",
"this cell replace just this example's ground truth instead of duplicating it."
]
},
{

52
docs/examples/simple/deploy.ipynb
View file

@ -28,56 +28,8 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "b7a9d250",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
]
},
{

54
docs/examples/simple/stacked-bars.ipynb
View file

@ -9,56 +9,8 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "f0cae36d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
]
},
{
@ -126,7 +78,7 @@
"ax.legend()\n",
"ax.set_ylabel(\"Ratio of all publications\")\n",
"fig.tight_layout()\n",
"fig.savefig(WORK_DIR / \"ss-distribution.png\", dpi=500)\n",
"fig.savefig(\"ss-distribution.png\", dpi=500)\n",
"None"
]
}

48
docs/examples/simple/train.ipynb
View file

@ -26,56 +26,18 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"id": "d8ae0655",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n"
]
}
],
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
"from docs.notebook_support import FULL_RUN"
]
},
{

21
docs/notebook_support.py
View file

@ -12,27 +12,6 @@ SURVEY_DIR = ROOT / "docs/surveys"
FULL_RUN = os.environ.get("GREAT_AI_NOTEBOOK_FULL") == "1"
def setup_notebook(workflow: str) -> Path:
"""Keep related notebooks' models and traces together, outside source files."""
from great_ai import ParallelTinyDbDriver, configure
from great_ai.large_file import LargeFileLocal
base = Path(os.environ.get("GREAT_AI_NOTEBOOK_WORKDIR", ROOT / ".cache/notebooks"))
suffix = "-full" if FULL_RUN else ""
work_dir = (base / (workflow + suffix)).resolve()
work_dir.mkdir(parents=True, exist_ok=True)
LargeFileLocal.cache_path = work_dir / "models-cache"
ParallelTinyDbDriver.path_to_db = work_dir / "tracing_database.json"
configure(
large_file_implementation=LargeFileLocal,
tracing_database_factory=ParallelTinyDbDriver,
disable_se4ml_banner=True,
seed=42,
)
print(f"Notebook artifacts: {work_dir}")
return work_dir
def mag_lines(path: Path, per_label: Optional[int] = None) -> List[str]:
"""Read the bundled MAG data, optionally limiting each class for a quick run."""
counts: Counter = Counter()

24
docs/surveys/surveys.ipynb
View file

@ -8,7 +8,8 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
],
"metadata": {}
},
@ -18,23 +19,7 @@
"metadata": {},
"execution_count": null,
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"surveys\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
"from docs.notebook_support import SURVEY_DIR"
],
"outputs": []
},
@ -46,7 +31,6 @@
"source": [
"import pandas as pd\n",
"\n",
"SURVEY_DIR = ROOT / \"docs/surveys\"\n",
"best_practices = pd.read_csv(SURVEY_DIR / \"Best practices assessment.csv\")\n",
"best_practices"
],
@ -161,7 +145,7 @@
")\n",
"plt.tight_layout()\n",
"\n",
"plt.savefig(WORK_DIR / \"best-practices.png\", dpi=150)\n",
"plt.savefig(\"best-practices.png\", dpi=150)\n",
"\n",
"stats.pearsonr(\n",
" [x for x, y in best_practice_score_se], [y for x, y in best_practice_score_se]\n",

29
docs/tutorial/deploy.ipynb
View file

@ -20,36 +20,11 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
],
"metadata": {}
},
{
"id": "bf8f9a33",
"cell_type": "code",
"metadata": {},
"execution_count": null,
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"tutorial\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
],
"outputs": []
},
{
"cell_type": "code",
"execution_count": null,

21
docs/tutorial/train.ipynb
View file

@ -22,7 +22,8 @@
"\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
"and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
],
"metadata": {}
},
@ -32,23 +33,7 @@
"metadata": {},
"execution_count": null,
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"tutorial\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
"from docs.notebook_support import DATA_DIR, FULL_RUN, mag_lines"
],
"outputs": []
},

12
scripts/run-notebooks.py
View file

@ -68,10 +68,16 @@ def main() -> None:
# A fresh workspace detects missing dependencies between notebook stages and
# avoids silently passing because an old model or database happens to exist.
with TemporaryDirectory(prefix="great-ai-notebooks-") as workspace:
env = dict(os.environ, GREAT_AI_NOTEBOOK_WORKDIR=workspace)
env = dict(os.environ)
env["PYTHONPATH"] = os.pathsep.join(
filter(None, [str(ROOT), env.get("PYTHONPATH")])
)
for name in args.notebooks or NOTEBOOKS:
path = (ROOT / name).resolve()
relative = path.relative_to(ROOT)
# Related notebooks share the library's default local storage.
work_dir = Path(workspace) / relative.parent
work_dir.mkdir(parents=True, exist_ok=True)
print(f"Running {relative}", flush=True)
notebook = nbformat.read(path, as_version=4)
nbformat.validate(notebook)
@ -86,7 +92,7 @@ def main() -> None:
]
client = NotebookClient(notebook, km=manager, timeout=args.timeout)
try:
client.execute(cwd=str(path.parent), env=env, cleanup_kc=True)
client.execute(cwd=str(work_dir), env=env, cleanup_kc=True)
finally:
destination = output_dir / relative
destination.parent.mkdir(parents=True, exist_ok=True)
@ -100,7 +106,7 @@ def main() -> None:
str(destination),
path.parent.name,
],
cwd=ROOT,
cwd=work_dir,
env=env,
check=True,
timeout=args.timeout,

42
tests/test_notebook_setup.py
View file

@ -1,42 +0,0 @@
import json
from pathlib import Path
from typing import Any, Dict
import pytest
from great_ai import ParallelTinyDbDriver
from great_ai.context import get_context
from great_ai.large_file import LargeFileLocal
ROOT = Path(__file__).resolve().parents[1]
NOTEBOOKS = sorted((ROOT / "docs").rglob("*.ipynb"))
@pytest.mark.parametrize(
"notebook", NOTEBOOKS, ids=lambda path: str(path.relative_to(ROOT))
)
def test_notebook_setup_uses_checkout_paths_from_another_directory(
isolated_context: None,
notebook: Path,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.syspath_prepend(str(ROOT))
from docs import notebook_support
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("GREAT_AI_NOTEBOOK_WORKDIR", str(tmp_path))
monkeypatch.setattr(notebook_support, "FULL_RUN", False)
monkeypatch.setattr(ParallelTinyDbDriver, "path_to_db", tmp_path / "traces.json")
cells = json.loads(notebook.read_text())["cells"]
setup = next(
"".join(cell["source"]) for cell in cells if cell["cell_type"] == "code"
)
namespace: Dict[str, Any] = {}
exec(compile(setup, str(notebook), "exec"), namespace)
work_dir = tmp_path / notebook.parent.name
assert work_dir.is_dir()
assert LargeFileLocal.cache_path == work_dir / "models-cache"
database = get_context().tracing_database
assert isinstance(database, ParallelTinyDbDriver)
assert database.path_to_db == work_dir / "tracing_database.json"
assert (notebook_support.DATA_DIR / "train.txt").is_file()