andras/great-ai
andras/great-ai
Watch
1
0
Fork
You've already forked great-ai
 0
Code Issues Pull requests 2 Projects Releases 4 Packages 1 Wiki Activity Actions

Simplify notebook setup

Browse source
This commit is contained in:
Andras Schmelczer 2026-10-28 22:34:22 +00:00
commit 13dadc74ed
14 changed files with 66 additions and 531 deletions
Download patch file Download diff file Expand all files Collapse all files

57
docs/examples/scibert/data.ipynb
View file

@ -26,56 +26,8 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
] "to share the default `.cache/` model store and `tracing_database.json` database.\n"
},
{
"cell_type": "code",
"execution_count": 1,
"id": "4bf6cce1",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"scibert\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
] ]
}, },
{ {
@ -94,7 +46,10 @@
], ],
"source": [ "source": [
"import json\n", "import json\n",
"from notebook_support import demo_annotations\n", "import os\n",
"from pathlib import Path\n",
"\n",
"from docs.notebook_support import demo_annotations\n",
"\n", "\n",
"annotation_dir = os.environ.get(\"GREAT_AI_ANNOTATIONS_DIR\")\n", "annotation_dir = os.environ.get(\"GREAT_AI_ANNOTATIONS_DIR\")\n",
"if annotation_dir:\n", "if annotation_dir:\n",

65
docs/examples/scibert/deploy.ipynb
View file

@ -22,68 +22,8 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
] "to share the default `.cache/` model store and `tracing_database.json` database.\n"
},
{
"cell_type": "code",
"execution_count": 9,
"id": "3775e11d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;196mConfiguration has been already initialised, overwriting.\n",
"Make sure to call `configure()` before importing your application code.\u001b[0m\n",
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"scibert\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "cell-3",
"metadata": {},
"outputs": [],
"source": [
"# The setup cell selects local model storage and a local TinyDB tracing database."
] ]
}, },
{ {
@ -144,6 +84,7 @@
], ],
"source": [ "source": [
"from functools import lru_cache\n", "from functools import lru_cache\n",
"from pathlib import Path\n",
"from great_ai import use_model\n", "from great_ai import use_model\n",
"from transformers import AutoModelForSequenceClassification, AutoTokenizer\n", "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
"\n", "\n",

60
docs/examples/scibert/train.ipynb
View file

@ -27,56 +27,18 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": null,
"id": "824837fa", "id": "824837fa",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n"
]
}
],
"source": [ "source": [
"import os\n", "from docs.notebook_support import FULL_RUN"
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"scibert\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
] ]
}, },
{ {
@ -244,7 +206,7 @@
" torch.set_num_threads(2)\n", " torch.set_num_threads(2)\n",
" vocabulary = [\"[PAD]\", \"[UNK]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"]\n", " vocabulary = [\"[PAD]\", \"[UNK]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"]\n",
" vocabulary += sorted(set(re.findall(r\"[a-z]+|[0-9]+|[^\\w\\s]\", \" \".join(X).lower())))\n", " vocabulary += sorted(set(re.findall(r\"[a-z]+|[0-9]+|[^\\w\\s]\", \" \".join(X).lower())))\n",
" vocab_path = WORK_DIR / \"vocab.txt\"\n", " vocab_path = Path(\"vocab.txt\")\n",
" vocab_path.write_text(\"\\n\".join(vocabulary) + \"\\n\", encoding=\"utf-8\")\n", " vocab_path.write_text(\"\\n\".join(vocabulary) + \"\\n\", encoding=\"utf-8\")\n",
" tokenizer = BertTokenizer(vocab=str(vocab_path), do_lower_case=True)\n", " tokenizer = BertTokenizer(vocab=str(vocab_path), do_lower_case=True)\n",
" model = BertForSequenceClassification(\n", " model = BertForSequenceClassification(\n",
@ -288,7 +250,7 @@
"\n", "\n",
"\n", "\n",
"training_args = TrainingArguments(\n", "training_args = TrainingArguments(\n",
" output_dir=WORK_DIR / \"models\",\n", " output_dir=\"models\",\n",
" per_device_train_batch_size=BATCH_SIZE,\n", " per_device_train_batch_size=BATCH_SIZE,\n",
" per_device_eval_batch_size=BATCH_SIZE,\n", " per_device_eval_batch_size=BATCH_SIZE,\n",
" save_total_limit=5,\n", " save_total_limit=5,\n",
@ -321,7 +283,7 @@
"# This evaluates the selected checkpoint on validation, not an untouched test set.\n", "# This evaluates the selected checkpoint on validation, not an untouched test set.\n",
"validation_metrics = trainer.evaluate(metric_key_prefix=\"validation\")\n", "validation_metrics = trainer.evaluate(metric_key_prefix=\"validation\")\n",
"trainer.save_metrics(\"validation\", validation_metrics)\n", "trainer.save_metrics(\"validation\", validation_metrics)\n",
"(WORK_DIR / \"models/validation-run.json\").write_text(\n", "Path(\"models/validation-run.json\").write_text(\n",
" json.dumps(\n", " json.dumps(\n",
" {\n", " {\n",
" \"mode\": \"scibert\" if FULL_RUN else \"tiny-random-bert-demo\",\n", " \"mode\": \"scibert\" if FULL_RUN else \"tiny-random-bert-demo\",\n",
@ -399,9 +361,9 @@
"\n", "\n",
"# Trainer has loaded the selected validation checkpoint. Save its tokenizer\n", "# Trainer has loaded the selected validation checkpoint. Save its tokenizer\n",
"# too, so deployment never needs to download a different vocabulary.\n", "# too, so deployment never needs to download a different vocabulary.\n",
"trainer.save_model(str(WORK_DIR / \"pretrained\"))\n", "trainer.save_model(\"pretrained\")\n",
"tokenizer.save_pretrained(WORK_DIR / \"pretrained\")\n", "tokenizer.save_pretrained(\"pretrained\")\n",
"save_model(WORK_DIR / \"pretrained\", key=\"scibert-highlights\", keep_last_n=2)" "save_model(Path(\"pretrained\"), key=\"scibert-highlights\", keep_last_n=2)"
] ]
}, },
{ {

56
docs/examples/simple-mag/train.ipynb
View file

@ -10,8 +10,8 @@
"By default this notebook uses 100 training and 30 test samples per class\n", "By default this notebook uses 100 training and 30 test samples per class\n",
"and a small Naive Bayes parameter search for a quick local run. Set\n", "and a small Naive Bayes parameter search for a quick local run. Set\n",
"`GREAT_AI_NOTEBOOK_FULL=1` before launching Jupyter to use all bundled data\n", "`GREAT_AI_NOTEBOOK_FULL=1` before launching Jupyter to use all bundled data\n",
"and the larger search. Generated figures are saved under the printed\n", "and the larger search. Generated figures are saved in the working directory.\n",
"artifact directory. Local subset scores do not reproduce the historical\n", "Local subset scores do not reproduce the historical\n",
"full-dataset comparison with the 2019 SciBERT baseline (macro F1 0.6571)." "full-dataset comparison with the 2019 SciBERT baseline (macro F1 0.6571)."
] ]
}, },
@ -24,56 +24,18 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": null,
"id": "4a11c71a", "id": "4a11c71a",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple-mag\n"
]
}
],
"source": [ "source": [
"import os\n", "from docs.notebook_support import DATA_DIR, FULL_RUN, mag_lines"
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple-mag\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
] ]
}, },
{ {
@ -149,7 +111,7 @@
"ax.bar(domains, counts)\n", "ax.bar(domains, counts)\n",
"ax.set_ylabel(\"Count\")\n", "ax.set_ylabel(\"Count\")\n",
"fig.tight_layout()\n", "fig.tight_layout()\n",
"fig.savefig(WORK_DIR / \"mag-distribution.png\", dpi=500)\n", "fig.savefig(\"mag-distribution.png\", dpi=500)\n",
"None" "None"
] ]
}, },
@ -2806,7 +2768,7 @@
" values_format=\".2f\",\n", " values_format=\".2f\",\n",
")\n", ")\n",
"plt.tight_layout()\n", "plt.tight_layout()\n",
"plt.savefig(WORK_DIR / \"mag-confusion.png\", dpi=600)" "plt.savefig(\"mag-confusion.png\", dpi=600)"
] ]
} }
], ],

56
docs/examples/simple/data.ipynb
View file

@ -29,56 +29,18 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": null,
"id": "a800dc4d", "id": "a800dc4d",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n"
]
}
],
"source": [ "source": [
"import os\n", "from docs.notebook_support import DATA_DIR, FULL_RUN, mag_lines"
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
] ]
}, },
{ {
@ -157,10 +119,10 @@
"source": [ "source": [
"### Load\n", "### Load\n",
"\n", "\n",
"Store the dataset in the local TinyDB tracing database. All notebooks in\n", "Store the dataset in the default local TinyDB tracing database,\n",
"this example share `.cache/notebooks/simple/` (or `simple-full/`). No S3 or\n", "`tracing_database.json`. Run the data, training and deployment notebooks\n",
"MongoDB credentials are needed. A dataset tag makes rerunning this cell\n", "from the same working directory to share it. A dataset tag makes rerunning\n",
"replace just this example's ground truth instead of duplicating it." "this cell replace just this example's ground truth instead of duplicating it."
] ]
}, },
{ {

52
docs/examples/simple/deploy.ipynb
View file

@ -28,56 +28,8 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
] "to share the default `.cache/` model store and `tracing_database.json` database.\n"
},
{
"cell_type": "code",
"execution_count": 1,
"id": "b7a9d250",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
] ]
}, },
{ {

54
docs/examples/simple/stacked-bars.ipynb
View file

@ -9,56 +9,8 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
] "to share the default `.cache/` model store and `tracing_database.json` database.\n"
},
{
"cell_type": "code",
"execution_count": 1,
"id": "f0cae36d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
] ]
}, },
{ {
@ -126,7 +78,7 @@
"ax.legend()\n", "ax.legend()\n",
"ax.set_ylabel(\"Ratio of all publications\")\n", "ax.set_ylabel(\"Ratio of all publications\")\n",
"fig.tight_layout()\n", "fig.tight_layout()\n",
"fig.savefig(WORK_DIR / \"ss-distribution.png\", dpi=500)\n", "fig.savefig(\"ss-distribution.png\", dpi=500)\n",
"None" "None"
] ]
} }

48
docs/examples/simple/train.ipynb
View file

@ -26,56 +26,18 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": null,
"id": "d8ae0655", "id": "d8ae0655",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n"
]
}
],
"source": [ "source": [
"import os\n", "from docs.notebook_support import FULL_RUN"
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"simple\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
] ]
}, },
{ {

21
docs/notebook_support.py
View file

@ -12,27 +12,6 @@ SURVEY_DIR = ROOT / "docs/surveys"
FULL_RUN = os.environ.get("GREAT_AI_NOTEBOOK_FULL") == "1" FULL_RUN = os.environ.get("GREAT_AI_NOTEBOOK_FULL") == "1"
def setup_notebook(workflow: str) -> Path:
"""Keep related notebooks' models and traces together, outside source files."""
from great_ai import ParallelTinyDbDriver, configure
from great_ai.large_file import LargeFileLocal
base = Path(os.environ.get("GREAT_AI_NOTEBOOK_WORKDIR", ROOT / ".cache/notebooks"))
suffix = "-full" if FULL_RUN else ""
work_dir = (base / (workflow + suffix)).resolve()
work_dir.mkdir(parents=True, exist_ok=True)
LargeFileLocal.cache_path = work_dir / "models-cache"
ParallelTinyDbDriver.path_to_db = work_dir / "tracing_database.json"
configure(
large_file_implementation=LargeFileLocal,
tracing_database_factory=ParallelTinyDbDriver,
disable_se4ml_banner=True,
seed=42,
)
print(f"Notebook artifacts: {work_dir}")
return work_dir
def mag_lines(path: Path, per_label: Optional[int] = None) -> List[str]: def mag_lines(path: Path, per_label: Optional[int] = None) -> List[str]:
"""Read the bundled MAG data, optionally limiting each class for a quick run.""" """Read the bundled MAG data, optionally limiting each class for a quick run."""
counts: Counter = Counter() counts: Counter = Counter()

24
docs/surveys/surveys.ipynb
View file

@ -8,7 +8,8 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
], ],
"metadata": {} "metadata": {}
}, },
@ -18,23 +19,7 @@
"metadata": {}, "metadata": {},
"execution_count": null, "execution_count": null,
"source": [ "source": [
"import os\n", "from docs.notebook_support import SURVEY_DIR"
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"surveys\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
], ],
"outputs": [] "outputs": []
}, },
@ -46,7 +31,6 @@
"source": [ "source": [
"import pandas as pd\n", "import pandas as pd\n",
"\n", "\n",
"SURVEY_DIR = ROOT / \"docs/surveys\"\n",
"best_practices = pd.read_csv(SURVEY_DIR / \"Best practices assessment.csv\")\n", "best_practices = pd.read_csv(SURVEY_DIR / \"Best practices assessment.csv\")\n",
"best_practices" "best_practices"
], ],
@ -161,7 +145,7 @@
")\n", ")\n",
"plt.tight_layout()\n", "plt.tight_layout()\n",
"\n", "\n",
"plt.savefig(WORK_DIR / \"best-practices.png\", dpi=150)\n", "plt.savefig(\"best-practices.png\", dpi=150)\n",
"\n", "\n",
"stats.pearsonr(\n", "stats.pearsonr(\n",
" [x for x, y in best_practice_score_se], [y for x, y in best_practice_score_se]\n", " [x for x, y in best_practice_score_se], [y for x, y in best_practice_score_se]\n",

29
docs/tutorial/deploy.ipynb
View file

@ -20,36 +20,11 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
], ],
"metadata": {} "metadata": {}
}, },
{
"id": "bf8f9a33",
"cell_type": "code",
"metadata": {},
"execution_count": null,
"source": [
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"tutorial\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
],
"outputs": []
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,

21
docs/tutorial/train.ipynb
View file

@ -22,7 +22,8 @@
"\n", "\n",
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n", "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n", "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n" "and full-data options. Run related notebooks from the same working directory\n",
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
], ],
"metadata": {} "metadata": {}
}, },
@ -32,23 +33,7 @@
"metadata": {}, "metadata": {},
"execution_count": null, "execution_count": null,
"source": [ "source": [
"import os\n", "from docs.notebook_support import DATA_DIR, FULL_RUN, mag_lines"
"import sys\n",
"from pathlib import Path\n",
"\n",
"# Works when opened from the repository root or any notebook directory.\n",
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
"ROOT = next(\n",
" p\n",
" for p in [_start, *_start.parents]\n",
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
")\n",
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
"from notebook_support import setup_notebook, mag_lines\n",
"\n",
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
"WORK_DIR = setup_notebook(ROOT, \"tutorial\")\n",
"DATA_DIR = ROOT / \"docs/tutorial/data\""
], ],
"outputs": [] "outputs": []
}, },

12
scripts/run-notebooks.py
View file

@ -68,10 +68,16 @@ def main() -> None:
# A fresh workspace detects missing dependencies between notebook stages and # A fresh workspace detects missing dependencies between notebook stages and
# avoids silently passing because an old model or database happens to exist. # avoids silently passing because an old model or database happens to exist.
with TemporaryDirectory(prefix="great-ai-notebooks-") as workspace: with TemporaryDirectory(prefix="great-ai-notebooks-") as workspace:
env = dict(os.environ, GREAT_AI_NOTEBOOK_WORKDIR=workspace) env = dict(os.environ)
env["PYTHONPATH"] = os.pathsep.join(
filter(None, [str(ROOT), env.get("PYTHONPATH")])
)
for name in args.notebooks or NOTEBOOKS: for name in args.notebooks or NOTEBOOKS:
path = (ROOT / name).resolve() path = (ROOT / name).resolve()
relative = path.relative_to(ROOT) relative = path.relative_to(ROOT)
# Related notebooks share the library's default local storage.
work_dir = Path(workspace) / relative.parent
work_dir.mkdir(parents=True, exist_ok=True)
print(f"Running {relative}", flush=True) print(f"Running {relative}", flush=True)
notebook = nbformat.read(path, as_version=4) notebook = nbformat.read(path, as_version=4)
nbformat.validate(notebook) nbformat.validate(notebook)
@ -86,7 +92,7 @@ def main() -> None:
] ]
client = NotebookClient(notebook, km=manager, timeout=args.timeout) client = NotebookClient(notebook, km=manager, timeout=args.timeout)
try: try:
client.execute(cwd=str(path.parent), env=env, cleanup_kc=True) client.execute(cwd=str(work_dir), env=env, cleanup_kc=True)
finally: finally:
destination = output_dir / relative destination = output_dir / relative
destination.parent.mkdir(parents=True, exist_ok=True) destination.parent.mkdir(parents=True, exist_ok=True)
@ -100,7 +106,7 @@ def main() -> None:
str(destination), str(destination),
path.parent.name, path.parent.name,
], ],
cwd=ROOT, cwd=work_dir,
env=env, env=env,
check=True, check=True,
timeout=args.timeout, timeout=args.timeout,

42
tests/test_notebook_setup.py
View file

@ -1,42 +0,0 @@
import json
from pathlib import Path
from typing import Any, Dict
import pytest
from great_ai import ParallelTinyDbDriver
from great_ai.context import get_context
from great_ai.large_file import LargeFileLocal
ROOT = Path(__file__).resolve().parents[1]
NOTEBOOKS = sorted((ROOT / "docs").rglob("*.ipynb"))
@pytest.mark.parametrize(
"notebook", NOTEBOOKS, ids=lambda path: str(path.relative_to(ROOT))
)
def test_notebook_setup_uses_checkout_paths_from_another_directory(
isolated_context: None,
notebook: Path,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.syspath_prepend(str(ROOT))
from docs import notebook_support
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("GREAT_AI_NOTEBOOK_WORKDIR", str(tmp_path))
monkeypatch.setattr(notebook_support, "FULL_RUN", False)
monkeypatch.setattr(ParallelTinyDbDriver, "path_to_db", tmp_path / "traces.json")
cells = json.loads(notebook.read_text())["cells"]
setup = next(
"".join(cell["source"]) for cell in cells if cell["cell_type"] == "code"
)
namespace: Dict[str, Any] = {}
exec(compile(setup, str(notebook), "exec"), namespace)
work_dir = tmp_path / notebook.parent.name
assert work_dir.is_dir()
assert LargeFileLocal.cache_path == work_dir / "models-cache"
database = get_context().tracing_database
assert isinstance(database, ParallelTinyDbDriver)
assert database.path_to_db == work_dir / "tracing_database.json"
assert (notebook_support.DATA_DIR / "train.txt").is_file()