Simplify notebook setup
This commit is contained in:
parent
6134938743
commit
13dadc74ed
3 changed files with 66 additions and 531 deletions
|
|
@ -26,56 +26,8 @@
|
|||
"\n",
|
||||
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
|
||||
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
|
||||
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4bf6cce1",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
||||
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"# Works when opened from the repository root or any notebook directory.\n",
|
||||
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
|
||||
"ROOT = next(\n",
|
||||
" p\n",
|
||||
" for p in [_start, *_start.parents]\n",
|
||||
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
|
||||
")\n",
|
||||
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
|
||||
"from notebook_support import setup_notebook, mag_lines\n",
|
||||
"\n",
|
||||
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
|
||||
"WORK_DIR = setup_notebook(ROOT, \"scibert\")\n",
|
||||
"DATA_DIR = ROOT / \"docs/tutorial/data\""
|
||||
"and full-data options. Run related notebooks from the same working directory\n",
|
||||
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -94,7 +46,10 @@
|
|||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"from notebook_support import demo_annotations\n",
|
||||
"import os\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"from docs.notebook_support import demo_annotations\n",
|
||||
"\n",
|
||||
"annotation_dir = os.environ.get(\"GREAT_AI_ANNOTATIONS_DIR\")\n",
|
||||
"if annotation_dir:\n",
|
||||
|
|
|
|||
|
|
@ -22,68 +22,8 @@
|
|||
"\n",
|
||||
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
|
||||
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
|
||||
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "3775e11d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;196mConfiguration has been already initialised, overwriting.\n",
|
||||
"Make sure to call `configure()` before importing your application code.\u001b[0m\n",
|
||||
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
||||
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"# Works when opened from the repository root or any notebook directory.\n",
|
||||
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
|
||||
"ROOT = next(\n",
|
||||
" p\n",
|
||||
" for p in [_start, *_start.parents]\n",
|
||||
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
|
||||
")\n",
|
||||
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
|
||||
"from notebook_support import setup_notebook, mag_lines\n",
|
||||
"\n",
|
||||
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
|
||||
"WORK_DIR = setup_notebook(ROOT, \"scibert\")\n",
|
||||
"DATA_DIR = ROOT / \"docs/tutorial/data\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "cell-3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# The setup cell selects local model storage and a local TinyDB tracing database."
|
||||
"and full-data options. Run related notebooks from the same working directory\n",
|
||||
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -144,6 +84,7 @@
|
|||
],
|
||||
"source": [
|
||||
"from functools import lru_cache\n",
|
||||
"from pathlib import Path\n",
|
||||
"from great_ai import use_model\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
|
||||
"\n",
|
||||
|
|
|
|||
|
|
@ -27,56 +27,18 @@
|
|||
"\n",
|
||||
"Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
|
||||
"See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
|
||||
"and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
|
||||
"and full-data options. Run related notebooks from the same working directory\n",
|
||||
"to share the default `.cache/` model store and `tracing_database.json` database.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"id": "824837fa",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
||||
"\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/scibert\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"# Works when opened from the repository root or any notebook directory.\n",
|
||||
"_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
|
||||
"ROOT = next(\n",
|
||||
" p\n",
|
||||
" for p in [_start, *_start.parents]\n",
|
||||
" if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
|
||||
")\n",
|
||||
"sys.path.insert(0, str(ROOT / \"docs\"))\n",
|
||||
"from notebook_support import setup_notebook, mag_lines\n",
|
||||
"\n",
|
||||
"FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
|
||||
"WORK_DIR = setup_notebook(ROOT, \"scibert\")\n",
|
||||
"DATA_DIR = ROOT / \"docs/tutorial/data\""
|
||||
"from docs.notebook_support import FULL_RUN"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -244,7 +206,7 @@
|
|||
" torch.set_num_threads(2)\n",
|
||||
" vocabulary = [\"[PAD]\", \"[UNK]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"]\n",
|
||||
" vocabulary += sorted(set(re.findall(r\"[a-z]+|[0-9]+|[^\\w\\s]\", \" \".join(X).lower())))\n",
|
||||
" vocab_path = WORK_DIR / \"vocab.txt\"\n",
|
||||
" vocab_path = Path(\"vocab.txt\")\n",
|
||||
" vocab_path.write_text(\"\\n\".join(vocabulary) + \"\\n\", encoding=\"utf-8\")\n",
|
||||
" tokenizer = BertTokenizer(vocab=str(vocab_path), do_lower_case=True)\n",
|
||||
" model = BertForSequenceClassification(\n",
|
||||
|
|
@ -288,7 +250,7 @@
|
|||
"\n",
|
||||
"\n",
|
||||
"training_args = TrainingArguments(\n",
|
||||
" output_dir=WORK_DIR / \"models\",\n",
|
||||
" output_dir=\"models\",\n",
|
||||
" per_device_train_batch_size=BATCH_SIZE,\n",
|
||||
" per_device_eval_batch_size=BATCH_SIZE,\n",
|
||||
" save_total_limit=5,\n",
|
||||
|
|
@ -321,7 +283,7 @@
|
|||
"# This evaluates the selected checkpoint on validation, not an untouched test set.\n",
|
||||
"validation_metrics = trainer.evaluate(metric_key_prefix=\"validation\")\n",
|
||||
"trainer.save_metrics(\"validation\", validation_metrics)\n",
|
||||
"(WORK_DIR / \"models/validation-run.json\").write_text(\n",
|
||||
"Path(\"models/validation-run.json\").write_text(\n",
|
||||
" json.dumps(\n",
|
||||
" {\n",
|
||||
" \"mode\": \"scibert\" if FULL_RUN else \"tiny-random-bert-demo\",\n",
|
||||
|
|
@ -399,9 +361,9 @@
|
|||
"\n",
|
||||
"# Trainer has loaded the selected validation checkpoint. Save its tokenizer\n",
|
||||
"# too, so deployment never needs to download a different vocabulary.\n",
|
||||
"trainer.save_model(str(WORK_DIR / \"pretrained\"))\n",
|
||||
"tokenizer.save_pretrained(WORK_DIR / \"pretrained\")\n",
|
||||
"save_model(WORK_DIR / \"pretrained\", key=\"scibert-highlights\", keep_last_n=2)"
|
||||
"trainer.save_model(\"pretrained\")\n",
|
||||
"tokenizer.save_pretrained(\"pretrained\")\n",
|
||||
"save_model(Path(\"pretrained\"), key=\"scibert-highlights\", keep_last_n=2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue