{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cell-0",
   "metadata": {},
   "source": [
    "# Hardening and deployment\n",
    "\n",
    "> Part 3: Create production inference function\n",
    "\n",
    "![position of this step in the lifecycle](https://great-ai.schmelczer.dev/media/scope-deploy.svg)\n",
    "> The blue boxes show the steps implemented in this notebook.\n",
    "\n",
    "In [Part 2](https://great-ai.schmelczer.dev/examples/simple/train), we trained our AI model. Now, it's time to create **G**eneral **R**obust **E**nd-to-end **A**utomated **T**rustworthy deployment.\n",
    "\n",
    "## Create the inference function\n",
    "\n",
    "Next to the prediction, we also return the top-n most influential words based on their weights."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8e78891",
   "metadata": {},
   "source": [
    "## Local setup\n",
    "\n",
    "Use Python 3.12 or 3.13 and install this checkout with `uv pip install --editable '.[notebooks]'`.\n",
    "See **Run notebooks locally** in the repository README for the kernel, execution order,\n",
    "and full-data options. Generated artifacts stay under `.cache/notebooks/`.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b7a9d250",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
      "\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
      "\u001b[38;5;39mGreatAI (v0.1.11): configured ✅\u001b[0m\n",
      "\u001b[38;5;39m  🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
      "\u001b[38;5;39m  🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
      "\u001b[38;5;39m  🔩 is_production: False\u001b[0m\n",
      "\u001b[38;5;39m  🔩 should_log_exception_stack: True\u001b[0m\n",
      "\u001b[38;5;39m  🔩 prediction_cache_size: 512\u001b[0m\n",
      "\u001b[38;5;39m  🔩 dashboard_table_size: 50\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Notebook artifacts: /Users/schmelczer/Projects/great-ai/.cache/notebooks/simple\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "from pathlib import Path\n",
    "\n",
    "# Works when opened from the repository root or any notebook directory.\n",
    "_start = Path(globals().get(\"__file__\", Path.cwd())).resolve()\n",
    "ROOT = next(\n",
    "    p\n",
    "    for p in [_start, *_start.parents]\n",
    "    if (p / \"pyproject.toml\").is_file() and (p / \"great_ai\").is_dir()\n",
    ")\n",
    "sys.path.insert(0, str(ROOT / \"docs\"))\n",
    "from notebook_support import setup_notebook, mag_lines\n",
    "\n",
    "FULL_RUN = os.environ.get(\"GREAT_AI_NOTEBOOK_FULL\") == \"1\"\n",
    "WORK_DIR = setup_notebook(ROOT, \"simple\")\n",
    "DATA_DIR = ROOT / \"docs/tutorial/data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cell-3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;39mFetching cached versions of small-domain-prediction\u001b[0m\n",
      "\u001b[38;5;39mLatest version of small-domain-prediction is 0 (from versions: 0)\u001b[0m\n",
      "\u001b[38;5;39mFile small-domain-prediction-0 found in cache\u001b[0m\n",
      "\u001b[38;5;39mFile small-domain-prediction-0 found in cache\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "from sklearn.pipeline import Pipeline\n",
    "from great_ai.utilities import clean\n",
    "from great_ai import (\n",
    "    MultiLabelClassificationOutput,\n",
    "    ClassificationOutput,\n",
    "    GreatAI,\n",
    "    use_model,\n",
    "    parameter,\n",
    ")\n",
    "\n",
    "\n",
    "@GreatAI.create\n",
    "@use_model(\"small-domain-prediction\", version=\"latest\")\n",
    "@parameter(\"target_confidence\", validate=lambda c: 0 <= c <= 100)\n",
    "def predict_domain(\n",
    "    text: str, model: Pipeline, target_confidence: int = 50\n",
    ") -> MultiLabelClassificationOutput:\n",
    "    \"\"\"\n",
    "    Predict the scientific domain of the input text.\n",
    "    Return labels until their sum likelihood is larger than `target_confidence`.\n",
    "    \"\"\"\n",
    "\n",
    "    preprocessed = re.sub(r\"[^a-zA-Z\\s]\", \"\", clean(text, convert_to_ascii=True))\n",
    "    features = model.named_steps[\"vectorizer\"].transform([preprocessed])\n",
    "    prediction = model.named_steps[\"classifier\"].predict_proba(features)[0]\n",
    "\n",
    "    best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)\n",
    "\n",
    "    results = MultiLabelClassificationOutput()\n",
    "    for class_index, probability in best_classes:\n",
    "        results.labels.append(\n",
    "            get_label(\n",
    "                model=model,\n",
    "                features=features,\n",
    "                class_index=class_index,\n",
    "                probability=probability,\n",
    "            )\n",
    "        )\n",
    "\n",
    "        if sum(r.confidence for r in results.labels) >= target_confidence:\n",
    "            break\n",
    "\n",
    "    return results\n",
    "\n",
    "\n",
    "def get_label(\n",
    "    model: Pipeline, features: np.ndarray, class_index: int, probability: float\n",
    ") -> ClassificationOutput:\n",
    "    return ClassificationOutput(\n",
    "        label=model.named_steps[\"classifier\"].classes_[class_index],\n",
    "        confidence=round(probability * 100),\n",
    "        explanation=[\n",
    "            word\n",
    "            for _, word in sorted(\n",
    "                (\n",
    "                    (weight, word)\n",
    "                    for weight, word, count in zip(\n",
    "                        model.named_steps[\"classifier\"].feature_log_prob_[class_index],\n",
    "                        model.named_steps[\"vectorizer\"].get_feature_names_out(),\n",
    "                        features.toarray()[0],\n",
    "                    )\n",
    "                    if count > 0\n",
    "                ),\n",
    "                reverse=True,\n",
    "            )\n",
    "        ][:5],\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-4",
   "metadata": {},
   "source": [
    "## Check accuracy on the test split\n",
    "\n",
    "Anything under `if __name__ == \"__main__\":` will not be run when the script is executed by the `great-ai` CLI app. This, combined with `query_ground_truth` and the `/traces/{trace_id}/feedback` endpoint are ideal for creating a continuous-integration job for checking the quality of the model before deployment.\n",
    "\n",
    "We keep each paper's full set of accepted labels fixed. **Top-1 hit rate** is the fraction of papers whose highest-ranked returned label belongs to that set (an empty prediction is a miss). It is an application-specific hit criterion, not ordinary multiclass accuracy.\n",
    "\n",
    "We also report per-class and micro, macro, weighted and sample-averaged multilabel precision, recall and F1 for **all returned labels**, using the default `target_confidence=50`. The label vocabulary comes from the train/test annotations, independently of predictions. Each class's support is the number of test papers annotated with that class. A lower confidence target can return fewer labels and trade recall for precision. `zero_division=0` assigns zero to undefined metrics.\n",
    "\n",
    "The earlier report and confusion matrix used prediction-dependent targets. Those outputs have been removed; rerun this cell to generate the corrected report for your model.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cell-5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 84/84 [00:01<00:00, 81.74it/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top-1 hit rate (any accepted label): 0.405\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    business       0.17      0.78      0.28         9\n",
      "   economics       0.22      0.82      0.35        11\n",
      "   geography       0.38      0.93      0.54        14\n",
      "    medicine       0.38      0.86      0.52        14\n",
      "    politics       0.14      0.36      0.21        11\n",
      "  psychology       0.28      0.69      0.40        16\n",
      "   sociology       0.17      0.67      0.27         9\n",
      "\n",
      "   micro avg       0.25      0.74      0.37        84\n",
      "   macro avg       0.25      0.73      0.37        84\n",
      "weighted avg       0.26      0.74      0.38        84\n",
      " samples avg       0.27      0.74      0.39        84\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    from great_ai import query_ground_truth\n",
    "    from sklearn import metrics\n",
    "    from sklearn.preprocessing import MultiLabelBinarizer\n",
    "\n",
    "    data = query_ground_truth([\"test\", \"local-domain-example\"])\n",
    "    if not data:\n",
    "        raise ValueError(\"No test samples are available for evaluation\")\n",
    "\n",
    "    X = [d.input for d in data]\n",
    "    y_actual = [d.feedback for d in data]\n",
    "\n",
    "    # Retain the complete returned label set, in descending confidence order.\n",
    "    y_predicted = [\n",
    "        [label.label for label in d.output.labels]\n",
    "        for d in predict_domain.process_batch(\n",
    "            X, do_not_persist_traces=True, concurrency=2\n",
    "        )\n",
    "    ]\n",
    "    top_1_hit_rate = sum(\n",
    "        bool(predicted) and predicted[0] in actual\n",
    "        for predicted, actual in zip(y_predicted, y_actual)\n",
    "    ) / len(y_actual)\n",
    "    print(f\"Top-1 hit rate (any accepted label): {top_1_hit_rate:.3f}\")\n",
    "\n",
    "    # Fix the label vocabulary from annotations, including classes absent in test.\n",
    "    label_binarizer = MultiLabelBinarizer()\n",
    "    label_binarizer.fit(\n",
    "        [d.feedback for d in query_ground_truth([\"train\", \"local-domain-example\"])]\n",
    "        + y_actual\n",
    "    )\n",
    "    unknown_labels = {label for labels in y_predicted for label in labels}.difference(\n",
    "        label_binarizer.classes_\n",
    "    )\n",
    "    if unknown_labels:\n",
    "        raise ValueError(f\"Predictions contain unrecognised labels: {unknown_labels}\")\n",
    "    y_true_multilabel = label_binarizer.transform(y_actual)\n",
    "    y_pred_multilabel = label_binarizer.transform(y_predicted)\n",
    "    print(\n",
    "        metrics.classification_report(\n",
    "            y_true_multilabel,\n",
    "            y_pred_multilabel,\n",
    "            target_names=label_binarizer.classes_,\n",
    "            zero_division=0,\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "98ae85f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'labels': [{'label': 'psychology', 'confidence': 17.0, 'explanation': ['neural', 'networks', 'features']}, {'label': 'medicine', 'confidence': 16.0, 'explanation': ['features', 'neural', 'networks']}, {'label': 'economics', 'confidence': 16.0, 'explanation': ['networks', 'neural', 'features']}, {'label': 'geography', 'confidence': 15.0, 'explanation': ['features', 'neural', 'networks']}]}\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    from fastapi.testclient import TestClient\n",
    "\n",
    "    with TestClient(predict_domain.app) as client:\n",
    "        response = client.post(\n",
    "            \"/predict\", json={\"text\": \"Neural networks learn useful features.\"}\n",
    "        )\n",
    "        response.raise_for_status()\n",
    "        assert response.json()[\"output\"][\"labels\"]\n",
    "        print(response.json()[\"output\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv (3.13.5)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}