Setup docs page

.github/workflows/docs.yaml

@@ -0,0 +1,29 @@
+name: Publish on PyPI
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: pip install --upgrade './[dev]'
+
+      - name: Build documentation
+        run: mkdocs gh-deploy
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.vscode/settings.json

@@ -73,5 +73,5 @@
   "python.linting.pylintEnabled": false,
   "python.linting.mypyEnabled": true,
   "python.testing.unittestEnabled": false,
-  "python.testing.pytestEnabled": true
+  "python.testing.pytestEnabled": true,
 }

README.md

@@ -40,7 +40,15 @@ Find the dashboard at [http://localhost:6060](http://localhost:6060/dashboard/).
 
 ### Contribute
 
+
+#### Install
+
 ```sh
 pip install 'great-ai[dev]'
-
 ```
+
+#### Documentation
+
+```sh
+mkdocs serve
+```

docs/great_ai_example-main/.dockerignore

@@ -0,0 +1,6 @@
+.venv
+.env
+**/.cache
+.git
+**/__pycache__
+.dockerignore

docs/great_ai_example-main/.gitignore

@@ -0,0 +1,10 @@
+.env
+.venv
+.DS_Store
+__pycache__
+.cache
+.mypy_cache
+.pytest_cache
+**/.ipynb_checkpoints
+**/tracing_database.json
+*.egg-info

docs/great_ai_example-main/.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+  "files.exclude": {
+    "**/__pycache__": true,
+    "**/.ipynb_checkpoints": true,
+    "**/.mypy_cache": true,
+    "**/.pytest_cache": true
+  },
+  "notebook.output.textLineLimit": 400,
+  "python.defaultInterpreterPath": ".env/bin/python"
+}

docs/great_ai_example-main/.vscode/tasks.json

@@ -0,0 +1,21 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Format and lint",
+      "type": "shell",
+      "command": "source .env/bin/activate; ./format.sh .",
+      "windows": {
+        "command": ".env\\bin\\activate.bat; .\\format.sh ."
+      },
+      "group": "test",
+      "presentation": {
+        "reveal": "always",
+        "panel": "new"
+      },
+      "options": {
+        "cwd": "${workspaceFolder}"
+      }
+    }
+  ]
+}

docs/great_ai_example-main/Dockerfile

@@ -0,0 +1,11 @@
+FROM test
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir ./great_ai
+
+COPY mongo.ini .
+COPY deploy.ipynb .
+
+# RUN python3 -m large_file --backend s3 -secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
+
+CMD ["deploy.ipynb"]

docs/great_ai_example-main/README.md

@@ -0,0 +1,23 @@
+# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)
+
+![steps](diagrams/scope.svg)
+
+## Install
+
+### System dependencies
+
+Make sure you have `python3`, `pip`, and `venv` installed.
+> On Ubuntu, execute: `sudo apt install -y python3 python3-pip python3-venv`
+
+### Install dependencies
+```sh
+python3 -m venv --copies .env 
+source .env/bin/activate
+
+pip install -r requirements.txt
+```
+## Execute 
+
+- [Part 1](src/data.ipynb)
+- [Part 2](src/train.ipynb)
+- [Part 3](src/deploy.ipynb)

docs/great_ai_example-main/format.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -e
+
+echo "Installing dependencies if necessary"
+python3 -m pip install --upgrade autoflake isort black[jupyter] mypy flake8
+
+echo "Formatting and checking $1"
+
+cd $1
+
+echo Running autoflake
+python3 -m autoflake --expand-star-imports --remove-all-unused-imports --ignore-init-module-imports --remove-unused-variables --in-place -r .
+
+echo Running isort
+python3 -m isort --profile black --skip .env .
+
+echo Running black
+python3 -m black . --exclude .env
+
+if ls *.py 1> /dev/null 2>&1; then
+    echo Running mypy
+    python3 -m mypy --namespace-packages --ignore-missing-imports --install-types --non-interactive --disallow-untyped-defs --disallow-incomplete-defs --pretty --follow-imports=silent --exclude=external/ --exclude=/build/ .
+fi
+
+echo Running Flake8
+python3 -m flake8 . --count --show-source --statistics --exclude=__init__.py,.env,external --ignore=E501,E722,E402,W503,E203
+
+cd -
+
+echo "Finished formatting"

docs/great_ai_example-main/requirements.txt

@@ -0,0 +1,4 @@
+great_ai
+notebook
+nbformat
+nbconvert

docs/great_ai_example-main/src/data.ipynb

@@ -0,0 +1,265 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Train a domain classifier on the [semantic scholar dataset](https://api.semanticscholar.org/corpus)\n",
+    "> Part 1: obtain and clean data\n",
+    "\n",
+    "![position of this step in the lifecycle](../diagrams/scope-data.svg)\n",
+    "> The blue boxes show the steps implemented in this notebook.\n",
+    "\n",
+    "### Extract\n",
+    "\n",
+    "This can be achieved by downloading a public dataset (such as in this case), or by having a Data Engineer setup and give us access to the organisation's data.\n",
+    "\n",
+    "In this case, we download the semantic scholar dataset from a public S3 bucket."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "MAX_CHUNK_COUNT = 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Processing 1 out of the 6002 available chunks'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import urllib.request\n",
+    "from random import shuffle\n",
+    "\n",
+    "manifest = (\n",
+    "    urllib.request.urlopen(\n",
+    "        \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n",
+    "    )\n",
+    "    .read()\n",
+    "    .decode()\n",
+    ")  # a list of available chunks separated by '\\n' characters\n",
+    "\n",
+    "lines = manifest.split()\n",
+    "shuffle(lines)\n",
+    "chunks = lines[:MAX_CHUNK_COUNT]\n",
+    "\n",
+    "f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Transform\n",
+    "\n",
+    "- Filter out non-English abstracts using `great_ai.utilities.predict_language`\n",
+    "- Project it to only keep the necessary components (text and labels), clean the textual content using `great_ai.utilities.clean`\n",
+    "- We will speed up processing using `great_ai.utilities.parallel_map`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Spacy model en_core_web_sm not found locally, downloading...\n",
+      "Collecting en-core-web-sm==3.3.0\n",
+      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl (12.8 MB)\n",
+      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.8/12.8 MB 3.6 MB/s eta 0:00:00\n",
+      "Requirement already satisfied: spacy<3.4.0,>=3.3.0.dev0 in ./.env/lib/python3.10/site-packages (from en-core-web-sm==3.3.0) (3.3.1)\n",
+      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.64.0)\n",
+      "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.9.1)\n",
+      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.4.3)\n",
+      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.4.1)\n",
+      "Requirement already satisfied: setuptools in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (59.6.0)\n",
+      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.7)\n",
+      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.2)\n",
+      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.7.8)\n",
+      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.7)\n",
+      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
+      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.28.0)\n",
+      "Requirement already satisfied: numpy>=1.15.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.23.0)\n",
+      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.6)\n",
+      "Requirement already satisfied: pathy>=0.3.5 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.6.1)\n",
+      "Requirement already satisfied: jinja2 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.1.2)\n",
+      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3.0)\n",
+      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.8.2)\n",
+      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.6)\n",
+      "Requirement already satisfied: thinc<8.1.0,>=8.0.14 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.0.17)\n",
+      "Requirement already satisfied: packaging>=20.0 in ./.env/lib/python3.10/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (21.3)\n",
+      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in ./.env/lib/python3.10/site-packages (from packaging>=20.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
+      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in ./.env/lib/python3.10/site-packages (from pathy>=0.3.5->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (5.2.1)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./.env/lib/python3.10/site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.2.0)\n",
+      "Requirement already satisfied: charset-normalizer~=2.0.0 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.12)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2022.6.15)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.env/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.26.9)\n",
+      "Requirement already satisfied: click<9.0.0,>=7.1.1 in ./.env/lib/python3.10/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.1.3)\n",
+      "Requirement already satisfied: MarkupSafe>=2.0 in ./.env/lib/python3.10/site-packages (from jinja2->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.1.1)\n",
+      "Installing collected packages: en-core-web-sm\n",
+      "Successfully installed en-core-web-sm-3.3.0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[38;5;226m2022-06-25 14:21:57,983 |  WARNING | Limiting concurrency to 1 because there are only 1 chunks\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:21:57,984 |     INFO | Starting parallel map (concurrency: 1, chunk size: 1)\u001b[0m\n",
+      "\u001b[38;5;226m2022-06-25 14:21:57,984 |  WARNING | Running in series, there is no reason for parallelism\u001b[0m\n",
+      "100%|██████████| 1/1 [03:26<00:00, 206.86s/it]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from typing import List, Tuple\n",
+    "import json\n",
+    "import gzip\n",
+    "from great_ai import parallel_map, clean, is_english, predict_language\n",
+    "\n",
+    "\n",
+    "def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
+    "    # Extract\n",
+    "    response = urllib.request.urlopen(\n",
+    "        f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}\"\n",
+    "    )  # a gzipped JSON Lines file\n",
+    "\n",
+    "    decompressed = gzip.decompress(response.read())\n",
+    "    decoded = decompressed.decode()\n",
+    "    chunk = [json.loads(line) for line in decoded.split(\"\\n\") if line]\n",
+    "\n",
+    "    # Transform\n",
+    "    return [\n",
+    "        (\n",
+    "            clean(\n",
+    "                f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
+    "                convert_to_ascii=True,\n",
+    "            ),  # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
+    "            c[\"fieldsOfStudy\"],\n",
+    "        )  # Create pairs of `(text, [...domains])`\n",
+    "        for c in chunk\n",
+    "        if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
+    "    ]\n",
+    "\n",
+    "\n",
+    "preprocessed_chunks = parallel_map(preprocess_chunk, chunks)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from itertools import chain\n",
+    "\n",
+    "preprocessed_data = list(chain(*preprocessed_chunks))\n",
+    "X, y = zip(\n",
+    "    *preprocessed_data\n",
+    ")  # X is the input, y is the expected (ground truth) output"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Load\n",
+    "\n",
+    "Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
+    "\n",
+    "#### Production-ready backend\n",
+    "\n",
+    "The MongoDB driver is automatically configured if  `mongo.ini` exists with the following scheme:\n",
+    "\n",
+    "```ini\n",
+    "mongo_connection_string=mongodb://localhost:27017/\n",
+    "mongo_database=my_great_ai_db\n",
+    "```\n",
+    "> You can install MongoDB from [here](https://www.mongodb.com/docs/manual/installation) or [use it as a service](https://www.mongodb.com/cloud/atlas/register)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[38;5;226m2022-06-25 14:25:24,989 |  WARNING | Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:25:24,990 |     INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising MongodbDriver\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:25:24,991 |     INFO | Found credentials file (/data/projects/great_ai_example/mongo.ini), initialising LargeFileMongo\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:25:24,992 |     INFO | Settings: configured ✅\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:25:24,993 |     INFO | 🔩 tracing_database: MongodbDriver\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:25:24,994 |     INFO | 🔩 large_file_implementation: LargeFileMongo\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:25:24,994 |     INFO | 🔩 is_production: False\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:25:24,995 |     INFO | 🔩 should_log_exception_stack: True\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:25:24,996 |     INFO | 🔩 prediction_cache_size: 512\u001b[0m\n",
+      "\u001b[38;5;39m2022-06-25 14:25:24,997 |     INFO | 🔩 dashboard_table_size: 50\u001b[0m\n",
+      "\u001b[38;5;226m2022-06-25 14:25:24,998 |  WARNING | You still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
+      "\u001b[38;5;226m2022-06-25 14:25:24,998 |  WARNING | > Find out more at https://se-ml.github.io/practices/\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "from great_ai import add_ground_truth\n",
+    "\n",
+    "add_ground_truth(X, y, train_split_ratio=0.8, test_split_ratio=0.2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Next: [Part 2](train.ipynb)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.10.4 ('.env': venv)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.4"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "c1f394f9662881005685eeb18d8f9f77079b1b8b9a5ece1f825bfa01fcb7f52f"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

docs/great_ai_example-main/src/mongo.ini

@@ -0,0 +1,2 @@
+mongo_connection_string=mongodb://localhost:27017/  # change this
+mongo_database=great_ai_example  # this will be automatically created