Rename library

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

.vscode/tasks.json

@@ -4,9 +4,9 @@
     {
       "label": "Format and lint Python",
       "type": "shell",
-      "command": "source .env/bin/activate && scripts/format-python.sh good_ai && scripts/format-python.sh example",
+      "command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh example",
       "windows": {
-        "command": ".env/bin/activate.bat; scripts/format-python.sh good_ai; scripts\\format-python.sh example"
+        "command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh example"
       },
       "group": "test",
       "presentation": {
@@ -20,9 +20,9 @@
     {
       "label": "Test Python",
       "type": "shell",
-      "command": "source .env/bin/activate && python3 -m pytest good_ai",
+      "command": "source .env/bin/activate && python3 -m pytest great_ai",
       "windows": {
-        "command": ".env/bin/activate.bat; python3 -m pytest good_ai"
+        "command": ".env/bin/activate.bat; python3 -m pytest great_ai"
       },
       "group": "test",
       "presentation": {

example/README.md

@@ -7,6 +7,6 @@ mkdir ss-data && cd ss-data
 wget https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt
 wget -B https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/ -i manifest.txt
 cd -
-python3 -m good_ai.open_s3 --secrets s3.ini --push ss-data
+python3 -m great_ai.open_s3 --secrets s3.ini --push ss-data
 rm -rf ss-data
 ```

example/main_batch.py

@@ -6,7 +6,7 @@ from random import shuffle
 from devtools import debug
 from predict_domain import predict_domain
 
-from good_ai import process_batch
+from great_ai import process_batch
 
 if __name__ == "__main__":
     with open(".cache/data-1/s2-corpus-323.json") as f:

example/main_service.py

@@ -3,7 +3,7 @@
 
 from predict_domain import predict_domain
 
-from good_ai import serve
+from great_ai import serve
 
 if __name__ == "__main__":
     serve(predict_domain)

example/predict_domain.py

@@ -5,8 +5,8 @@ from preprocess import preprocess
 from pydantic import BaseModel
 from sklearn.pipeline import Pipeline
 
-from good_ai import log_argument, log_metric, use_model
+from great_ai import log_argument, log_metric, use_model
-from good_ai.utilities.clean import clean
+from great_ai.utilities.clean import clean
 
 
 class DomainPrediction(BaseModel):

example/preprocess.py

@@ -1,8 +1,8 @@
 import re
 
-from good_ai.utilities.lemmatize_text import lemmatize_text
+from great_ai.utilities.lemmatize_text import lemmatize_text
 
 
 def preprocess(text: str) -> str:
-    lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(text)]
+    lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatize_text(text)]
     return " ".join(lemmas)

great_ai/.gitignore

@@ -0,0 +1 @@
+build

great_ai/README.md

@@ -0,0 +1,33 @@
+# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
+
+> amogus
+
+## Exports
+
+- [clean](src/sus/clean.py)
+- [unique](src/sus/unique.py)
+- [parallel_map](src/sus/parallel_map.py)
+- [match_names](src/sus/match_names/match_names.py)
+- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+
+### Requires loading spacy model
+
+> This is automatic but will require some time.
+
+> Add this to the Dockerfile for caching the spaCy model:
+>
+> ```docker
+> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
+> ```
+
+- [spacy model (nlp)](src/sus/nlp.py)
+- [get_sentences](src/sus/get_sentences.py)
+- [lemmatize_text](src/sus/lemmatize_text.py)
+- [lemmatize_token](src/sus/lemmatize_token.py)
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
+
+## Development
+
+- Optional booleans must have a default value of `False`.
+- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
+- Should only be updated through a PR

great_ai/example_secrets.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+aws_region_name = your_region_like_eu-west-2
+aws_access_key_id = YOUR_ACCESS_KEY_ID
+aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
+large_files_bucket_name = create_a_bucket_and_put_its_name_here
+aws_endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com

great_ai/open_s3.md

@@ -0,0 +1,119 @@
+# [open(S3)](https://pypi.org/project/open-large/)
+
+Storing, versioning, and downloading files from S3 made as easy as using `open()` in Python. Caching included.
+
+## Motivation
+
+Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation [doesn't support the deletion](https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository) of large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.
+
+## Solution
+
+```
+pip install open-large
+```
+
+### Simple example
+
+```python
+from open_s3 import LargeFile
+
+LargeFile.configure_credentials({
+    "aws_region_name": "your_region_like_eu-west-2",
+    "aws_access_key_id": "YOUR_ACCESS_KEY_ID",
+    "aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
+    "large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
+})
+
+# Creates a new version and deletes the older version leaving the 3 most recently used intact
+with LargeFile("test.txt", "w", keep_last_n=3) as f:
+    for i in range(100000):
+        f.write('test\n')
+
+# By default the latest version is returned
+# but an optional `version` keyword argument can be provided as well
+with LargeFile("test.txt", "r") as f:
+    print(f.readlines()[0])
+```
+
+> Automatically creates a file, writes to it, uploads it to S3, and then queries the most recent version of it.
+> In this case, the latest version is already in the local cache, no download is required.
+
+### More details
+
+`LargeFile` behaves like an opened file (in the background it is a temp file after all). Binary reading and writing is supported along with the [different keywords](https://docs.python.org/3/library/functions.html#open) `open()` accepts.
+
+The local cache can be configured with these properties:
+
+```python
+LargeFile.cache_path = Path('.cache')
+LargeFile.max_cache_size = "30 GB"
+```
+
+#### I only need a path
+
+In case you only need a path to the "remote" file, this pattern can be applied:
+
+```python
+path_to_model = LargeFile("folder-of-my-bert-model", version=31).get()
+```
+
+> This will first download the file/folder into your local cache folder. Then, it returns a `Path` object to the local version. Which can be turned into a string with `str(path_to_model)`.
+
+The same approach works for uploads:
+
+```python
+LargeFile("folder-of-my-bert-model").push('path_to_local/folder_or_file')
+```
+
+> This way, both regular files and folders can be handled. The uploaded file is called **folder-of-my-bert-model**, the local name is ignored.
+
+Lastly, all version of the remote object can be deleted by calling `LargeFile("my-file").delete()`. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.
+
+### Command-line example
+
+The package can be used as a module from the command-line to give you more flexibility.
+
+#### Setup
+
+Create an .ini file (or use _~/.aws/credentials_). It may look like this:
+
+```ini
+[DEFAULT]
+aws_region_name = your_region_like_eu-west-2
+aws_access_key_id = YOUR_ACCESS_KEY_ID
+aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
+large_files_bucket_name = my_large_files
+endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com
+```
+
+> Just like in [example secrets](example_secrets.ini).
+
+#### Print the expected options
+
+```sh
+python3 -m open_s3 --help
+```
+
+#### Upload some files
+
+```sh
+python3 -m open_s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
+```
+
+> Only the filename is used as the S3 name, the rest of the path is ignored.
+
+#### Download some files to the local cache
+
+This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later.
+
+```sh
+python3 -m open_s3 --secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
+```
+
+> Versions may be specified by using `:`-s.
+
+#### Delete remote files
+
+```sh
+python3 -m open_s3 --secrets ~/.aws/credentials --delete my_first_file.json
+```

great_ai/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "setuptools-git",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

great_ai/requirements.txt

@@ -0,0 +1,11 @@
+unidecode >= 1.3.0
+multiprocess >= 0.70.0.0
+tqdm >= 4.0.0
+psutil >= 5.9.0
+beautifulsoup4 >= 4.10.0
+lxml >= 4.6.0
+spacy >= 3.2.0
+pydantic >= 1.8.0
+scikit-learn >= 1.0.0
+matplotlib >= 3.5.0
+numpy >= 1.22.0

great_ai/setup.cfg

@@ -0,0 +1,39 @@
+[metadata]
+name = sus
+version = 0.0.3
+author = ScoutinScience B.V.
+author_email = andras@scoutinscience.com
+description = [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.com/ScoutinScience/platform
+project_urls =
+    Bug Tracker = https://github.com/ScoutinScience/platform/issues
+classifiers =
+    Programming Language :: Python :: 3
+    Operating System :: OS Independent
+
+[options]
+package_dir =
+    = src
+packages = find:
+include_package_data = True
+python_requires = >=3.8
+install_requires =
+    click < 8.1.0
+    unidecode >= 1.3.0
+    multiprocess >= 0.70.0.0
+    tqdm >= 4.0.0
+    psutil >= 5.9.0
+    beautifulsoup4 >= 4.10.0
+    lxml >= 4.6.0
+    spacy >= 3.2.0
+    pydantic >= 1.8.0
+    scikit-learn >= 1.0.0
+    matplotlib >= 3.5.0
+    numpy >= 1.22.0
+    langcodes[data] >= 3.3.0
+    langdetect >= 1.0.9
+
+[options.packages.find]
+where = src

great_ai/src/great_ai/__init__.py

@@ -0,0 +1,3 @@
+from .great_ai import *
+from .open_s3 import *
+from .utilities import *

great_ai/src/great_ai/great_ai/__init__.py

@@ -0,0 +1,4 @@
+from .context import set_default_config
+from .deploy import process_batch, process_single, serve
+from .metrics import log_argument, log_metric
+from .models import save_model, use_model

great_ai/src/great_ai/great_ai/context/__init__.py

@@ -0,0 +1,2 @@
+from .context import Context
+from .get_context import get_context, set_default_config

great_ai/src/great_ai/great_ai/context/context.py

@@ -0,0 +1,15 @@
+from logging import Logger
+
+from pydantic import BaseModel
+
+from ..persistence import PersistenceDriver
+
+
+class Context(BaseModel):
+    metrics_path: str
+    persistence: PersistenceDriver
+    is_production: bool
+    logger: Logger
+
+    class Config:
+        arbitrary_types_allowed = True

great_ai/src/great_ai/great_ai/context/get_context.py

@@ -0,0 +1,85 @@
+import os
+import random
+from logging import INFO, Logger
+from pathlib import Path
+from typing import Optional, cast
+
+from great_ai.open_s3 import LargeFile
+from great_ai.utilities.logger import create_logger
+
+from ..persistence import ParallelTinyDbDriver, PersistenceDriver
+from .context import Context
+
+_context: Optional[Context] = None
+PRODUCTION_KEY = "production"
+
+
+def get_context() -> Context:
+    if _context is None:
+        set_default_config()
+
+    return cast(Context, _context)
+
+
+def set_default_config(
+    log_level: int = INFO,
+    s3_config: Path = Path("s3.ini"),
+    seed: int = 42,
+    persistence_driver: PersistenceDriver = ParallelTinyDbDriver(
+        Path("tracing_database.json")
+    ),
+    is_production_mode_override: Optional[bool] = None,
+) -> None:
+    global _context
+
+    logger = create_logger("great_ai", level=log_level)
+
+    is_production = _is_in_production_mode(
+        override=is_production_mode_override, logger=logger
+    )
+    _initialize_large_file(s3_config, logger=logger)
+    _set_seed(seed)
+
+    if not persistence_driver.is_threadsafe:
+        logger.warning(
+            f"The selected persistence driver ({type(persistence_driver).__name__}) is not threadsafe"
+        )
+    _context = Context(
+        metrics_path="/metrics",
+        persistence=persistence_driver,
+        is_production=is_production,
+        logger=logger,
+    )
+
+    logger.info("Defaults: configured ✅")
+
+
+def _is_in_production_mode(override: Optional[bool], logger: Logger) -> bool:
+    environment = os.environ.get("ENVIRONMENT", PRODUCTION_KEY).lower()
+    is_production = environment == PRODUCTION_KEY if override is None else override
+
+    if is_production:
+        logger.info("Running in production mode ✅")
+    else:
+        logger.warning("Running in development mode ‼️")
+
+    return is_production
+
+
+def _initialize_large_file(s3_config: Path, logger: Logger) -> None:
+    if s3_config.exists():
+        LargeFile.configure_credentials_from_file(s3_config)
+    else:
+        logger.info(
+            f"Provided S3 config ({s3_config.resolve()}) not found, skipping LargeFile initialisation"
+        )
+
+
+def _set_seed(seed: int) -> None:
+    random.seed(seed)
+    try:
+        import numpy
+
+        numpy.random.seed(seed + 1)
+    except ImportError:
+        pass