Add files

Signed-off-by: András Schmelczer <andras@schmelczer.dev>

.gitignore

@@ -0,0 +1,4 @@
+.env
+.DS_Store
+__pycache__
+.cache

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "cSpell.words": ["pydantic", "pyplot", "sklearn", "Tfidf", "Vectorizer"]
+}

example/.gitignore

@@ -0,0 +1,2 @@
+data
+.ipynb_checkpoints

example/config.py

@@ -0,0 +1 @@
+model_key = 

example/helper/__init__.py

@@ -0,0 +1 @@
+from .preprocess import preprocess

example/helper/preprocess.py

@@ -0,0 +1,12 @@
+from sus.clean import clean
+from sus.lemmatize_text import lemmatize_text
+import re
+
+
+def preprocess(text: str) -> str:
+    cleaned = clean(text, convert_to_ascii=True)
+    lemmas = [
+        re.sub(r'\d+', 'NUM', lemma)
+        for lemma in lemmatize_text(cleaned)
+    ]
+    return " ".join(lemmas)

example/models/__init__.py

@@ -0,0 +1 @@
+from .domain_prediction import DomainPrediction

example/models/domain_prediction.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+from typing import List
+
+
+class DomainPrediction(BaseModel):
+    domain: str
+    probability: float
+    explanation: List[str]

example/predict.py

@@ -0,0 +1,56 @@
+
+from models import DomainPrediction
+from typing import Iterable, List, Dict
+from sklearn.pipeline import Pipeline 
+from helper import preprocess
+import re
+# from sus.use_model import use_model
+from config import model_key
+
+
+# @use_model(model_key, version="latest")
+def predict(text: str, model: Pipeline, cut_off_probability: float=0.2) -> List[DomainPrediction]:
+    assert 0 <= cut_off_probability <= 1
+
+    feature_names = model.named_steps['vectorizer'].get_feature_names_out()
+
+    token_mapping = {
+        preprocess(original): original
+        for original in re.sub(r'[^a-zA-Z0-9]', ' ', text).split(' ')
+    }
+    
+    features = model.named_steps['vectorizer'].transform([text])
+    prediction = model.named_steps['classifier'].predict_proba(features)[0]
+    best_classes = sorted(enumerate(prediction), key=lambda v: v[1], reverse=True)
+
+    results: List[DomainPrediction] = []
+    for class_index, probability in best_classes:
+        weights = model.named_steps['classifier'].feature_log_prob_[class_index]
+       
+        results.append(DomainPrediction(
+            domain=model.named_steps['classifier'].classes_[class_index],
+            probability=round(probability * 100),
+            explanation=_get_explanation(
+                feature_names=feature_names,
+                features=features.A[0],
+                weights=weights,
+                token_mapping=token_mapping
+            )
+        ))
+
+        if sum(r.probability for r in results) >= cut_off_probability:
+            break
+    
+    return results
+
+
+def _get_explanation(feature_names: Iterable[str], features: Iterable[float], weights: Iterable[float], token_mapping: Dict[str, str]) -> List[str]:
+    influential = [
+        (value * weight, name)
+        for name, value, weight in zip(feature_names, features, weights)
+        if value
+    ]
+    
+    most_influential = sorted(influential, reverse=True)[:5]
+
+    return [token_mapping[v[1]] for v in most_influential]

example/secrets-backblaze.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+aws_region_name = us-west-4
+aws_access_key_id = 00411a2454be9f90000000001
+aws_secret_access_key = K004YOSYHePVUHymVzm/vb+AjGahXl8
+large_files_bucket_name = sa-large-files
+endpoint_url = https://s3.us-west-004.backblazeb2.com

good_ai/.gitignore

@@ -0,0 +1 @@
+build

good_ai/README.md

@@ -0,0 +1,33 @@
+# **S**coutinScience **U**tilitie**S** for text processing [![Lint and test ScoutinScience utilities](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml/badge.svg)](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
+
+> amogus
+
+## Exports
+
+- [clean](src/sus/clean.py)
+- [unique](src/sus/unique.py)
+- [parallel_map](src/sus/parallel_map.py)
+- [match_names](src/sus/match_names/match_names.py)
+- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
+
+### Requires loading spacy model
+
+> This is automatic but will require some time.
+
+> Add this to the Dockerfile for caching the spaCy model:
+>
+> ```docker
+> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
+> ```
+
+- [spacy model (nlp)](src/sus/nlp.py)
+- [get_sentences](src/sus/get_sentences.py)
+- [lemmatize_text](src/sus/lemmatize_text.py)
+- [lemmatize_token](src/sus/lemmatize_token.py)
+- [publication TEI](src/sus/publication_tei/publication_tei.py)
+
+## Development
+
+- Optional booleans must have a default value of `False`.
+- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
+- Should only be updated through a PR

good_ai/example_secrets.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+aws_region_name = your_region_like_eu-west-2
+aws_access_key_id = YOUR_ACCESS_KEY_ID
+aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
+large_files_bucket_name = create_a_bucket_and_put_its_name_here
+endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com

good_ai/open_s3.md

@@ -0,0 +1,119 @@
+# [open(S3)](https://pypi.org/project/open-large/)
+
+Storing, versioning, and downloading files from S3 made as easy as using `open()` in Python. Caching included.
+
+## Motivation
+
+Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation [doesn't support the deletion](https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository) of large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.
+
+## Solution
+
+```
+pip install open-large
+```
+
+### Simple example
+
+```python
+from open_s3 import LargeFile
+
+LargeFile.configure_credentials({
+    "aws_region_name": "your_region_like_eu-west-2",
+    "aws_access_key_id": "YOUR_ACCESS_KEY_ID",
+    "aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
+    "large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
+})
+
+# Creates a new version and deletes the older version leaving the 3 most recently used intact
+with LargeFile("test.txt", "w", keep_last_n=3) as f:
+    for i in range(100000):
+        f.write('test\n')
+
+# By default the latest version is returned
+# but an optional `version` keyword argument can be provided as well
+with LargeFile("test.txt", "r") as f:
+    print(f.readlines()[0])
+```
+
+> Automatically creates a file, writes to it, uploads it to S3, and then queries the most recent version of it.
+> In this case, the latest version is already in the local cache, no download is required.
+
+### More details
+
+`LargeFile` behaves like an opened file (in the background it is a temp file after all). Binary reading and writing is supported along with the [different keywords](https://docs.python.org/3/library/functions.html#open) `open()` accepts.
+
+The local cache can be configured with these properties:
+
+```python
+LargeFile.cache_path = Path('.cache')
+LargeFile.max_cache_size = "30 GB"
+```
+
+#### I only need a path
+
+In case you only need a path to the "remote" file, this pattern can be applied:
+
+```python
+path_to_model = LargeFile("folder-of-my-bert-model", version=31).get()
+```
+
+> This will first download the file/folder into your local cache folder. Then, it returns a `Path` object to the local version. Which can be turned into a string with `str(path_to_model)`.
+
+The same approach works for uploads:
+
+```python
+LargeFile("folder-of-my-bert-model").push('path_to_local/folder_or_file')
+```
+
+> This way, both regular files and folders can be handled. The uploaded file is called **folder-of-my-bert-model**, the local name is ignored.
+
+Lastly, all version of the remote object can be deleted by calling `LargeFile("my-file").delete()`. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.
+
+### Command-line example
+
+The package can be used as a module from the command-line to give you more flexibility.
+
+#### Setup
+
+Create an .ini file (or use _~/.aws/credentials_). It may look like this:
+
+```ini
+[DEFAULT]
+aws_region_name = your_region_like_eu-west-2
+aws_access_key_id = YOUR_ACCESS_KEY_ID
+aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
+large_files_bucket_name = my_large_files
+endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com
+```
+
+> Just like in [example secrets](example_secrets.ini).
+
+#### Print the expected options
+
+```sh
+python3 -m open_s3 --help
+```
+
+#### Upload some files
+
+```sh
+python3 -m open_s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
+```
+
+> Only the filename is used as the S3 name, the rest of the path is ignored.
+
+#### Download some files to the local cache
+
+This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later.
+
+```sh
+python3 -m open_s3 --secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
+```
+
+> Versions may be specified by using `:`-s.
+
+#### Delete remote files
+
+```sh
+python3 -m open_s3 --secrets ~/.aws/credentials --delete my_first_file.json
+```

good_ai/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "setuptools-git",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

good_ai/requirements.txt

@@ -0,0 +1,11 @@
+unidecode >= 1.3.0
+multiprocess >= 0.70.0.0
+tqdm >= 4.0.0
+psutil >= 5.9.0
+beautifulsoup4 >= 4.10.0
+lxml >= 4.6.0
+spacy >= 3.2.0
+pydantic >= 1.8.0
+scikit-learn >= 1.0.0
+matplotlib >= 3.5.0
+numpy >= 1.22.0

good_ai/setup.cfg

@@ -0,0 +1,39 @@
+[metadata]
+name = sus
+version = 0.0.3
+author = ScoutinScience B.V.
+author_email = andras@scoutinscience.com
+description = [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.com/ScoutinScience/platform
+project_urls =
+    Bug Tracker = https://github.com/ScoutinScience/platform/issues
+classifiers =
+    Programming Language :: Python :: 3
+    Operating System :: OS Independent
+
+[options]
+package_dir =
+    = src
+packages = find:
+include_package_data = True
+python_requires = >=3.8
+install_requires =
+    click < 8.1.0
+    unidecode >= 1.3.0
+    multiprocess >= 0.70.0.0
+    tqdm >= 4.0.0
+    psutil >= 5.9.0
+    beautifulsoup4 >= 4.10.0
+    lxml >= 4.6.0
+    spacy >= 3.2.0
+    pydantic >= 1.8.0
+    scikit-learn >= 1.0.0
+    matplotlib >= 3.5.0
+    numpy >= 1.22.0
+    langcodes[data] >= 3.3.0
+    langdetect >= 1.0.9
+
+[options.packages.find]
+where = src

good_ai/src/__init__.py

@@ -0,0 +1 @@
+

good_ai/src/open_s3/__init__.py

@@ -0,0 +1 @@
+from .large_file import LargeFile