Rename library
Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
parent
01befc54a6
commit
04404f2fc4
141 changed files with 839 additions and 837 deletions
8
.vscode/tasks.json
vendored
8
.vscode/tasks.json
vendored
|
|
@ -4,9 +4,9 @@
|
|||
{
|
||||
"label": "Format and lint Python",
|
||||
"type": "shell",
|
||||
"command": "source .env/bin/activate && scripts/format-python.sh good_ai && scripts/format-python.sh example",
|
||||
"command": "source .env/bin/activate && scripts/format-python.sh great_ai && scripts/format-python.sh example",
|
||||
"windows": {
|
||||
"command": ".env/bin/activate.bat; scripts/format-python.sh good_ai; scripts\\format-python.sh example"
|
||||
"command": ".env/bin/activate.bat; scripts/format-python.sh great_ai; scripts\\format-python.sh example"
|
||||
},
|
||||
"group": "test",
|
||||
"presentation": {
|
||||
|
|
@ -20,9 +20,9 @@
|
|||
{
|
||||
"label": "Test Python",
|
||||
"type": "shell",
|
||||
"command": "source .env/bin/activate && python3 -m pytest good_ai",
|
||||
"command": "source .env/bin/activate && python3 -m pytest great_ai",
|
||||
"windows": {
|
||||
"command": ".env/bin/activate.bat; python3 -m pytest good_ai"
|
||||
"command": ".env/bin/activate.bat; python3 -m pytest great_ai"
|
||||
},
|
||||
"group": "test",
|
||||
"presentation": {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,6 @@ mkdir ss-data && cd ss-data
|
|||
wget https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt
|
||||
wget -B https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/ -i manifest.txt
|
||||
cd -
|
||||
python3 -m good_ai.open_s3 --secrets s3.ini --push ss-data
|
||||
python3 -m great_ai.open_s3 --secrets s3.ini --push ss-data
|
||||
rm -rf ss-data
|
||||
```
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from random import shuffle
|
|||
from devtools import debug
|
||||
from predict_domain import predict_domain
|
||||
|
||||
from good_ai import process_batch
|
||||
from great_ai import process_batch
|
||||
|
||||
if __name__ == "__main__":
|
||||
with open(".cache/data-1/s2-corpus-323.json") as f:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
from predict_domain import predict_domain
|
||||
|
||||
from good_ai import serve
|
||||
from great_ai import serve
|
||||
|
||||
if __name__ == "__main__":
|
||||
serve(predict_domain)
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ from preprocess import preprocess
|
|||
from pydantic import BaseModel
|
||||
from sklearn.pipeline import Pipeline
|
||||
|
||||
from good_ai import log_argument, log_metric, use_model
|
||||
from good_ai.utilities.clean import clean
|
||||
from great_ai import log_argument, log_metric, use_model
|
||||
from great_ai.utilities.clean import clean
|
||||
|
||||
|
||||
class DomainPrediction(BaseModel):
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import re
|
||||
|
||||
from good_ai.utilities.lemmatize_text import lemmatize_text
|
||||
from great_ai.utilities.lemmatize_text import lemmatize_text
|
||||
|
||||
|
||||
def preprocess(text: str) -> str:
|
||||
lemmas = [re.sub(r"\d+", "NUM", lemma) for lemma in lemmatize_text(text)]
|
||||
lemmas = [re.sub(r"\d[\d.,]*", "NUM", lemma) for lemma in lemmatize_text(text)]
|
||||
return " ".join(lemmas)
|
||||
|
|
|
|||
|
|
@ -24,10 +24,10 @@
|
|||
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
||||
"from sklearn.model_selection import GridSearchCV\n",
|
||||
"\n",
|
||||
"from good_ai.utilities.clean import clean\n",
|
||||
"from good_ai.utilities.parallel_map import parallel_map\n",
|
||||
"from good_ai.utilities.language import is_english, predict_language\n",
|
||||
"from good_ai import save_model, set_default_config, LargeFile\n",
|
||||
"from great_ai.utilities.clean import clean\n",
|
||||
"from great_ai.utilities.parallel_map import parallel_map\n",
|
||||
"from great_ai.utilities.language import is_english, predict_language\n",
|
||||
"from great_ai import save_model, set_default_config, LargeFile\n",
|
||||
"\n",
|
||||
"from preprocess import preprocess"
|
||||
]
|
||||
|
|
@ -97,56 +97,56 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/5 [00:00<?, ?it/s]INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,7093)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,811)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,196)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,561)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1370)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,215)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,330)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,603)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing environment: 'document' @(26,0)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,533)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,713)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6244)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6549)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,997)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,549)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\)' @(1,674)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,748)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,70)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,128)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,112)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,175)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,321)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,700)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1180)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,754)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,487)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,390)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,942)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1471)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2235)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,27)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1176)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2502)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2545)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2856)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,84)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,257)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,391)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,648)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,346)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,716)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,782)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,958)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,964)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1017)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1287)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,104)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,119)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1139)\n",
|
||||
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,366)\n",
|
||||
" 0%| | 0/5 [00:00<?, ?it/s]INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,7093)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,811)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,196)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,561)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1370)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,215)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,330)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,603)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing environment: 'document' @(26,0)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,533)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,713)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6244)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6549)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,997)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,549)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\)' @(1,674)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,748)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,70)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,128)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,112)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,175)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,321)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,700)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1180)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,754)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,487)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,390)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,942)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1471)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2235)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,27)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1176)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2502)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2545)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2856)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,84)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,257)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,391)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,648)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,346)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,716)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,782)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,958)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,964)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1017)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1287)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,104)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,119)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1139)\n",
|
||||
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,366)\n",
|
||||
"100%|██████████| 5/5 [34:00<00:00, 408.07s/it] \n"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
1
great_ai/.gitignore
vendored
Normal file
1
great_ai/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
build
|
||||
33
great_ai/README.md
Normal file
33
great_ai/README.md
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# **S**coutinScience **U**tilitie**S** for text processing [](https://github.com/ScoutinScience/platform/actions/workflows/sus-general.yaml)
|
||||
|
||||
> amogus
|
||||
|
||||
## Exports
|
||||
|
||||
- [clean](src/sus/clean.py)
|
||||
- [unique](src/sus/unique.py)
|
||||
- [parallel_map](src/sus/parallel_map.py)
|
||||
- [match_names](src/sus/match_names/match_names.py)
|
||||
- [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
|
||||
|
||||
### Requires loading spacy model
|
||||
|
||||
> This is automatic but will require some time.
|
||||
|
||||
> Add this to the Dockerfile for caching the spaCy model:
|
||||
>
|
||||
> ```docker
|
||||
> RUN pip install --no-cache-dir en-core-web-lg@https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.2.0/en_core_web_lg-3.2.0-py3-none-any.whl
|
||||
> ```
|
||||
|
||||
- [spacy model (nlp)](src/sus/nlp.py)
|
||||
- [get_sentences](src/sus/get_sentences.py)
|
||||
- [lemmatize_text](src/sus/lemmatize_text.py)
|
||||
- [lemmatize_token](src/sus/lemmatize_token.py)
|
||||
- [publication TEI](src/sus/publication_tei/publication_tei.py)
|
||||
|
||||
## Development
|
||||
|
||||
- Optional booleans must have a default value of `False`.
|
||||
- No imports in top-level `__init__.py`, in order to not load anything unnecessary automatically
|
||||
- Should only be updated through a PR
|
||||
6
great_ai/example_secrets.ini
Normal file
6
great_ai/example_secrets.ini
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
[DEFAULT]
|
||||
aws_region_name = your_region_like_eu-west-2
|
||||
aws_access_key_id = YOUR_ACCESS_KEY_ID
|
||||
aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
|
||||
large_files_bucket_name = create_a_bucket_and_put_its_name_here
|
||||
aws_endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com
|
||||
119
great_ai/open_s3.md
Normal file
119
great_ai/open_s3.md
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
# [open(S3)](https://pypi.org/project/open-large/)
|
||||
|
||||
Storing, versioning, and downloading files from S3 made as easy as using `open()` in Python. Caching included.
|
||||
|
||||
## Motivation
|
||||
|
||||
Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation [doesn't support the deletion](https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository) of large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.
|
||||
|
||||
## Solution
|
||||
|
||||
```
|
||||
pip install open-large
|
||||
```
|
||||
|
||||
### Simple example
|
||||
|
||||
```python
|
||||
from open_s3 import LargeFile
|
||||
|
||||
LargeFile.configure_credentials({
|
||||
"aws_region_name": "your_region_like_eu-west-2",
|
||||
"aws_access_key_id": "YOUR_ACCESS_KEY_ID",
|
||||
"aws_secret_access_key": "YOUR_VERY_SECRET_ACCESS_KEY",
|
||||
"large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
|
||||
})
|
||||
|
||||
# Creates a new version and deletes the older version leaving the 3 most recently used intact
|
||||
with LargeFile("test.txt", "w", keep_last_n=3) as f:
|
||||
for i in range(100000):
|
||||
f.write('test\n')
|
||||
|
||||
# By default the latest version is returned
|
||||
# but an optional `version` keyword argument can be provided as well
|
||||
with LargeFile("test.txt", "r") as f:
|
||||
print(f.readlines()[0])
|
||||
```
|
||||
|
||||
> Automatically creates a file, writes to it, uploads it to S3, and then queries the most recent version of it.
|
||||
> In this case, the latest version is already in the local cache, no download is required.
|
||||
|
||||
### More details
|
||||
|
||||
`LargeFile` behaves like an opened file (in the background it is a temp file after all). Binary reading and writing is supported along with the [different keywords](https://docs.python.org/3/library/functions.html#open) `open()` accepts.
|
||||
|
||||
The local cache can be configured with these properties:
|
||||
|
||||
```python
|
||||
LargeFile.cache_path = Path('.cache')
|
||||
LargeFile.max_cache_size = "30 GB"
|
||||
```
|
||||
|
||||
#### I only need a path
|
||||
|
||||
In case you only need a path to the "remote" file, this pattern can be applied:
|
||||
|
||||
```python
|
||||
path_to_model = LargeFile("folder-of-my-bert-model", version=31).get()
|
||||
```
|
||||
|
||||
> This will first download the file/folder into your local cache folder. Then, it returns a `Path` object to the local version. Which can be turned into a string with `str(path_to_model)`.
|
||||
|
||||
The same approach works for uploads:
|
||||
|
||||
```python
|
||||
LargeFile("folder-of-my-bert-model").push('path_to_local/folder_or_file')
|
||||
```
|
||||
|
||||
> This way, both regular files and folders can be handled. The uploaded file is called **folder-of-my-bert-model**, the local name is ignored.
|
||||
|
||||
Lastly, all version of the remote object can be deleted by calling `LargeFile("my-file").delete()`. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.
|
||||
|
||||
### Command-line example
|
||||
|
||||
The package can be used as a module from the command-line to give you more flexibility.
|
||||
|
||||
#### Setup
|
||||
|
||||
Create an .ini file (or use _~/.aws/credentials_). It may look like this:
|
||||
|
||||
```ini
|
||||
[DEFAULT]
|
||||
aws_region_name = your_region_like_eu-west-2
|
||||
aws_access_key_id = YOUR_ACCESS_KEY_ID
|
||||
aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
|
||||
large_files_bucket_name = my_large_files
|
||||
endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com
|
||||
```
|
||||
|
||||
> Just like in [example secrets](example_secrets.ini).
|
||||
|
||||
#### Print the expected options
|
||||
|
||||
```sh
|
||||
python3 -m open_s3 --help
|
||||
```
|
||||
|
||||
#### Upload some files
|
||||
|
||||
```sh
|
||||
python3 -m open_s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
|
||||
```
|
||||
|
||||
> Only the filename is used as the S3 name, the rest of the path is ignored.
|
||||
|
||||
#### Download some files to the local cache
|
||||
|
||||
This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later.
|
||||
|
||||
```sh
|
||||
python3 -m open_s3 --secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
|
||||
```
|
||||
|
||||
> Versions may be specified by using `:`-s.
|
||||
|
||||
#### Delete remote files
|
||||
|
||||
```sh
|
||||
python3 -m open_s3 --secrets ~/.aws/credentials --delete my_first_file.json
|
||||
```
|
||||
7
great_ai/pyproject.toml
Normal file
7
great_ai/pyproject.toml
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
[build-system]
|
||||
requires = [
|
||||
"setuptools>=42",
|
||||
"setuptools-git",
|
||||
"wheel"
|
||||
]
|
||||
build-backend = "setuptools.build_meta"
|
||||
11
great_ai/requirements.txt
Normal file
11
great_ai/requirements.txt
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
unidecode >= 1.3.0
|
||||
multiprocess >= 0.70.0.0
|
||||
tqdm >= 4.0.0
|
||||
psutil >= 5.9.0
|
||||
beautifulsoup4 >= 4.10.0
|
||||
lxml >= 4.6.0
|
||||
spacy >= 3.2.0
|
||||
pydantic >= 1.8.0
|
||||
scikit-learn >= 1.0.0
|
||||
matplotlib >= 3.5.0
|
||||
numpy >= 1.22.0
|
||||
39
great_ai/setup.cfg
Normal file
39
great_ai/setup.cfg
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
[metadata]
|
||||
name = sus
|
||||
version = 0.0.3
|
||||
author = ScoutinScience B.V.
|
||||
author_email = andras@scoutinscience.com
|
||||
description = [S]coutinScience [u]tilitie[s]: reusable utilities for text processing
|
||||
long_description = file: README.md
|
||||
long_description_content_type = text/markdown
|
||||
url = https://github.com/ScoutinScience/platform
|
||||
project_urls =
|
||||
Bug Tracker = https://github.com/ScoutinScience/platform/issues
|
||||
classifiers =
|
||||
Programming Language :: Python :: 3
|
||||
Operating System :: OS Independent
|
||||
|
||||
[options]
|
||||
package_dir =
|
||||
= src
|
||||
packages = find:
|
||||
include_package_data = True
|
||||
python_requires = >=3.8
|
||||
install_requires =
|
||||
click < 8.1.0
|
||||
unidecode >= 1.3.0
|
||||
multiprocess >= 0.70.0.0
|
||||
tqdm >= 4.0.0
|
||||
psutil >= 5.9.0
|
||||
beautifulsoup4 >= 4.10.0
|
||||
lxml >= 4.6.0
|
||||
spacy >= 3.2.0
|
||||
pydantic >= 1.8.0
|
||||
scikit-learn >= 1.0.0
|
||||
matplotlib >= 3.5.0
|
||||
numpy >= 1.22.0
|
||||
langcodes[data] >= 3.3.0
|
||||
langdetect >= 1.0.9
|
||||
|
||||
[options.packages.find]
|
||||
where = src
|
||||
0
great_ai/src/__init__.py
Normal file
0
great_ai/src/__init__.py
Normal file
3
great_ai/src/great_ai/__init__.py
Normal file
3
great_ai/src/great_ai/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from .great_ai import *
|
||||
from .open_s3 import *
|
||||
from .utilities import *
|
||||
4
great_ai/src/great_ai/great_ai/__init__.py
Normal file
4
great_ai/src/great_ai/great_ai/__init__.py
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
from .context import set_default_config
|
||||
from .deploy import process_batch, process_single, serve
|
||||
from .metrics import log_argument, log_metric
|
||||
from .models import save_model, use_model
|
||||
2
great_ai/src/great_ai/great_ai/context/__init__.py
Normal file
2
great_ai/src/great_ai/great_ai/context/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
from .context import Context
|
||||
from .get_context import get_context, set_default_config
|
||||
15
great_ai/src/great_ai/great_ai/context/context.py
Normal file
15
great_ai/src/great_ai/great_ai/context/context.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
from logging import Logger
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..persistence import PersistenceDriver
|
||||
|
||||
|
||||
class Context(BaseModel):
|
||||
metrics_path: str
|
||||
persistence: PersistenceDriver
|
||||
is_production: bool
|
||||
logger: Logger
|
||||
|
||||
class Config:
|
||||
arbitrary_types_allowed = True
|
||||
85
great_ai/src/great_ai/great_ai/context/get_context.py
Normal file
85
great_ai/src/great_ai/great_ai/context/get_context.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import os
|
||||
import random
|
||||
from logging import INFO, Logger
|
||||
from pathlib import Path
|
||||
from typing import Optional, cast
|
||||
|
||||
from great_ai.open_s3 import LargeFile
|
||||
from great_ai.utilities.logger import create_logger
|
||||
|
||||
from ..persistence import ParallelTinyDbDriver, PersistenceDriver
|
||||
from .context import Context
|
||||
|
||||
_context: Optional[Context] = None
|
||||
PRODUCTION_KEY = "production"
|
||||
|
||||
|
||||
def get_context() -> Context:
|
||||
if _context is None:
|
||||
set_default_config()
|
||||
|
||||
return cast(Context, _context)
|
||||
|
||||
|
||||
def set_default_config(
|
||||
log_level: int = INFO,
|
||||
s3_config: Path = Path("s3.ini"),
|
||||
seed: int = 42,
|
||||
persistence_driver: PersistenceDriver = ParallelTinyDbDriver(
|
||||
Path("tracing_database.json")
|
||||
),
|
||||
is_production_mode_override: Optional[bool] = None,
|
||||
) -> None:
|
||||
global _context
|
||||
|
||||
logger = create_logger("great_ai", level=log_level)
|
||||
|
||||
is_production = _is_in_production_mode(
|
||||
override=is_production_mode_override, logger=logger
|
||||
)
|
||||
_initialize_large_file(s3_config, logger=logger)
|
||||
_set_seed(seed)
|
||||
|
||||
if not persistence_driver.is_threadsafe:
|
||||
logger.warning(
|
||||
f"The selected persistence driver ({type(persistence_driver).__name__}) is not threadsafe"
|
||||
)
|
||||
_context = Context(
|
||||
metrics_path="/metrics",
|
||||
persistence=persistence_driver,
|
||||
is_production=is_production,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
logger.info("Defaults: configured ✅")
|
||||
|
||||
|
||||
def _is_in_production_mode(override: Optional[bool], logger: Logger) -> bool:
|
||||
environment = os.environ.get("ENVIRONMENT", PRODUCTION_KEY).lower()
|
||||
is_production = environment == PRODUCTION_KEY if override is None else override
|
||||
|
||||
if is_production:
|
||||
logger.info("Running in production mode ✅")
|
||||
else:
|
||||
logger.warning("Running in development mode ‼️")
|
||||
|
||||
return is_production
|
||||
|
||||
|
||||
def _initialize_large_file(s3_config: Path, logger: Logger) -> None:
|
||||
if s3_config.exists():
|
||||
LargeFile.configure_credentials_from_file(s3_config)
|
||||
else:
|
||||
logger.info(
|
||||
f"Provided S3 config ({s3_config.resolve()}) not found, skipping LargeFile initialisation"
|
||||
)
|
||||
|
||||
|
||||
def _set_seed(seed: int) -> None:
|
||||
random.seed(seed)
|
||||
try:
|
||||
import numpy
|
||||
|
||||
numpy.random.seed(seed + 1)
|
||||
except ImportError:
|
||||
pass
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue