andras/great-ai
andras/great-ai
Watch
1
0
Fork
You've already forked great-ai
 0
Code Issues Pull requests 2 Projects Releases 4 Packages 1 Wiki Activity Actions

Move packages

Browse source 
Signed-off-by: András Schmelczer <andras@schmelczer.dev>
This commit is contained in:
Andras Schmelczer 2022-04-02 21:55:48 +02:00
commit acbdd96883
78 changed files with 123 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

5
example/main.py
View file

@ -1,9 +1,8 @@
import json
from open_s3 import LargeFile
from predict import predict_domain
from good_ai import LargeFile, process_batch
from predict_domain import predict_domain
from good_ai import process_batch
if __name__ == "__main__":
with open("data/s2-corpus-1583.json") as f:

2
example/predict_domain.py
View file

@ -7,7 +7,7 @@ from models import DomainPrediction
from sklearn.pipeline import Pipeline
from good_ai import use_model
from sus.clean import clean
from good_ai.utilities.clean import clean
@use_model(model_key, version="latest")
def predict_domain(

4
example/preprocess.py
View file

@ -1,7 +1,7 @@
import re
from sus.clean import clean
from sus.lemmatize_text import lemmatize_text
from good_ai.utilities.clean import clean
from good_ai.utilities.lemmatize_text import lemmatize_text
def preprocess(text: str) -> str:

117
example/train.ipynb
View file

@ -18,7 +18,23 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"outputs": [
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'open_s3'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/Users/andras/Projects/good-ai/example/train.ipynb Cell 2'\u001b[0m in \u001b[0;36m<cell line: 18>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=15'>16</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mre\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=16'>17</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mconfig\u001b[39;00m \u001b[39mimport\u001b[39;00m model_key\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=17'>18</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgood_ai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mclean\u001b[39;00m \u001b[39mimport\u001b[39;00m clean\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=18'>19</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgood_ai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mparallel_map\u001b[39;00m \u001b[39mimport\u001b[39;00m parallel_map\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/andras/Projects/good-ai/example/train.ipynb#ch0000001?line=19'>20</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgood_ai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mlanguage\u001b[39;00m \u001b[39mimport\u001b[39;00m is_english, predict_language\n",
"File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/__init__.py:1\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/__init__.py?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mgood_ai\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/__init__.py?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mutilities\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/__init__.py?line=2'>3</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mopen_s3\u001b[39;00m \u001b[39mimport\u001b[39;00m \u001b[39m*\u001b[39m\n",
"File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/good_ai/__init__.py:2\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/__init__.py?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mdeploy\u001b[39;00m \u001b[39mimport\u001b[39;00m process_batch\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/__init__.py?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mmodels\u001b[39;00m \u001b[39mimport\u001b[39;00m save_model, use_model\n",
"File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/good_ai/models/__init__.py:1\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/__init__.py?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39msave_model\u001b[39;00m \u001b[39mimport\u001b[39;00m save_model\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/__init__.py?line=1'>2</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39muse_model\u001b[39;00m \u001b[39mimport\u001b[39;00m use_model\n",
"File \u001b[0;32m~/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py:6\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=2'>3</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtyping\u001b[39;00m \u001b[39mimport\u001b[39;00m Optional, Union\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=4'>5</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mjoblib\u001b[39;00m \u001b[39mimport\u001b[39;00m dump\n\u001b[0;32m----> <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=5'>6</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mopen_s3\u001b[39;00m \u001b[39mimport\u001b[39;00m LargeFile\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=7'>8</a>\u001b[0m logger \u001b[39m=\u001b[39m logging\u001b[39m.\u001b[39mgetLogger(\u001b[39m\"\u001b[39m\u001b[39mmodels\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=10'>11</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msave_model\u001b[39m(\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=11'>12</a>\u001b[0m model: Union[Path, \u001b[39mstr\u001b[39m, \u001b[39mobject\u001b[39m], key: \u001b[39mstr\u001b[39m, keep_last_n: Optional[\u001b[39mint\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m <a href='file:///Users/andras/Projects/good-ai/good_ai/src/good_ai/good_ai/models/save_model.py?line=12'>13</a>\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mint\u001b[39m:\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'open_s3'"
]
}
],
"source": [
"import json\n",
"import matplotlib.pyplot as plt\n",
@ -30,19 +46,16 @@
"import pandas as pd\n",
"from joblib import dump\n",
"from sklearn.pipeline import Pipeline\n",
"from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
"from sklearn import metrics\n",
"from random import shuffle, seed\n",
"from sklearn.model_selection import GridSearchCV\n",
"from pprint import pprint\n",
"import re\n",
"from config import model_key\n",
"from sus.clean import clean\n",
"from sus.parallel_map import parallel_map\n",
"from sus.language import is_english, predict_language\n",
"from sus.lemmatize_text import lemmatize_text\n",
"from open_s3 import LargeFile\n",
"from good_ai import save_model\n",
"from good_ai.utilities.clean import clean\n",
"from good_ai.utilities.parallel_map import parallel_map\n",
"from good_ai.utilities.language import is_english, predict_language\n",
"from good_ai import LargeFile, save_model\n",
"from preprocess import preprocess\n",
"from predict_domain import predict_domain"
]
@ -90,7 +103,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 5/5 [00:00<00:00, 43062.67it/s]\n"
"100%|██████████| 5/5 [00:00<00:00, 50412.31it/s]\n"
]
}
],
@ -306,7 +319,89 @@
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO:open_s3:Fetching online versions of domain-prediction-v2\n",
"INFO:open_s3:Found versions: [1, 2]\n",
"INFO:open_s3:Copying file for domain-prediction-v2-3\n",
"INFO:open_s3:Compressing domain-prediction-v2-3\n",
"INFO:open_s3:Uploading domain-prediction-v2-3 to S3 from /var/folders/5g/yd_5_wg548q0yb4lnvg0y3q40000gn/T/large-file-hkyimkq5\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 262144/15169091 bytes (1.7%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 524288/15169091 bytes (3.5%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 786432/15169091 bytes (5.2%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 1048576/15169091 bytes (6.9%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 1310720/15169091 bytes (8.6%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 1572864/15169091 bytes (10.4%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 1835008/15169091 bytes (12.1%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 2097152/15169091 bytes (13.8%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 2359296/15169091 bytes (15.6%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 2621440/15169091 bytes (17.3%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 2883584/15169091 bytes (19.0%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 3145728/15169091 bytes (20.7%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 3407872/15169091 bytes (22.5%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 3670016/15169091 bytes (24.2%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 3932160/15169091 bytes (25.9%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 4194304/15169091 bytes (27.7%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 4456448/15169091 bytes (29.4%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 4718592/15169091 bytes (31.1%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 4980736/15169091 bytes (32.8%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 5242880/15169091 bytes (34.6%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 5505024/15169091 bytes (36.3%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 5767168/15169091 bytes (38.0%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 6029312/15169091 bytes (39.7%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 6291456/15169091 bytes (41.5%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 6553600/15169091 bytes (43.2%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 6815744/15169091 bytes (44.9%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 7077888/15169091 bytes (46.7%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 7340032/15169091 bytes (48.4%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 7602176/15169091 bytes (50.1%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 7864320/15169091 bytes (51.8%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 8126464/15169091 bytes (53.6%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 8388608/15169091 bytes (55.3%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 8650752/15169091 bytes (57.0%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 8912896/15169091 bytes (58.8%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 9175040/15169091 bytes (60.5%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 9437184/15169091 bytes (62.2%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 9699328/15169091 bytes (63.9%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 9961472/15169091 bytes (65.7%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 10223616/15169091 bytes (67.4%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 10485760/15169091 bytes (69.1%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 10747904/15169091 bytes (70.9%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11010048/15169091 bytes (72.6%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11272192/15169091 bytes (74.3%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11534336/15169091 bytes (76.0%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 11796480/15169091 bytes (77.8%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12023363/15169091 bytes (79.3%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12285507/15169091 bytes (81.0%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12547651/15169091 bytes (82.7%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 12809795/15169091 bytes (84.4%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13071939/15169091 bytes (86.2%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13334083/15169091 bytes (87.9%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13596227/15169091 bytes (89.6%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 13858371/15169091 bytes (91.4%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14120515/15169091 bytes (93.1%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14382659/15169091 bytes (94.8%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14644803/15169091 bytes (96.5%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 14906947/15169091 bytes (98.3%)\n",
"INFO:open_s3:Uploading domain-prediction-v2-3.tar.gz 15169091/15169091 bytes (100.0%)\n",
"INFO:open_s3:Removing old version (keep_last_n=1): domain-prediction-v2/1\n",
"INFO:models:Model domain-prediction-v2 uploaded with version 3\n"
]
},
{
"data": {
"text/plain": [
"3"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import logging\n",
"import sys\n",

5
good_ai/src/good_ai/__init__.py
View file

@ -1,2 +1,3 @@
from .deploy import process_batch
from .models import save_model, use_model
from .good_ai import *
from .utilities import *
from .open_s3 import *

2
good_ai/src/good_ai/good_ai/__init__.py Normal file
View file

@ -0,0 +1,2 @@
from .deploy import process_batch
from .models import save_model, use_model

2
good_ai/src/good_ai/good_ai/core/__init__.py Normal file
View file

@ -0,0 +1,2 @@
from .function_registry import function_registry
from .plugin import Plugin

23
good_ai/src/good_ai/good_ai/core/function_registry.py Normal file
View file

@ -0,0 +1,23 @@
from collections import defaultdict
from typing import Any, Callable, DefaultDict, List
from .plugin import Plugin
class FunctionRegistry:
def __init__(self) -> None:
self._registered_functions: DefaultDict[int, List[Plugin]] = defaultdict(
lambda: []
)
def add_plugin(self, function: Callable[..., Any], plugin: Plugin):
self._registered_functions[id(function)].append(plugin)
def get_plugins(self, function: Callable[..., Any]) -> List[Plugin]:
plugins = self._registered_functions[id(function)]
for p in plugins:
p.initialize()
return plugins
function_registry = FunctionRegistry()

19
good_ai/src/good_ai/good_ai/core/plugin.py Normal file
View file

@ -0,0 +1,19 @@
from typing import Any, Callable
class Plugin:
def __init__(self, function: Callable[[Any], Any]):
self._function = function
self._initialized = False
def initialize(self):
if not self._initialized:
self.on_initialize()
self._initialized = True
def on_initialize(self):
pass
def __call__(self, *args: Any, **kwargs: Any) -> Any:
assert self._initialized
return self._function(*args, **kwargs)

1
good_ai/src/good_ai/good_ai/deploy/__init__.py Normal file
View file

@ -0,0 +1 @@
from .process_batch import process_batch

16
good_ai/src/good_ai/good_ai/deploy/process_batch.py Normal file
View file

@ -0,0 +1,16 @@
from functools import partial, reduce
from typing import Any, Callable, Iterable, Optional, Sequence
from good_ai.utilities.parallel_map import parallel_map
from ..core import function_registry
def process_batch(
function: Callable[..., Any],
batch: Iterable[Any],
concurrency: Optional[int] = None,
) -> Sequence[Any]:
plugins = function_registry.get_plugins(function)
composed = partial(reduce, lambda r, f: f(r), plugins)
return parallel_map(composed, batch, concurrency=concurrency)

0
good_ai/src/__init__.py → good_ai/src/good_ai/good_ai/deploy/serve.py
View file

2
good_ai/src/good_ai/good_ai/models/__init__.py Normal file
View file

@ -0,0 +1,2 @@
from .save_model import save_model
from .use_model import use_model

19
good_ai/src/good_ai/good_ai/models/load_model.py Normal file
View file

@ -0,0 +1,19 @@
import logging
from typing import Any, Optional
from joblib import load
from good_ai.open_s3 import LargeFile
logger = logging.getLogger("models")
def load_model(
key: str, version: Optional[int] = None, return_path: bool = False
) -> Any:
file = LargeFile(name=key, mode="rb", version=version)
if return_path:
return file.get()
with file as f:
return load(f)

24
good_ai/src/good_ai/good_ai/models/save_model.py Normal file
View file

@ -0,0 +1,24 @@
import logging
from pathlib import Path
from typing import Optional, Union
from joblib import dump
from good_ai.open_s3 import LargeFile
logger = logging.getLogger("models")
def save_model(
model: Union[Path, str, object], key: str, keep_last_n: Optional[int] = None
) -> int:
file = LargeFile(name=key, mode="wb", keep_last_n=keep_last_n)
if isinstance(model, Path) or isinstance(model, str):
file.push(model)
else:
with file as f:
dump(model, f)
logger.info(f"Model {key} uploaded with version {file.version}")
return file.version

24
good_ai/src/good_ai/good_ai/models/use_model.py Normal file
View file

@ -0,0 +1,24 @@
from typing import Literal, Union
from ..core import function_registry
from .use_model_plugin import UseModelPlugin
def use_model(
key: str, version: Union[int, Literal["latest"]] = None, return_path: bool = False
):
assert isinstance(version, int) or version == "latest"
def inner(f):
function_registry.add_plugin(
f,
UseModelPlugin(
f,
key=key,
version=version if isinstance(version, int) else None,
return_path=return_path,
),
)
return f
return inner

28
good_ai/src/good_ai/good_ai/models/use_model_plugin.py Normal file
View file

@ -0,0 +1,28 @@
from typing import Any, Callable, Optional
from ..core import Plugin
from .load_model import load_model
class UseModelPlugin(Plugin):
def __init__(
self,
function: Callable[[Any], Any],
key: str,
version: Optional[int],
return_path: bool,
):
def wrapper(*args, **kwargs):
return function(*args, **kwargs, model=self._model)
super().__init__(wrapper)
self._key = key
self._version = version
self._return_path = return_path
def on_initialize(self) -> None:
super().on_initialize()
self._model = load_model(
key=self._key, version=self._version, return_path=self._return_path
)

1
good_ai/src/good_ai/open_s3/__init__.py Normal file
View file

@ -0,0 +1 @@
from .large_file import LargeFile

35
good_ai/src/good_ai/open_s3/__main__.py Normal file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env python3
import logging
from pathlib import Path
from large_file import LargeFile
from parse_arguments import parse_arguments
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
parser, args = parse_arguments()
LargeFile.configure_credentials_from_file(args.secrets)
if not args.cache and not args.push and not args.delete:
logging.warn("No action required.")
parser.print_help()
try:
if args.cache:
for c in args.cache:
split = c.split(":")
file_name = split[0]
version = None if len(split) == 1 else int(split[1])
LargeFile(file_name, "r", version=version).get()
if args.push:
for p in args.push:
path = Path(p)
LargeFile(path.name, "w").push(path)
if args.delete:
for f in args.delete:
LargeFile(f).delete()
except Exception as e:
logging.error(e)

Some files were not shown because too many files have changed in this diff Show more