Add auto-reload, evaluation endpoint, show docs

great_ai/src/great_ai/__main__.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import re
+from importlib import import_module
+from sys import argv
+
+import uvicorn
+from uvicorn.config import LOGGING_CONFIG
+
+from great_ai.great_ai.context import get_context
+
+if __name__ == "__main__":
+    if len(argv) < 2:
+        raise ValueError(f"Provide a filename such as: {argv[0]} my_app.py")
+
+    module_name = re.sub(r"\.py\b", "", argv[1])
+    if ":" not in module_name:
+        module_name += ":app"
+
+    base = module_name.split(":")[0]
+    module = import_module(base)
+    get_context().logger.info("Starting server")
+
+    uvicorn.run(
+        module_name,
+        host="0.0.0.0",
+        port=6060,
+        workers=1,
+        reload=not get_context().is_production,
+        log_config={
+            **LOGGING_CONFIG,
+            "formatters": {
+                "default": {
+                    "()": "great_ai.logger.CustomFormatter",
+                    "fmt": "%(asctime)s | %(levelname)8s | %(message)s",
+                },
+                "access": {
+                    "()": "great_ai.logger.CustomFormatter",
+                    "fmt": "%(asctime)s | %(levelname)8s | %(message)s",  # noqa: E501
+                },
+            },
+        },
+    )

great_ai/src/great_ai/great_ai/__init__.py

@@ -1,4 +1,4 @@
-from .context import set_default_config
-from .deploy import process_batch, process_single, serve
+from .context import configure
+from .deploy import create_service, process_batch, process_single
 from .metrics import log_argument, log_metric
 from .models import save_model, use_model

great_ai/src/great_ai/great_ai/context/__init__.py

@@ -1,2 +1,2 @@
 from .context import Context
-from .get_context import get_context, set_default_config
+from .get_context import configure, get_context

great_ai/src/great_ai/great_ai/context/get_context.py

@@ -16,26 +16,35 @@ PRODUCTION_KEY = "production"
 
 def get_context() -> Context:
     if _context is None:
-        set_default_config()
+        configure()
 
     return cast(Context, _context)
 
 
-def set_default_config(
+def configure(
     log_level: int = INFO,
     s3_config: Path = Path("s3.ini"),
     seed: int = 42,
     persistence_driver: PersistenceDriver = ParallelTinyDbDriver(
         Path("tracing_database.json")
     ),
-    is_production_mode_override: Optional[bool] = None,
+    development_mode_override: Optional[bool] = None,
 ) -> None:
     global _context
 
     logger = create_logger("great_ai", level=log_level)
 
+    if _context is not None:
+        logger.warn(
+            "Configuration has been already initialised, overwriting.\n"
+            + 'Make sure to call "configure()" before importing your application code.'
+        )
+
     is_production = _is_in_production_mode(
-        override=is_production_mode_override, logger=logger
+        override=None
+        if development_mode_override is None
+        else not development_mode_override,
+        logger=logger,
     )
     _initialize_large_file(s3_config, logger=logger)
     _set_seed(seed)
@@ -51,7 +60,7 @@ def set_default_config(
         logger=logger,
     )
 
-    logger.info("Defaults: configured ✅")
+    logger.info("Options: configured ✅")
 
 
 def _is_in_production_mode(override: Optional[bool], logger: Logger) -> bool:

great_ai/src/great_ai/great_ai/deploy/__init__.py

@@ -1,3 +1,3 @@
+from .create_service import create_service
 from .process_batch import process_batch
 from .process_single import process_single
-from .serve import serve

great_ai/src/great_ai/great_ai/deploy/create_service.py

@@ -0,0 +1,94 @@
+from pathlib import Path
+from typing import Any, Callable, Dict, List
+
+from fastapi import FastAPI, HTTPException, status
+from fastapi.middleware.wsgi import WSGIMiddleware
+from fastapi.openapi.docs import get_swagger_ui_html
+from fastapi.responses import RedirectResponse
+from fastapi.staticfiles import StaticFiles
+from starlette.responses import HTMLResponse
+
+from ..context import get_context
+from ..helper import snake_case_to_text
+from ..metrics import create_dash_app
+from ..tracing import TracingContext
+from ..views import EvaluationFeedbackRequest, HealthCheckResponse, Query, Trace
+
+PATH = Path(__file__).parent.resolve()
+
+
+def create_service(
+    func: Callable[..., Any], disable_docs: bool = False, disable_metrics: bool = False
+) -> FastAPI:
+    function_name = func.__name__
+    function_docs = func.__doc__
+
+    documentation = (
+        f"REST API wrapper for interacting with the '{function_name}' function.\n"
+    )
+    if function_docs:
+        documentation += function_docs
+
+    app = FastAPI(
+        title=snake_case_to_text(function_name),
+        description=documentation,
+        docs_url=None,
+        redoc_url=None,
+    )
+
+    @app.post("/evaluations", status_code=status.HTTP_200_OK, response_model=Trace)
+    def score(input: Any) -> Trace:
+        with TracingContext() as t:
+            result = func(input)
+            output = t.log_output(result)
+        return output
+
+    @app.get("/evaluations/:evaluation_id", status_code=status.HTTP_200_OK)
+    def get_evaluation(evaluation_id: str) -> Trace:
+        result = get_context().persistence.get_trace(evaluation_id)
+        if result is None:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+        return result
+
+    @app.post(
+        "/evaluations/:evaluation_id/feedback", status_code=status.HTTP_202_ACCEPTED
+    )
+    def give_feedback(evaluation_id: str, input: EvaluationFeedbackRequest) -> None:
+        get_context().persistence.add_evaluation(evaluation_id, input.evaluation)
+
+    if not disable_docs:
+
+        @app.get("/docs", include_in_schema=False)
+        def custom_swagger_ui_html() -> HTMLResponse:
+            return get_swagger_ui_html(openapi_url="openapi.json", title=app.title)
+
+        @app.get("/docs/index.html", include_in_schema=False)
+        def redirect_to_entrypoint() -> RedirectResponse:
+            return RedirectResponse("/docs")
+
+    if not disable_metrics:
+        dash_app = create_dash_app(function_name, documentation)
+        app.mount(get_context().metrics_path, WSGIMiddleware(dash_app))
+
+        @app.get("/", include_in_schema=False)
+        def redirect_to_entrypoint() -> RedirectResponse:
+            return RedirectResponse("/metrics")
+
+        app.mount(
+            "/assets", StaticFiles(directory=PATH / "../metrics/assets"), name="static"
+        )
+
+        @app.post("/query", status_code=status.HTTP_200_OK)
+        def query_metrics(query: Query) -> List[Dict[str, Any]]:
+            return get_context().persistence.query(
+                conjunctive_filters=query.filter,
+                sort_by=query.sort,
+                skip=query.skip,
+                take=query.take,
+            )
+
+    @app.get("/health", status_code=status.HTTP_200_OK)
+    def check_health() -> HealthCheckResponse:
+        return HealthCheckResponse(is_healthy=True)
+
+    return app

great_ai/src/great_ai/great_ai/deploy/serve.py

@@ -1,48 +0,0 @@
-from typing import Any, Callable
-
-import uvicorn
-from fastapi import FastAPI, status
-from uvicorn.config import LOGGING_CONFIG
-
-from ..tracing import TracingContext
-from ..views import Trace
-from .create_fastapi_app import create_fastapi_app
-
-
-def serve(
-    function: Callable[..., Any],
-    disable_docs: bool = False,
-    disable_metrics: bool = False,
-    configure: Callable[[FastAPI], None] = lambda _: None,
-) -> None:
-    app = create_fastapi_app(
-        function.__name__, disable_docs=disable_docs, disable_metrics=disable_metrics
-    )
-
-    @app.post("/score", status_code=status.HTTP_200_OK, response_model=Trace)
-    def process(input: Any) -> Trace:
-        with TracingContext() as t:
-            result = function(input)
-            output = t.log_output(result)
-        return output
-
-    configure(app)
-
-    uvicorn.run(
-        app,
-        host="0.0.0.0",
-        port=5050,
-        log_config={
-            **LOGGING_CONFIG,
-            "formatters": {
-                "default": {
-                    "()": "great_ai.logger.CustomFormatter",
-                    "fmt": "%(asctime)s | %(levelname)8s | %(message)s",
-                },
-                "access": {
-                    "()": "great_ai.logger.CustomFormatter",
-                    "fmt": "%(asctime)s | %(levelname)8s | %(message)s",  # noqa: E501
-                },
-            },
-        },
-    )

great_ai/src/great_ai/great_ai/exceptions/__init__.py

@@ -1 +1,2 @@
 from .argument_validation_error import ArgumentValidationError
+from .missing_argument_error import MissingArgumentError

great_ai/src/great_ai/great_ai/exceptions/missing_argument_error.py

@@ -0,0 +1,2 @@
+class MissingArgumentError(Exception):
+    pass

great_ai/src/great_ai/great_ai/helper/__init__.py

@@ -1,3 +1,4 @@
 from .get_args import get_args
 from .snake_case_to_text import snake_case_to_text
+from .strip_lines import strip_lines
 from .text_to_hex_color import text_to_hex_color

great_ai/src/great_ai/great_ai/helper/strip_lines.py

@@ -0,0 +1,2 @@
+def strip_lines(text: str) -> str:
+    return "\n".join(line.strip() for line in text.split("\n"))

great_ai/src/great_ai/great_ai/metrics/create_dash_app.py

@@ -17,7 +17,7 @@ from .get_filter_from_datatable import get_filter_from_datatable
 from .get_footer import get_footer
 
 
-def create_dash_app(function_name: str) -> Flask:
+def create_dash_app(function_name: str, function_docs: str) -> Flask:
     accent_color = text_to_hex_color(function_name)
 
     flask_app = Flask(__name__)
@@ -55,7 +55,9 @@ def create_dash_app(function_name: str) -> Flask:
             html.Div(
                 [
                     get_description(
-                        function_name=function_name, accent_color=accent_color
+                        function_name=function_name,
+                        function_docs=function_docs,
+                        accent_color=accent_color,
                     ),
                     execution_time_histogram,
                 ],

great_ai/src/great_ai/great_ai/metrics/get_description.py

@@ -1,27 +1,36 @@
 from dash import dcc, html
 
-from ..helper import snake_case_to_text
+from ..helper import snake_case_to_text, strip_lines
 
 
-def get_description(function_name: str, accent_color: str) -> html.Div:
-    markdown_text = f"""
-    View the live data of your deployments here.
-
-    ## Using the API
-
-    You can find the available endpoints at [/docs](/docs).
-
-    ## Metrics
-
-    Recent traces and aggregated metrics are presented below. Try filtering the table.
-    """
-
+def get_description(
+    function_name: str, function_docs: str, accent_color: str
+) -> html.Div:
     return html.Div(
         [
             html.H1(
                 f"{snake_case_to_text(function_name)} - metrics",
                 style={"color": accent_color},
             ),
-            dcc.Markdown(markdown_text, className="description"),
+            dcc.Markdown(
+                strip_lines(
+                    f"""
+                > View the live data of your deployments here.
+
+                ## Using the API
+
+                You can find the available endpoints at [/docs](/docs).
+
+                ### Details
+
+                {function_docs}
+
+                ## Metrics
+
+                Recent traces and aggregated metrics are presented below. Try filtering the table.
+                """
+                ),
+                className="description",
+            ),
         ]
     )

great_ai/src/great_ai/great_ai/persistence/mongodb_driver.py

@@ -1,7 +1,5 @@
-from math import remainder
 from .persistence_driver import PersistenceDriver
 
 
 class MongoDbDriver(PersistenceDriver):
     pass
-

great_ai/src/great_ai/great_ai/persistence/parallel_tinydb_driver.py

@@ -22,9 +22,25 @@ class ParallelTinyDbDriver(PersistenceDriver):
         super().__init__()
         self._path_to_db = path_to_db
 
-    def save_document(self, trace: Trace) -> str:
+    def save_trace(self, trace: Trace) -> str:
         return self._safe_execute(lambda db: db.insert(trace.dict()))
 
+    def add_evaluation(self, id: str, evaluation: Any) -> None:
+        self._safe_execute(
+            lambda db: db.update(
+                fields={"evaluation": evaluation},
+                cond=lambda d: d["evaluation_id"] == id,
+            )
+        )
+
+    def get_trace(self, id: str) -> Optional[Trace]:
+        value = self._safe_execute(
+            lambda db: db.get(lambda d: d["evaluation_id"] == id)
+        )
+        if value:
+            value = Trace.parse_obj(value)
+        return value
+
     def get_traces(self) -> List[Trace]:
         return self._safe_execute(lambda db: [Trace.parse_obj(t) for t in db.all()])
 

great_ai/src/great_ai/great_ai/persistence/persistence_driver.py

@@ -10,7 +10,15 @@ class PersistenceDriver(ABC):
     is_threadsafe: bool
 
     @abstractmethod
-    def save_document(self, document: Trace) -> str:
+    def save_trace(self, document: Trace) -> str:
+        pass
+
+    @abstractmethod
+    def add_evaluation(self, id: str, evaluation: Any) -> None:
+        pass
+
+    @abstractmethod
+    def get_trace(self, id: str) -> Optional[Trace]:
         pass
 
     @abstractmethod

great_ai/src/great_ai/great_ai/tracing/tracing_context.py

@@ -59,7 +59,7 @@ class TracingContext:
 
         if type is None:
             assert self._trace is not None
-            get_context().persistence.save_document(self._trace)
+            get_context().persistence.save_trace(self._trace)
         else:
             get_context().logger.exception(f"Could not finish operation: {exception}")
 

great_ai/src/great_ai/great_ai/views/__init__.py

@@ -1,3 +1,4 @@
+from .evaluation_feedback_request import EvaluationFeedbackRequest
 from .filter import Filter
 from .health_check_response import HealthCheckResponse
 from .model import Model

great_ai/src/great_ai/great_ai/views/evaluation_feedback_request.py

@@ -0,0 +1,7 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class EvaluationFeedbackRequest(BaseModel):
+    evaluation: Any

great_ai/src/great_ai/great_ai/views/trace.py

@@ -29,6 +29,7 @@ class Trace(BaseModel):
             "created": self.created,
             "execution_time_ms": self.execution_time_ms,
             "models": ", ".join(f"{m.key}:{m.version}" for m in self.models),
+            "output": self.output,
             "evaluation": self.evaluation,
             **self.logged_values,
         }