Replace trafaret by pydantic for services config (iteration 1/2) (#1890)

- storage service updated
   - EXTRA: adds .env-devel and Makefile recipe for local development
   - EXTRA: adds swagger UI at /dev/doc
- webserver keeps for the moment both
More details in PR#1890

Author: pcrespov

packages/models-library/src/models_library/basic_types.py

@@ -0,0 +1,42 @@
+from enum import Enum
+
+from pydantic import conint, constr, confloat
+
+PortInt = conint(gt=0, lt=65535)
+
+VersionTag = constr(regex=r"^v\d$")
+
+NonNegativeFloat = confloat(ge=0)  # NOTE: = 0.0 + PositiveFloat
+
+
+class LogLevel(str, Enum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+
+
+class BootModeEnum(str, Enum):
+    """
+    Values taken by SC_BOOT_MODE environment variable
+    set in Dockerfile and used during docker/boot.sh
+    """
+
+    DEFAULT = "default"
+    LOCAL = "local-development"
+    DEBUG = "debug-ptvsd"
+    PRODUCTION = "production"
+    DEVELOPMENT = "development"
+
+
+class BuildTargetEnum(str, Enum):
+    """
+    Values taken by SC_BUILD_TARGET environment variable
+    set in Dockerfile that defines the stage targeted in the
+    docker image build
+    """
+
+    BUILD = "build"
+    CACHE = "cache"
+    PRODUCTION = "production"
+    DEVELOPMENT = "development"

packages/models-library/src/models_library/settings/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Common settings
+"""

packages/models-library/src/models_library/settings/application_bases.py

@@ -0,0 +1,53 @@
+"""
+     Bases to build ApplicationSettings
+
+     TODO: when py 3.7 the use https://pydantic-docs.helpmanual.io/usage/models/#generic-models
+"""
+import logging
+from typing import Optional
+from pydantic import Field, BaseSettings, validator, AnyHttpUrl
+from ..basic_types import PortInt, BootModeEnum, LogLevel, VersionTag
+
+
+class BaseFastApiAppSettings(BaseSettings):
+    # DOCKER
+    boot_mode: Optional[BootModeEnum] = Field(..., env="SC_BOOT_MODE")
+
+    # LOGGING
+    log_level: LogLevel = Field("DEBUG", env=["LOG_LEVEL", "LOGLEVEL"])
+
+    @property
+    def logging_level(self) -> int:
+        return getattr(logging, self.log_level)
+
+    # SERVER (see : https://www.uvicorn.org/settings/)
+    host: str = "0.0.0.0"  # nosec
+    port: PortInt = 8000
+    debug: bool = False  # If True, debug tracebacks should be returned on errors.
+
+    # DEBUGGING
+    remote_debug_port: PortInt = 3000
+
+
+class BaseAiohttpAppSettings(BaseFastApiAppSettings):
+    port: PortInt = 8080
+
+
+class BaseServiceAPISettings(BaseSettings):
+    host: str
+    port: PortInt
+    vtag: VersionTag = Field("v0", description="Service API's version tag")
+
+    url: Optional[AnyHttpUrl] = None
+
+    @validator("url", pre=True)
+    @classmethod
+    def autofill_url(cls, v, values):
+        if v is None:
+            return AnyHttpUrl.build(
+                scheme="http",
+                host=values["host"],
+                port=f"{values['port']}",
+                path=f"/{values['vtag']}",
+            )
+        return v

packages/models-library/src/models_library/celery.py renamed to packages/models-library/src/models_library/settings/celery.py

@@ -5,13 +5,16 @@
 
 class CeleryConfig(BaseSettings):
     @classmethod
-    def create_from_env(cls) -> "CeleryConfig":
+    def create_from_env(cls, **override) -> "CeleryConfig":
         # this calls trigger env parsers
-        return cls(rabbit=RabbitConfig(), redis=RedisConfig())
+        return cls(rabbit=RabbitConfig(), redis=RedisConfig(), **override)
 
+    # NOTE: Defaults are evaluated at import time, use create_from_env to build instances
     rabbit: RabbitConfig = RabbitConfig()
     redis: RedisConfig = RedisConfig()
+
     task_name: str = "simcore.comp.task"
+
     publication_timeout: PositiveInt = 60
 
     @property

packages/models-library/src/models_library/settings/postgres.py

@@ -0,0 +1,51 @@
+from typing import Optional
+
+from pydantic import BaseSettings, Field, PostgresDsn, SecretStr, conint, validator
+
+from ..basic_types import PortInt
+
+
+class PostgresSettings(BaseSettings):
+    dns: Optional[PostgresDsn] = None
+
+    # entrypoint
+    host: str
+    port: PortInt = 5432
+
+    # auth
+    user: str
+    password: SecretStr
+
+    # database
+    db: str
+
+    # pool connection limits
+    minsize: conint(ge=1) = 10
+    maxsize: conint(ge=1) = 10
+
+    dsn: Optional[PostgresDsn] = Field(None, description="Database Source Name")
+
+    @validator("maxsize")
+    @classmethod
+    def check_size(cls, v, values):
+        if not (values["minsize"] <= v):
+            raise ValueError(f"assert minsize={values['minsize']} <= maxsize={v}")
+        return v
+
+    @validator("dsn", pre=True)
+    @classmethod
+    def autofill_dsn(cls, v, values):
+        if v is None:
+            return PostgresDsn.build(
+                scheme="postgresql",
+                user=values["user"],
+                password=values["password"].get_secret_value(),
+                host=values["host"],
+                port=f"{values['port']}",
+                path=f"/{values['db']}",
+            )
+        return v
+
+    class Config:
+        case_sensitive = False
+        env_prefix = "POSTGRES_"

packages/models-library/src/models_library/rabbit.py renamed to packages/models-library/src/models_library/settings/rabbit.py

@@ -0,0 +1,13 @@
+from pydantic import BaseSettings
+
+
+class S3Config(BaseSettings):
+    endpoint: str = "minio:9000"
+    access_key: str = "12345678"
+    secret_key: str = "12345678"
+    bucket_name: str = "simcore"
+    secure: bool = False
+
+    class Config:
+        case_sensitive = False
+        env_prefix = "S3_"

packages/models-library/src/models_library/redis.py renamed to packages/models-library/src/models_library/settings/redis.py

@@ -12,7 +12,7 @@
 import pytest
 import tenacity
 
-from models_library.rabbit import RabbitConfig
+from models_library.settings.rabbit import RabbitConfig
 
 from .helpers.utils_docker import get_service_published_port
 

packages/models-library/src/models_library/settings/s3.py

@@ -8,7 +8,7 @@
 import aioredis
 import pytest
 import tenacity
-from models_library.redis import RedisConfig
+from models_library.settings.redis import RedisConfig
 from servicelib.redis_utils import RedisRetryPolicyUponInitialization
 from yarl import URL
 

packages/pytest-simcore/src/pytest_simcore/rabbit_service.py

@@ -19,3 +19,4 @@ tenacity
 attrs
 trafaret
 aiodebug
+pydantic

packages/pytest-simcore/src/pytest_simcore/redis_service.py

@@ -11,6 +11,7 @@ aiozipkin==0.7.1          # via -r requirements/_base.in
 async-timeout==3.0.1      # via aiohttp
 attrs==20.2.0             # via -r requirements/_base.in, aiohttp, jsonschema, openapi-core
 chardet==3.0.4            # via aiohttp
+dataclasses==0.7          # via pydantic
 idna-ssl==1.1.0           # via aiohttp
 idna==2.10                # via idna-ssl, yarl
 importlib-metadata==2.0.0  # via jsonschema
@@ -22,6 +23,7 @@ openapi-core==0.12.0      # via -r requirements/_base.in
 openapi-spec-validator==0.2.9  # via openapi-core
 prometheus-client==0.8.0  # via -r requirements/_base.in
 psycopg2-binary==2.8.6    # via -r requirements/_base.in, aiopg, sqlalchemy
+pydantic==1.6.1           # via -r requirements/_base.in
 pyrsistent==0.17.3        # via jsonschema
 pyyaml==5.3.1             # via -r requirements/_base.in, openapi-spec-validator
 six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-spec-validator, tenacity

packages/service-library/requirements/_base.in

@@ -43,6 +43,7 @@ prometheus-client==0.8.0  # via -r requirements/_base.txt
 psycopg2-binary==2.8.6    # via -r requirements/_base.txt, aiopg, sqlalchemy
 py==1.9.0                 # via pytest
 pycparser==2.20           # via cffi
+pydantic==1.6.1           # via -r requirements/_base.txt
 pylint==2.6.0             # via -r requirements/_test.in
 pynacl==1.4.0             # via paramiko
 pyparsing==2.4.7          # via packaging

packages/service-library/requirements/_base.txt

@@ -1,17 +1,14 @@
-"""
-
-    Adds aiohttp middleware for tracing using zipkin server instrumentation.
-
+""" Adds aiohttp middleware for tracing using zipkin server instrumentation.
 
 """
-
 import asyncio
 import logging
-from typing import Dict
+from typing import Dict, Optional
 
 import aiozipkin as az
 import trafaret as T
 from aiohttp import web
+from pydantic import AnyHttpUrl, BaseSettings
 
 log = logging.getLogger(__name__)
 
@@ -35,3 +32,8 @@ def setup_tracing(
         T.Key("zipkin_endpoint", default="http://jaeger:9411"): T.String(),
     }
 )
+
+
+class TracingSettings(BaseSettings):
+    enabled: Optional[bool] = True
+    zipkin_endpoint: AnyHttpUrl = "http://jaeger:9411"

packages/service-library/requirements/_test.txt

@@ -1,27 +1,35 @@
 """ Basic configuration file for S3
 
 """
-from os import environ as env
 import logging
+import warnings
+from os import environ as env
 
 import trafaret as T
 
+warnings.warn(
+    f"Module '{__name__}' will be remove in future versions. "
+    "All common settings have been moved to packages/model-slibrary. "
+    "Use instead 'models_library.settings.s3' ",
+    DeprecationWarning,
+)
+
 log = logging.getLogger(__name__)
 
 
-CONFIG_SCHEMA = T.Dict({
-    "endpoint": T.String(),
-    "access_key": T.String(),
-    "secret_key": T.String(),
-    "bucket_name": T.String(),
-    T.Key("secure", default=0): T.ToInt(),
-})
+CONFIG_SCHEMA = T.Dict(
+    {
+        "endpoint": T.String(),
+        "access_key": T.String(),
+        "secret_key": T.String(),
+        "bucket_name": T.String(),
+        T.Key("secure", default=0): T.ToInt(),
+    }
+)
 
 
-# TODO: deprecate!
-class Config():
+class Config:
     def __init__(self):
-        # TODO: uniform config classes . see server.config file
         S3_ENDPOINT = env.get("S3_ENDPOINT", "minio:9000")
         S3_ACCESS_KEY = env.get("S3_ACCESS_KEY", "12345678")
         S3_SECRET_KEY = env.get("S3_SECRET_KEY", "12345678")

packages/service-library/src/servicelib/tracing.py

@@ -863,13 +863,7 @@
         "required": [
           "key",
           "version",
-          "label",
-          "progress",
-          "inputs",
-          "inputAccess",
-          "inputNodes",
-          "outputNodes",
-          "position"
+          "label"
         ],
         "type": "object",
         "properties": {
@@ -976,7 +970,16 @@
             ]
           },
           "position": {
-            "$ref": "#/components/schemas/Position"
+            "title": "Position",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Position"
+              }
+            ],
+            "deprecated": true
+          },
+          "state": {
+            "$ref": "#/components/schemas/RunningState"
           }
         },
         "additionalProperties": false
@@ -1037,6 +1040,22 @@
         },
         "additionalProperties": false
       },
+      "RunningState": {
+        "title": "RunningState",
+        "enum": [
+          "UNKNOWN",
+          "PUBLISHED",
+          "NOT_STARTED",
+          "PENDING",
+          "STARTED",
+          "RETRY",
+          "SUCCESS",
+          "FAILURE",
+          "ABORTED"
+        ],
+        "type": "string",
+        "description": "An enumeration."
+      },
       "SelectBox": {
         "title": "SelectBox",
         "required": [
@@ -1375,6 +1394,7 @@
       "ServiceType": {
         "title": "ServiceType",
         "enum": [
+          "frontend",
           "computational",
           "dynamic"
         ],

packages/simcore-sdk/src/simcore_sdk/config/s3.py

@@ -74,11 +74,10 @@ async def request_wrapper(zelf: "DirectorApi", path: str, *args, **kwargs):
             normalized_path = path.lstrip("/")
             resp = await request_func(zelf, path=normalized_path, *args, **kwargs)
         except Exception as err:
-            #pylint: disable=protected-access
             logger.exception(
                 "Failed request %s to %s%s",
                 request_func.__name__,
-                zelf._client.base_url,
+                zelf.client.base_url,
                 normalized_path,
             )
             raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE) from err

services/catalog/openapi.json

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import Optional
 
-from models_library.celery import CeleryConfig
+from models_library.settings.celery import CeleryConfig
 
 SERVICES_MAX_NANO_CPUS: int = min(
     multiprocessing.cpu_count() * pow(10, 9),