♻️ Dask: maintenance, refactor, upgrade (#4051)

Author: sanderegg

packages/models-library/src/models_library/services.py

@@ -7,7 +7,7 @@
 import re
 from datetime import datetime
 from enum import Enum
-from typing import Any, Optional, Union
+from typing import Any
 from uuid import UUID
 
 from pydantic import (
@@ -60,10 +60,16 @@
 class ServiceKey(ConstrainedStr):
     regex = re.compile(SERVICE_KEY_RE)
 
+    class Config:
+        frozen = True
+
 
 class ServiceVersion(ConstrainedStr):
     regex = re.compile(VERSION_RE)
 
+    class Config:
+        frozen = True
+
 
 RunID = UUID
 
@@ -126,7 +132,7 @@ class Author(BaseModel):
         examples=["[email protected]", "[email protected]"],
         description="Email address",
     )
-    affiliation: Optional[str] = Field(
+    affiliation: str | None = Field(
         None, examples=["Sense8", "Babylon 5"], description="Affiliation of the author"
     )
 
@@ -142,7 +148,7 @@ class BaseServiceIOModel(BaseModel):
     ## management
 
     ### human readable descriptors
-    display_order: Optional[float] = Field(
+    display_order: float | None = Field(
         None,
         alias="displayOrder",
         deprecated=True,
@@ -177,22 +183,22 @@ class BaseServiceIOModel(BaseModel):
         regex=PROPERTY_TYPE_RE,
     )
 
-    content_schema: Optional[dict[str, Any]] = Field(
+    content_schema: dict[str, Any] | None = Field(
         None,
         description="jsonschema of this input/output. Required when type='ref_contentSchema'",
         alias="contentSchema",
     )
 
     # value
-    file_to_key_map: Optional[dict[FileName, ServicePortKey]] = Field(
+    file_to_key_map: dict[FileName, ServicePortKey] | None = Field(
         None,
         alias="fileToKeyMap",
         description="Place the data associated with the named keys in files",
         examples=[{"dir/input1.txt": "key_1", "dir33/input2.txt": "key2"}],
     )
 
     # TODO: should deprecate since content_schema include units
-    unit: Optional[str] = Field(
+    unit: str | None = Field(
         None,
         description="Units, when it refers to a physical quantity",
     )
@@ -249,11 +255,11 @@ class ServiceInput(BaseServiceIOModel):
     """
 
     # TODO: should deprecate since content_schema include defaults as well
-    default_value: Optional[Union[StrictBool, StrictInt, StrictFloat, str]] = Field(
+    default_value: StrictBool | StrictInt | StrictFloat | str | None = Field(
         None, alias="defaultValue", examples=["Dog", True]
     )
 
-    widget: Optional[Widget] = Field(
+    widget: Widget | None = Field(
         None,
         description="custom widget to use instead of the default one determined from the data-type",
     )
@@ -323,7 +329,7 @@ def from_json_schema(cls, port_schema: dict[str, Any]) -> "ServiceInput":
 
 
 class ServiceOutput(BaseServiceIOModel):
-    widget: Optional[Widget] = Field(
+    widget: Widget | None = Field(
         None,
         description="custom widget to use instead of the default one determined from the data-type",
         deprecated=True,
@@ -370,30 +376,26 @@ def from_json_schema(cls, port_schema: dict[str, Any]) -> "ServiceOutput":
 class ServiceKeyVersion(BaseModel):
     """This pair uniquely identifies a services"""
 
-    key: str = Field(
+    key: ServiceKey = Field(
         ...,
         description="distinctive name for the node based on the docker registry path",
-        regex=KEY_RE,
-        examples=[
-            "simcore/services/comp/itis/sleeper",
-            "simcore/services/dynamic/3dviewer",
-        ],
     )
-    version: str = Field(
+    version: ServiceVersion = Field(
         ...,
         description="service version number",
-        regex=VERSION_RE,
-        examples=["1.0.0", "0.0.1"],
     )
 
+    class Config:
+        frozen = True
+
 
 class _BaseServiceCommonDataModel(BaseModel):
     name: str = Field(
         ...,
         description="short, human readable name for the node",
         example="Fast Counter",
     )
-    thumbnail: Optional[HttpUrl] = Field(
+    thumbnail: HttpUrl | None = Field(
         None,
         description="url to the thumbnail",
         examples=[
@@ -428,7 +430,7 @@ class ServiceDockerData(ServiceKeyVersion, _BaseServiceCommonDataModel):
     This is one to one with node-meta-v0.0.1.json
     """
 
-    integration_version: Optional[str] = Field(
+    integration_version: str | None = Field(
         None,
         alias="integration-version",
         description="integration version number",
@@ -442,22 +444,22 @@ class ServiceDockerData(ServiceKeyVersion, _BaseServiceCommonDataModel):
         examples=["computational"],
     )
 
-    badges: Optional[list[Badge]] = Field(None)
+    badges: list[Badge] | None = Field(None)
 
     authors: list[Author] = Field(..., min_items=1)
     contact: LowerCaseEmailStr = Field(
         ...,
         description="email to correspond to the authors about the node",
         examples=["[email protected]"],
     )
-    inputs: Optional[ServiceInputsDict] = Field(
+    inputs: ServiceInputsDict | None = Field(
         ..., description="definition of the inputs of this node"
     )
-    outputs: Optional[ServiceOutputsDict] = Field(
+    outputs: ServiceOutputsDict | None = Field(
         ..., description="definition of the outputs of this node"
     )
 
-    boot_options: Optional[BootOptions] = Field(
+    boot_options: BootOptions | None = Field(
         None,
         alias="boot-options",
         description="Service defined boot options. These get injected in the service as env variables.",
@@ -466,6 +468,7 @@ class ServiceDockerData(ServiceKeyVersion, _BaseServiceCommonDataModel):
     class Config:
         description = "Description of a simcore node 'class' with input and output"
         extra = Extra.forbid
+        frozen = False  # it inherits from ServiceKeyVersion.
 
         schema_extra = {
             "examples": [
@@ -567,16 +570,16 @@ class ServiceMetaData(_BaseServiceCommonDataModel):
     #        it should be implemented with a different model e.g. ServiceMetaDataUpdate
     #
 
-    name: Optional[str]
-    thumbnail: Optional[HttpUrl]
-    description: Optional[str]
-    deprecated: Optional[datetime] = Field(
+    name: str | None
+    thumbnail: HttpUrl | None
+    description: str | None
+    deprecated: datetime | None = Field(
         default=None,
         description="If filled with a date, then the service is to be deprecated at that date (e.g. cannot start anymore)",
     )
 
     # user-defined metatada
-    classifiers: Optional[list[str]]
+    classifiers: list[str] | None
     quality: dict[str, Any] = {}
 
     class Config:

services/api-server/VERSION

@@ -1 +1 @@
-0.4.3
+0.4.4

services/api-server/openapi.json

@@ -3,7 +3,7 @@
   "info": {
     "title": "osparc.io web API",
     "description": "osparc-simcore public web API specifications",
-    "version": "0.4.2",
+    "version": "0.4.4",
     "x-logo": {
       "url": "https://raw.githubusercontent.com/ITISFoundation/osparc-manual/b809d93619512eb60c827b7e769c6145758378d0/_media/osparc-logo.svg",
       "altText": "osparc-simcore logo"
@@ -548,6 +548,7 @@
             "required": true,
             "schema": {
               "title": "Version",
+              "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$",
               "type": "string"
             },
             "name": "version",
@@ -608,6 +609,7 @@
             "required": true,
             "schema": {
               "title": "Version",
+              "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$",
               "type": "string"
             },
             "name": "version",

services/api-server/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.3
+current_version = 0.4.4
 commit = True
 message = services/api-server version: {current_version} → {new_version}
 tag = False
@@ -10,7 +10,7 @@ commit_args = --no-verify
 [tool:pytest]
 asyncio_mode = auto
 addopts = --strict-markers
-markers =
+markers = 
 	slow: marks tests as slow (deselect with '-m "not slow"')
 	acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows."
 	testit: "marks test to run during development"

services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py

@@ -3,7 +3,7 @@
 
 import logging
 from collections import deque
-from typing import Callable, Optional, Union
+from typing import Callable
 from uuid import UUID
 
 from fastapi import APIRouter, Depends, status
@@ -62,7 +62,7 @@ def _compose_job_resource_name(solver_key, solver_version, job_id) -> str:
 )
 async def list_jobs(
     solver_key: SolverKeyId,
-    version: str,
+    version: VersionStr,
     user_id: PositiveInt = Depends(get_current_user_id),
     catalog_client: CatalogApi = Depends(get_api_client(CatalogApi)),
     webserver_api: AuthSession = Depends(get_webserver_session),
@@ -97,7 +97,7 @@ async def list_jobs(
 )
 async def create_job(
     solver_key: SolverKeyId,
-    version: str,
+    version: VersionStr,
     inputs: JobInputs,
     user_id: PositiveInt = Depends(get_current_user_id),
     catalog_client: CatalogApi = Depends(get_api_client(CatalogApi)),
@@ -186,7 +186,7 @@ async def start_job(
     solver_key: SolverKeyId,
     version: VersionStr,
     job_id: UUID,
-    cluster_id: Optional[ClusterID] = None,
+    cluster_id: ClusterID | None = None,
     user_id: PositiveInt = Depends(get_current_user_id),
     director2_api: DirectorV2Api = Depends(get_api_client(DirectorV2Api)),
     product_name: str = Depends(get_product_name),
@@ -269,7 +269,7 @@ async def get_job_outputs(
     assert len(node_ids) == 1  # nosec
 
     outputs: dict[
-        str, Union[float, int, bool, BaseFileLink, str, None]
+        str, float | int | bool | BaseFileLink | str | None
     ] = await get_solver_output_results(
         user_id=user_id,
         project_uuid=job_id,

services/api-server/src/simcore_service_api_server/models/basic_types.py

@@ -1,4 +1,9 @@
+import re
+
 from models_library.basic_regex import VERSION_RE
-from pydantic import constr
+from pydantic import ConstrainedStr
+
 
-VersionStr = constr(strip_whitespace=True, regex=VERSION_RE)  # as M.m.p
+class VersionStr(ConstrainedStr):
+    strip_whitespace = True
+    regex = re.compile(VERSION_RE)

services/api-server/src/simcore_service_api_server/models/schemas/solvers.py

@@ -1,11 +1,12 @@
+import re
 import urllib.parse
-from typing import Any, Literal, Optional, Union
+from typing import Any, Literal
 
 import packaging.version
 from models_library.basic_regex import PUBLIC_VARIABLE_NAME_RE
 from models_library.services import COMPUTATIONAL_SERVICE_KEY_RE, ServiceDockerData
 from packaging.version import LegacyVersion, Version
-from pydantic import BaseModel, Extra, Field, HttpUrl, constr
+from pydantic import BaseModel, ConstrainedStr, Extra, Field, HttpUrl
 
 from ..api_resources import compose_resource_name
 from ..basic_types import VersionStr
@@ -28,11 +29,10 @@
 #
 SOLVER_RESOURCE_NAME_RE = r"^solvers/([^\s/]+)/releases/([\d\.]+)$"
 
-SolverKeyId = constr(
-    strip_whitespace=True,
-    regex=COMPUTATIONAL_SERVICE_KEY_RE,
-    # TODO: should we use here a less restrictive regex that does not impose simcore/comp/?? this should be catalog responsibility
-)
+
+class SolverKeyId(ConstrainedStr):
+    strip_whitespace = True
+    regex = re.compile(COMPUTATIONAL_SERVICE_KEY_RE)
 
 
 class Solver(BaseModel):
@@ -49,13 +49,13 @@ class Solver(BaseModel):
 
     # Human readables Identifiers
     title: str = Field(..., description="Human readable name")
-    description: Optional[str]
+    description: str | None
     maintainer: str
     # TODO: consider released: Optional[datetime]   required?
     # TODO: consider version_aliases: list[str] = []  # remaining tags
 
     # Get links to other resources
-    url: Optional[HttpUrl] = Field(..., description="Link to get this resource")
+    url: HttpUrl | None = Field(..., description="Link to get this resource")
 
     class Config:
         extra = Extra.ignore
@@ -86,7 +86,7 @@ def create_from_image(cls, image_meta: ServiceDockerData) -> "Solver":
         )
 
     @property
-    def pep404_version(self) -> Union[Version, LegacyVersion]:
+    def pep404_version(self) -> Version | LegacyVersion:
         """Rich version type that can be used e.g. to compare"""
         return packaging.version.parse(self.version)
 
@@ -118,7 +118,7 @@ class SolverPort(BaseModel):
         title="Key name",
     )
     kind: PortKindStr
-    content_schema: Optional[dict[str, Any]] = Field(
+    content_schema: dict[str, Any] | None = Field(
         None,
         description="jsonschema for the port's value. SEE https://json-schema.org",
     )

services/catalog/tests/unit/with_dbs/test_services_access_rights.py

@@ -5,8 +5,9 @@
 from typing import Callable
 
 from fastapi import FastAPI
-from models_library.services import ServiceDockerData
+from models_library.services import ServiceDockerData, ServiceVersion
 from models_library.services_db import ServiceAccessRightsAtDB
+from pydantic import parse_obj_as
 from simcore_service_catalog.db.repositories.services import ServicesRepository
 from simcore_service_catalog.models.domain.group import GroupAtDB
 from simcore_service_catalog.services.access_rights import (
@@ -112,7 +113,7 @@ async def test_auto_upgrade_policy(
     new_service_metadata = ServiceDockerData.parse_obj(
         ServiceDockerData.Config.schema_extra["examples"][MOST_UPDATED_EXAMPLE]
     )
-    new_service_metadata.version = "1.0.11"
+    new_service_metadata.version = parse_obj_as(ServiceVersion, "1.0.11")
 
     # we have three versions of the service in the database for which the sorting matters: (1.0.11 should inherit from 1.0.10 not 1.0.9)
     await services_db_tables_injector(