Merge branch 'master' into feature/previews-and-scene

Author: odeimaiz

packages/models-library/src/models_library/services_resources.py

@@ -1,6 +1,6 @@
 import logging
 from enum import auto
-from typing import Any, Final, Optional, Union
+from typing import Any, Final, TypeAlias
 
 from models_library.docker import DockerGenericTag
 from models_library.utils.enums import StrAutoEnum
@@ -37,8 +37,8 @@
 
 
 class ResourceValue(BaseModel):
-    limit: Union[StrictInt, StrictFloat, str]
-    reservation: Union[StrictInt, StrictFloat, str]
+    limit: StrictInt | StrictFloat | str
+    reservation: StrictInt | StrictFloat | str
 
     @root_validator()
     @classmethod
@@ -100,15 +100,15 @@ class Config:
         }
 
 
-ServiceResourcesDict = dict[DockerGenericTag, ImageResources]
+ServiceResourcesDict: TypeAlias = dict[DockerGenericTag, ImageResources]
 
 
 class ServiceResourcesDictHelpers:
     @staticmethod
     def create_from_single_service(
         image: DockerGenericTag,
         resources: ResourcesDict,
-        boot_modes: Optional[list[BootMode]] = None,
+        boot_modes: list[BootMode] | None = None,
     ) -> ServiceResourcesDict:
         if boot_modes is None:
             boot_modes = [BootMode.CPU]

packages/postgres-database/setup.cfg

@@ -9,5 +9,6 @@ commit_args = --no-verify
 
 [tool:pytest]
 asyncio_mode = auto
-markers = 
+addopts = -W error::sqlalchemy.exc.SAWarning
+markers =
 	testit: "marks test to run during development"

packages/postgres-database/src/simcore_postgres_database/migration/versions/add0afaaf728_migrate_projects_workbench_step1.py

@@ -0,0 +1,58 @@
+"""migrate projects workbench step1
+
+Revision ID: add0afaaf728
+Revises: 6e91067932f2
+Create Date: 2023-06-22 14:45:38.827559+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "add0afaaf728"
+down_revision = "6e91067932f2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    projects_table = sa.table(
+        "projects",
+        sa.column("uuid"),
+        sa.column("workbench"),
+        sa.column("creation_date"),
+        sa.column("last_change_date"),
+    )
+    projects_nodes_table = sa.table(
+        "projects_nodes",
+        sa.column("project_uuid"),
+        sa.column("node_id"),
+        sa.column("created"),
+        sa.column("modified"),
+    )
+
+    connection = op.get_bind()
+
+    for project_uuid, workbench, creation_date, last_change_date in connection.execute(
+        projects_table.select()
+    ):
+        for node_id in workbench.keys():
+            connection.execute(
+                projects_nodes_table.insert().values(
+                    project_uuid=project_uuid,
+                    node_id=node_id,
+                    created=creation_date,
+                    modified=last_change_date,
+                )
+            )
+
+
+def downgrade():
+    projects_nodes_table = sa.table(
+        "projects_nodes",
+        sa.column("project_uuid"),
+        sa.column("node_id"),
+    )
+    connection = op.get_bind()
+
+    connection.execute(projects_nodes_table.delete())

packages/postgres-database/src/simcore_postgres_database/utils_models.py

@@ -0,0 +1,13 @@
+from typing import TypeVar
+
+from aiopg.sa.result import RowProxy
+
+ModelType = TypeVar("ModelType")
+
+
+class FromRowMixin:
+    """Mixin to allow instance construction from aiopg.sa.result.RowProxy"""
+
+    @classmethod
+    def from_row(cls: type[ModelType], row: RowProxy) -> ModelType:
+        return cls(**dict(row.items()))

packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py

@@ -1,12 +1,14 @@
 import datetime
 import uuid
-from dataclasses import asdict, dataclass, field
+from dataclasses import asdict, dataclass, field, fields
+from typing import Any
 
 import sqlalchemy
 from aiopg.sa.connection import SAConnection
 
 from .errors import ForeignKeyViolation, UniqueViolation
 from .models.projects_nodes import projects_nodes
+from .utils_models import FromRowMixin
 
 
 #
@@ -34,12 +36,16 @@ class ProjectNodesDuplicateNode(BaseProjectNodesError):
 
 @dataclass(frozen=True, slots=True, kw_only=True)
 class ProjectNodeCreate:
-    required_resources: dict = field(default_factory=dict)
+    node_id: uuid.UUID
+    required_resources: dict[str, Any] = field(default_factory=dict)
+
+    @staticmethod
+    def get_field_names(*, exclude: set[str]) -> set[str]:
+        return {f.name for f in fields(ProjectNodeCreate) if f.name not in exclude}
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
-class ProjectNode(ProjectNodeCreate):
-    node_id: uuid.UUID
+class ProjectNode(ProjectNodeCreate, FromRowMixin):
     created: datetime.datetime
     modified: datetime.datetime
 
@@ -52,9 +58,8 @@ async def add(
         self,
         connection: SAConnection,
         *,
-        node_id: uuid.UUID,
-        node: ProjectNodeCreate,
-    ) -> ProjectNode:
+        nodes: list[ProjectNodeCreate],
+    ) -> list[ProjectNode]:
         """creates a new entry in *projects_nodes* and *projects_to_projects_nodes* tables
 
         NOTE: Do not use this in an asyncio.gather call as this will fail!
@@ -65,51 +70,62 @@ async def add(
             ProjectsNodesNodeNotFound: in case the node does not exist
 
         """
-        async with connection.begin():
-            try:
-                result = await connection.execute(
-                    projects_nodes.insert()
-                    .values(
-                        project_uuid=f"{self.project_uuid}",
-                        node_id=f"{node_id}",
+        if not nodes:
+            return []
+        insert_stmt = (
+            projects_nodes.insert()
+            .values(
+                [
+                    {
+                        "project_uuid": f"{self.project_uuid}",
                         **asdict(node),
-                    )
-                    .returning(
-                        *[
-                            c
-                            for c in projects_nodes.c
-                            if c is not projects_nodes.c.project_uuid
-                        ]
-                    )
-                )
-                created_node_db = await result.first()
-                assert created_node_db  # nosec
-                created_node = ProjectNode(**dict(created_node_db.items()))
-
-                return created_node
-            except ForeignKeyViolation as exc:
-                # this happens when the project does not exist, as we first check the node exists
-                raise ProjectNodesProjectNotFound(
-                    f"Project {self.project_uuid} not found"
-                ) from exc
-            except UniqueViolation as exc:
-                # this happens if the node already exists on creation
-                raise ProjectNodesDuplicateNode(
-                    f"Project node {node_id} already exists"
-                ) from exc
+                    }
+                    for node in nodes
+                ]
+            )
+            .returning(
+                *[
+                    c
+                    for c in projects_nodes.columns
+                    if c is not projects_nodes.c.project_uuid
+                ]
+            )
+        )
+
+        try:
+            result = await connection.execute(insert_stmt)
+            assert result  # nosec
+            rows = await result.fetchall()
+            assert rows is not None  # nosec
+            return [ProjectNode.from_row(r) for r in rows]
+        except ForeignKeyViolation as exc:
+            # this happens when the project does not exist, as we first check the node exists
+            raise ProjectNodesProjectNotFound(
+                f"Project {self.project_uuid} not found"
+            ) from exc
+        except UniqueViolation as exc:
+            # this happens if the node already exists on creation
+            raise ProjectNodesDuplicateNode(
+                f"Project node already exists: {exc}"
+            ) from exc
 
     async def list(self, connection: SAConnection) -> list[ProjectNode]:
         """list the nodes in the current project
 
         NOTE: Do not use this in an asyncio.gather call as this will fail!
         """
         list_stmt = sqlalchemy.select(
-            *[c for c in projects_nodes.c if c is not projects_nodes.c.project_uuid]
+            *[
+                c
+                for c in projects_nodes.columns
+                if c is not projects_nodes.c.project_uuid
+            ]
         ).where(projects_nodes.c.project_uuid == f"{self.project_uuid}")
-        nodes = [
-            ProjectNode(**dict(row.items()))
-            async for row in connection.execute(list_stmt)
-        ]
+        result = await connection.execute(list_stmt)
+        assert result  # nosec
+        rows = await result.fetchall()
+        assert rows is not None  # nosec
+        nodes = [ProjectNode.from_row(row) for row in rows]
         return nodes
 
     async def get(self, connection: SAConnection, *, node_id: uuid.UUID) -> ProjectNode:
@@ -134,7 +150,7 @@ async def get(self, connection: SAConnection, *, node_id: uuid.UUID) -> ProjectN
         if row is None:
             raise ProjectNodesNodeNotFound(f"Node with {node_id} not found")
         assert row  # nosec
-        return ProjectNode(**dict(row.items()))
+        return ProjectNode.from_row(row)
 
     async def update(
         self, connection: SAConnection, *, node_id: uuid.UUID, **values
@@ -158,11 +174,11 @@ async def update(
             )
         )
         result = await connection.execute(update_stmt)
-        updated_entry = await result.first()
-        if not updated_entry:
+        row = await result.first()
+        if not row:
             raise ProjectNodesNodeNotFound(f"Node with {node_id} not found")
-        assert updated_entry  # nosec
-        return ProjectNode(**dict(updated_entry.items()))
+        assert row  # nosec
+        return ProjectNode.from_row(row)
 
     async def delete(self, connection: SAConnection, *, node_id: uuid.UUID) -> None:
         """delete a node in the current project