Merge branch 'master' into pr-osparc-fix-metric-raising-errors

Author: GitHK

api/specs/web-server/_projects_nodes.py

@@ -21,9 +21,9 @@
     ServiceResourcesDict,
 )
 from models_library.generics import Envelope
+from models_library.groups import GroupID
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID
-from models_library.users import GroupID
 from simcore_service_webserver._meta import API_VTAG
 from simcore_service_webserver.projects._crud_handlers import ProjectPathParams
 from simcore_service_webserver.projects._nodes_handlers import (

api/specs/web-server/_wallets.py

@@ -27,8 +27,8 @@
     WalletPaymentInitiated,
 )
 from models_library.generics import Envelope
+from models_library.groups import GroupID
 from models_library.rest_pagination import Page, PageQueryParameters
-from models_library.users import GroupID
 from models_library.wallets import WalletID
 from simcore_service_webserver._meta import API_VTAG
 from simcore_service_webserver.wallets._groups_api import WalletGroupGet

packages/aws-library/src/aws_library/ec2/_client.py

@@ -181,7 +181,8 @@ async def launch_instances(
             )
             instance_ids = [i["InstanceId"] for i in instances["Instances"]]
             _logger.info(
-                "New instances launched: %s, waiting for them to start now...",
+                "%s New instances launched: %s, waiting for them to start now...",
+                len(instance_ids),
                 instance_ids,
             )
 

packages/common-library/src/common_library/groups_enums.py

@@ -0,0 +1,13 @@
+import enum
+
+
+class GroupType(enum.Enum):
+    """
+    standard: standard group, e.g. any group that is not a primary group or special group such as the everyone group
+    primary: primary group, e.g. the primary group is the user own defined group that typically only contain the user (same as in linux)
+    everyone: the only group for all users
+    """
+
+    STANDARD = "standard"
+    PRIMARY = "primary"
+    EVERYONE = "everyone"

packages/models-library/src/models_library/api_schemas_catalog/service_access_rights.py

@@ -1,7 +1,7 @@
 from pydantic import BaseModel
 
+from ..groups import GroupID
 from ..services import ServiceKey, ServiceVersion
-from ..users import GroupID
 
 
 class ServiceAccessRightsGet(BaseModel):

packages/models-library/src/models_library/api_schemas_catalog/services.py

@@ -6,6 +6,7 @@
 
 from ..boot_options import BootOptions
 from ..emails import LowerCaseEmailStr
+from ..groups import GroupID
 from ..services_access import ServiceAccessRights, ServiceGroupAccessRightsV2
 from ..services_authoring import Author
 from ..services_enums import ServiceType
@@ -18,7 +19,6 @@
 )
 from ..services_resources import ServiceResourcesDict
 from ..services_types import ServiceKey, ServiceVersion
-from ..users import GroupID
 from ..utils.change_case import snake_to_camel
 
 

packages/models-library/src/models_library/api_schemas_webserver/folders.py

@@ -3,8 +3,8 @@
 
 from models_library.basic_types import IDStr
 from models_library.folders import FolderID
+from models_library.groups import GroupID
 from models_library.projects_access import AccessRights
-from models_library.users import GroupID
 from models_library.utils.common_validators import null_or_none_str_to_none_validator
 from pydantic import ConfigDict, PositiveInt, field_validator
 

packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py

@@ -6,7 +6,7 @@
 from ..access_rights import AccessRights
 from ..basic_types import IDStr
 from ..folders import FolderID
-from ..users import GroupID
+from ..groups import GroupID
 from ..utils.common_validators import null_or_none_str_to_none_validator
 from ..workspaces import WorkspaceID
 from ._base import InputSchema, OutputSchema

packages/models-library/src/models_library/api_schemas_webserver/socketio.py

@@ -1,5 +1,6 @@
 from ..basic_types import IDStr
-from ..users import GroupID, UserID
+from ..groups import GroupID
+from ..users import UserID
 
 
 class SocketIORoomStr(IDStr):

packages/models-library/src/models_library/api_schemas_webserver/wallets.py

@@ -5,7 +5,7 @@
 from pydantic import ConfigDict, Field, HttpUrl, ValidationInfo, field_validator
 
 from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal
-from ..users import GroupID
+from ..groups import GroupID
 from ..wallets import WalletID, WalletStatus
 from ._base import InputSchema, OutputSchema
 

packages/models-library/src/models_library/api_schemas_webserver/workspaces.py

@@ -2,7 +2,7 @@
 from typing import NamedTuple
 
 from models_library.basic_types import IDStr
-from models_library.users import GroupID
+from models_library.groups import GroupID
 from models_library.workspaces import WorkspaceID
 from pydantic import ConfigDict, PositiveInt
 

packages/models-library/src/models_library/clusters.py

@@ -5,7 +5,7 @@
 from pydantic import AnyUrl, BaseModel, ConfigDict, Field, HttpUrl, field_validator
 from pydantic.types import NonNegativeInt
 
-from .users import GroupID
+from .groups import GroupID
 from .utils.common_validators import create_enums_pre_validator
 from .utils.enums import StrAutoEnum
 

packages/models-library/src/models_library/folders.py

@@ -12,7 +12,8 @@
 )
 
 from .access_rights import AccessRights
-from .users import GroupID, UserID
+from .groups import GroupID
+from .users import UserID
 from .utils.enums import StrAutoEnum
 from .workspaces import WorkspaceID
 

packages/models-library/src/models_library/groups.py

@@ -1,38 +1,27 @@
-import enum
 from typing import Annotated, Final, NamedTuple, TypeAlias
 
 from common_library.basic_types import DEFAULT_FACTORY
+from common_library.groups_enums import GroupType as GroupType
 from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_validator
 from pydantic.types import PositiveInt
 from typing_extensions import TypedDict
 
 from .basic_types import IDStr
-from .users import GroupID, UserID
+from .users import UserID
 from .utils.common_validators import create_enums_pre_validator
 
 EVERYONE_GROUP_ID: Final[int] = 1
 
+GroupID: TypeAlias = PositiveInt
 
-__all__: tuple[str, ...] = ("GroupID",)
-
-
-class GroupTypeInModel(str, enum.Enum):
-    """
-    standard: standard group, e.g. any group that is not a primary group or special group such as the everyone group
-    primary: primary group, e.g. the primary group is the user own defined group that typically only contain the user (same as in linux)
-    everyone: the only group for all users
-    """
-
-    STANDARD = "standard"
-    PRIMARY = "primary"
-    EVERYONE = "everyone"
+__all__: tuple[str, ...] = ("GroupType",)
 
 
 class Group(BaseModel):
     gid: PositiveInt
     name: str
     description: str
-    group_type: Annotated[GroupTypeInModel, Field(alias="type")]
+    group_type: Annotated[GroupType, Field(alias="type")]
     thumbnail: str | None
 
     inclusion_rules: Annotated[
@@ -43,7 +32,7 @@ class Group(BaseModel):
     ] = DEFAULT_FACTORY
 
     _from_equivalent_enums = field_validator("group_type", mode="before")(
-        create_enums_pre_validator(GroupTypeInModel)
+        create_enums_pre_validator(GroupType)
     )
 
     model_config = ConfigDict(populate_by_name=True)

packages/models-library/src/models_library/services_access.py

@@ -4,7 +4,7 @@
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from .users import GroupID
+from .groups import GroupID
 from .utils.change_case import snake_to_camel
 
 

packages/models-library/src/models_library/users.py

@@ -5,7 +5,6 @@
 
 UserID: TypeAlias = PositiveInt
 UserNameID: TypeAlias = IDStr
-GroupID: TypeAlias = PositiveInt
 
 
 FirstNameStr: TypeAlias = Annotated[

packages/models-library/src/models_library/workspaces.py

@@ -12,7 +12,8 @@
 )
 
 from .access_rights import AccessRights
-from .users import GroupID, UserID
+from .groups import GroupID
+from .users import UserID
 from .utils.enums import StrAutoEnum
 
 WorkspaceID: TypeAlias = PositiveInt

packages/models-library/tests/test_api_schemas_webserver_socketio.py

@@ -3,7 +3,8 @@
 import pytest
 from faker import Faker
 from models_library.api_schemas_webserver.socketio import SocketIORoomStr
-from models_library.users import GroupID, UserID
+from models_library.groups import GroupID
+from models_library.users import UserID
 
 
 @pytest.fixture

packages/notifications-library/tests/with_db/conftest.py

@@ -11,8 +11,9 @@
 import pytest
 import sqlalchemy as sa
 from models_library.basic_types import IDStr
+from models_library.groups import GroupID
 from models_library.products import ProductName
-from models_library.users import GroupID, UserID
+from models_library.users import UserID
 from notifications_library._templates import get_default_named_templates
 from pydantic import validate_call
 from simcore_postgres_database.models.jinja2_templates import jinja2_templates

packages/postgres-database/requirements/prod.txt

@@ -8,4 +8,5 @@
 --requirement _base.txt
 --requirement _migration.txt
 
+simcore-common-library @ ../common-library/
 simcore-postgres-database @ .

packages/postgres-database/src/simcore_postgres_database/models/groups.py

@@ -4,27 +4,16 @@
     - Groups have a ID, name and a list of users that belong to the group
 """
 
-import enum
 
 import sqlalchemy as sa
+from common_library.groups_enums import GroupType
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.sql import func
 
 from ._common import RefActions
 from .base import metadata
 
-
-class GroupType(enum.Enum):
-    """
-    standard: standard group, e.g. any group that is not a primary group or special group such as the everyone group
-    primary: primary group, e.g. the primary group is the user own defined group that typically only contain the user (same as in linux)
-    everyone: the only group for all users
-    """
-
-    STANDARD = "standard"
-    PRIMARY = "primary"
-    EVERYONE = "everyone"
-
+__all__: tuple[str, ...] = ("GroupType",)
 
 groups = sa.Table(
     "groups",

packages/pytest-simcore/src/pytest_simcore/helpers/autoscaling.py

@@ -0,0 +1,73 @@
+from collections.abc import Callable
+
+import arrow
+from aws_library.ec2 import EC2InstanceData
+from models_library.generated_models.docker_rest_api import (
+    Availability,
+    Node,
+    NodeState,
+)
+from pytest_mock import MockType
+from simcore_service_autoscaling.models import AssociatedInstance, Cluster
+from simcore_service_autoscaling.utils.utils_docker import (
+    _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY,
+    _OSPARC_SERVICE_READY_LABEL_KEY,
+    _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY,
+)
+
+
+def assert_cluster_state(
+    spied_cluster_analysis: MockType, *, expected_calls: int, expected_num_machines: int
+) -> Cluster:
+    assert spied_cluster_analysis.call_count == expected_calls
+
+    assert isinstance(spied_cluster_analysis.spy_return, Cluster)
+    assert (
+        spied_cluster_analysis.spy_return.total_number_of_machines()
+        == expected_num_machines
+    )
+    print("current cluster state:", spied_cluster_analysis.spy_return)
+    cluster = spied_cluster_analysis.spy_return
+    spied_cluster_analysis.reset_mock()
+    return cluster
+
+
+def create_fake_association(
+    create_fake_node: Callable[..., Node],
+    drained_machine_id: str | None,
+    terminating_machine_id: str | None,
+):
+    fake_node_to_instance_map = {}
+
+    async def _fake_node_creator(
+        _nodes: list[Node], ec2_instances: list[EC2InstanceData]
+    ) -> tuple[list[AssociatedInstance], list[EC2InstanceData]]:
+        def _create_fake_node_with_labels(instance: EC2InstanceData) -> Node:
+            if instance not in fake_node_to_instance_map:
+                fake_node = create_fake_node()
+                assert fake_node.spec
+                fake_node.spec.availability = Availability.active
+                assert fake_node.status
+                fake_node.status.state = NodeState.ready
+                assert fake_node.spec.labels
+                fake_node.spec.labels |= {
+                    _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY: arrow.utcnow().isoformat(),
+                    _OSPARC_SERVICE_READY_LABEL_KEY: (
+                        "true" if instance.id != drained_machine_id else "false"
+                    ),
+                }
+                if instance.id == terminating_machine_id:
+                    fake_node.spec.labels |= {
+                        _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY: arrow.utcnow().isoformat()
+                    }
+                fake_node_to_instance_map[instance] = fake_node
+            return fake_node_to_instance_map[instance]
+
+        associated_instances = [
+            AssociatedInstance(node=_create_fake_node_with_labels(i), ec2_instance=i)
+            for i in ec2_instances
+        ]
+
+        return associated_instances, []
+
+    return _fake_node_creator

packages/pytest-simcore/src/pytest_simcore/helpers/webserver_workspaces.py

@@ -1,6 +1,6 @@
 import sqlalchemy as sa
 from aiohttp import web
-from models_library.users import GroupID
+from models_library.groups import GroupID
 from models_library.workspaces import WorkspaceID
 from simcore_postgres_database.models.workspaces_access_rights import (
     workspaces_access_rights,

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/auth/api_keys.py

@@ -26,7 +26,7 @@ async def create_api_key(
         product_name=product_name,
         api_key=api_key,
     )
-    assert isinstance(result, ApiKeyGet)
+    assert isinstance(result, ApiKeyGet)  # nosec
     return result
 
 
@@ -45,7 +45,7 @@ async def get_api_key(
         product_name=product_name,
         api_key_id=api_key_id,
     )
-    assert isinstance(result, ApiKeyGet)
+    assert isinstance(result, ApiKeyGet)  # nosec
     return result
 
 
@@ -63,4 +63,4 @@ async def delete_api_key(
         product_name=product_name,
         api_key_id=api_key_id,
     )
-    assert result is None
+    assert result is None  # nosec