From 0c0153c0678d7e661c5edc40056e7e956a1f54c3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 13 Mar 2025 13:17:31 +0100 Subject: [PATCH 01/12] ignore warnings --- packages/models-library/src/models_library/docker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index db5f51ef359..d6f5decdf1a 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -63,9 +63,9 @@ def from_key(cls, key: str) -> "DockerLabelKey": _UNDEFINED_LABEL_VALUE_INT: Final[str] = "0" -DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY: Final[ - DockerLabelKey -] = TypeAdapter(DockerLabelKey).validate_python("ec2-instance-type") +DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY: Final[DockerLabelKey] = ( + TypeAdapter(DockerLabelKey).validate_python("ec2-instance-type") +) def to_simcore_runtime_docker_label_key(key: str) -> DockerLabelKey: @@ -143,7 +143,7 @@ def to_simcore_runtime_docker_labels(self) -> dict[DockerLabelKey, str]: """returns a dictionary of strings as required by docker""" return { to_simcore_runtime_docker_label_key(k): f"{v}" - for k, v in sorted(self.model_dump().items()) + for k, v in sorted(self.model_dump(warnings=False).items()) } @classmethod From 2d89445b83fb7936ce8bb89a18591db35de6d6d8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 18 Mar 2025 13:53:13 +0100 Subject: [PATCH 02/12] revalidate before dumping --- packages/models-library/src/models_library/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index d6f5decdf1a..15806f7fd07 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -143,7 +143,7 @@ def to_simcore_runtime_docker_labels(self) -> dict[DockerLabelKey, str]: """returns a dictionary of strings as required by docker""" return { to_simcore_runtime_docker_label_key(k): f"{v}" - for k, v in sorted(self.model_dump(warnings=False).items()) + for k, v in sorted(self.model_validate(self).model_dump().items()) } @classmethod From 3dea4915f0cc0b1f789c7a8ec6ff3b3f77c1bd51 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 18 Mar 2025 13:55:03 +0100 Subject: [PATCH 03/12] use model_copy --- packages/models-library/src/models_library/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index 15806f7fd07..a7cef8b4d5b 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -143,7 +143,7 @@ def to_simcore_runtime_docker_labels(self) -> dict[DockerLabelKey, str]: """returns a dictionary of strings as required by docker""" return { to_simcore_runtime_docker_label_key(k): f"{v}" - for k, v in sorted(self.model_validate(self).model_dump().items()) + for k, v in sorted(self.model_copy().model_dump().items()) } @classmethod From 3ef87d692fff6da2bd0b61215bbc1d6936a9eaf7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 18 Mar 2025 15:29:14 +0100 Subject: [PATCH 04/12] use model_validate --- .../src/models_library/docker.py | 2 +- .../dynamic_sidecar/docker_compose_specs.py | 26 ++++++++++--------- .../simcore_service_director_v2/utils/dask.py | 26 ++++++++++--------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index a7cef8b4d5b..224fb798944 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -143,7 +143,7 @@ def to_simcore_runtime_docker_labels(self) -> dict[DockerLabelKey, str]: """returns a dictionary of strings as required by docker""" return { to_simcore_runtime_docker_label_key(k): f"{v}" - for k, v in sorted(self.model_copy().model_dump().items()) + for k, v in sorted(self.model_dump().items()) } @classmethod diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py index c68b957c74a..f72287b0082 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py @@ -128,9 +128,9 @@ def _update_paths_mappings( ) env_vars["DY_SIDECAR_PATH_INPUTS"] = f"{path_mappings.inputs_path}" env_vars["DY_SIDECAR_PATH_OUTPUTS"] = f"{path_mappings.outputs_path}" - env_vars[ - "DY_SIDECAR_STATE_PATHS" - ] = f"{json_dumps( { f'{p}' for p in path_mappings.state_paths } )}" + env_vars["DY_SIDECAR_STATE_PATHS"] = ( + f"{json_dumps( { f'{p}' for p in path_mappings.state_paths } )}" + ) service_content["environment"] = _EnvironmentSection.export_as_list(env_vars) @@ -241,15 +241,17 @@ def _update_container_labels( spec_service_key, default_limits ) - label_keys = StandardSimcoreDockerLabels.model_construct( - user_id=user_id, - project_id=project_id, - node_id=node_id, - simcore_user_agent=simcore_user_agent, - product_name=product_name, - swarm_stack_name=swarm_stack_name, - memory_limit=ByteSize(container_limits["memory"]), - cpu_limit=container_limits["cpu"], + label_keys = StandardSimcoreDockerLabels.model_validate( + { + "user_id": user_id, + "project_id": project_id, + "node_id": node_id, + "simcore_user_agent": simcore_user_agent, + "product_name": product_name, + "swarm_stack_name": swarm_stack_name, + "memory_limit": ByteSize(container_limits["memory"]), + "cpu_limit": container_limits["cpu"], + } ) docker_labels = [ f"{k}={v}" for k, v in label_keys.to_simcore_runtime_docker_labels().items() diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index df7db6886ff..c2ab4781cc7 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -315,17 +315,19 @@ def compute_task_labels( ValidationError """ product_name = run_metadata.get("product_name", UNDEFINED_DOCKER_LABEL) - standard_simcore_labels = StandardSimcoreDockerLabels.model_construct( - user_id=user_id, - project_id=project_id, - node_id=node_id, - product_name=product_name, - simcore_user_agent=run_metadata.get( - "simcore_user_agent", UNDEFINED_DOCKER_LABEL - ), - swarm_stack_name=UNDEFINED_DOCKER_LABEL, # NOTE: there is currently no need for this label in the comp backend - memory_limit=node_requirements.ram, - cpu_limit=node_requirements.cpu, + standard_simcore_labels = StandardSimcoreDockerLabels.model_validate( + { + "user_id": user_id, + "project_id": project_id, + "node_id": node_id, + "product_name": product_name, + "simcore_user_agent": run_metadata.get( + "simcore_user_agent", UNDEFINED_DOCKER_LABEL + ), + "swarm_stack_name": UNDEFINED_DOCKER_LABEL, # NOTE: there is currently no need for this label in the comp backend + "memory_limit": node_requirements.ram, + "cpu_limit": node_requirements.cpu, + } ).to_simcore_runtime_docker_labels() return standard_simcore_labels | TypeAdapter(ContainerLabelsDict).validate_python( { @@ -633,7 +635,7 @@ def check_if_cluster_is_able_to_run_pipeline( async def wrap_client_async_routine( - client_coroutine: Coroutine[Any, Any, Any] | Any | None + client_coroutine: Coroutine[Any, Any, Any] | Any | None, ) -> Any: """Dask async behavior does not go well with Pylance as it returns a union of types. this wrapper makes both mypy and pylance happy""" From 2fd7af13be58af73a39d5e245a5a8067996442a8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 21 Mar 2025 14:22:28 +0100 Subject: [PATCH 05/12] fix test --- .../unit/test_modules_dynamic_sidecar_docker_compose_specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_compose_specs.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_compose_specs.py index e449cba3fc1..94726173395 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_compose_specs.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_compose_specs.py @@ -227,7 +227,7 @@ def test_regression_service_has_no_reservations(): f"{to_simcore_runtime_docker_label_key('user-id')}={USER_ID}", f"{to_simcore_runtime_docker_label_key('node-id')}={NODE_ID}", f"{to_simcore_runtime_docker_label_key('swarm-stack-name')}={SWARM_STACK_NAME}", - f"{to_simcore_runtime_docker_label_key('cpu-limit')}=0", + f"{to_simcore_runtime_docker_label_key('cpu-limit')}=0.0", f"{to_simcore_runtime_docker_label_key('memory-limit')}=0", ] ) From aa6242532f4f3d12604bdba2feb75107f616677f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 21 Mar 2025 16:51:17 +0100 Subject: [PATCH 06/12] maybe the fix --- .../src/models_library/docker.py | 14 +++++++++---- packages/models-library/tests/test_docker.py | 21 ++++++++++++++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index 224fb798944..ae23ba7eec4 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -122,18 +122,24 @@ def _backwards_compatibility(cls, values: dict[str, Any]) -> dict[str, Any]: mapped_values.setdefault( f"{_SIMCORE_RUNTIME_DOCKER_LABEL_PREFIX}memory-limit", - _UNDEFINED_LABEL_VALUE_INT, + values.get("memory_limit", _UNDEFINED_LABEL_VALUE_INT), ) def _convert_nano_cpus_to_cpus(nano_cpu: str) -> str: with contextlib.suppress(ValidationError): - return f"{TypeAdapter(float).validate_python(nano_cpu) / (1.0*10**9):.2f}" + return f"{TypeAdapter(float).validate_python(nano_cpu) / (1.0 * 10**9):.2f}" return _UNDEFINED_LABEL_VALUE_INT mapped_values.setdefault( f"{_SIMCORE_RUNTIME_DOCKER_LABEL_PREFIX}cpu-limit", - _convert_nano_cpus_to_cpus( - values.get("nano_cpus_limit", _UNDEFINED_LABEL_VALUE_INT) + values.get( + "cpu_limit", + _convert_nano_cpus_to_cpus( + values.get( + "nano_cpus_limit", + _UNDEFINED_LABEL_VALUE_INT, + ) + ), ), ) return mapped_values diff --git a/packages/models-library/tests/test_docker.py b/packages/models-library/tests/test_docker.py index dd5fed89951..ae1c636a9e2 100644 --- a/packages/models-library/tests/test_docker.py +++ b/packages/models-library/tests/test_docker.py @@ -4,6 +4,7 @@ from typing import Any +from uuid import UUID import pytest from faker import Faker @@ -13,7 +14,7 @@ DockerLabelKey, StandardSimcoreDockerLabels, ) -from pydantic import TypeAdapter, ValidationError +from pydantic import ByteSize, TypeAdapter, ValidationError _faker = Faker() @@ -83,11 +84,11 @@ def test_docker_label_key(label_key: str, valid: bool): True, ), ( - f"registry:5000/si.m--c_ore/services/1234/jupyter-smash:{'A'*128}", + f"registry:5000/si.m--c_ore/services/1234/jupyter-smash:{'A' * 128}", True, ), ( - f"registry:5000/si.m--c_ore/services/1234/jupyter-smash:{'A'*129}", + f"registry:5000/si.m--c_ore/services/1234/jupyter-smash:{'A' * 129}", False, ), ), @@ -122,3 +123,17 @@ def test_simcore_service_docker_label_keys(obj_data: dict[str, Any]): ).validate_python(exported_dict) assert re_imported_docker_label_keys assert simcore_service_docker_label_keys == re_imported_docker_label_keys + + +def test_simcore_service_docker_label_keys_construction(): + simcore_service_docker_label_keys = StandardSimcoreDockerLabels( + user_id=8268, + project_id=UUID("5ea24ce0-0e4d-4ee6-a3f1-e4799752a684"), + node_id=UUID("c17c6279-23c6-412f-8826-867323a7711a"), + product_name="osparc", + simcore_user_agent="oePqmjQbZndJghceKRJR", + swarm_stack_name="UNDEFINED_DOCKER_LABEL", # NOTE: there is currently no need for this label in the comp backend + memory_limit=ByteSize(23424324), + cpu_limit=1.0, + ) + assert simcore_service_docker_label_keys.cpu_limit == 1.0 From 18c5babe5739b033de7c06372e738013fe54b4d3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Mar 2025 13:08:48 +0100 Subject: [PATCH 07/12] always validate compose_spec --- .../models/dynamic_services_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 56c2a27c9c2..18795861881 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -540,8 +540,8 @@ def from_service_inspect( def as_label_data(self) -> str: # compose_spec needs to be json encoded before encoding it to json # and storing it in the label - return self.model_copy( - update={"compose_spec": json.dumps(self.compose_spec)}, deep=True + return self.model_validate( + {**self.model_dump(), "compose_spec": json.dumps(self.compose_spec)} ).model_dump_json() model_config = ConfigDict(extra="allow", populate_by_name=True) From 6df017018a600a9b991e1462752f3fe1cc72736a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Mar 2025 13:20:01 +0100 Subject: [PATCH 08/12] revert --- .../models/dynamic_services_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 18795861881..56c2a27c9c2 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -540,8 +540,8 @@ def from_service_inspect( def as_label_data(self) -> str: # compose_spec needs to be json encoded before encoding it to json # and storing it in the label - return self.model_validate( - {**self.model_dump(), "compose_spec": json.dumps(self.compose_spec)} + return self.model_copy( + update={"compose_spec": json.dumps(self.compose_spec)}, deep=True ).model_dump_json() model_config = ConfigDict(extra="allow", populate_by_name=True) From a5ec9998bd443acd0d1e17f26e3b52caed627151 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Mar 2025 13:21:58 +0100 Subject: [PATCH 09/12] validate compose_spec --- .../models/dynamic_services_scheduler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 56c2a27c9c2..8c12bae5bf9 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -540,8 +540,10 @@ def from_service_inspect( def as_label_data(self) -> str: # compose_spec needs to be json encoded before encoding it to json # and storing it in the label - return self.model_copy( - update={"compose_spec": json.dumps(self.compose_spec)}, deep=True + return self.model_validate( + self.model_copy( + update={"compose_spec": json.dumps(self.compose_spec)}, deep=True + ) ).model_dump_json() model_config = ConfigDict(extra="allow", populate_by_name=True) From 831ff21f5d383e65beaa46d0ac22348b7de26bff Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Mar 2025 14:00:48 +0100 Subject: [PATCH 10/12] validate compose_spec --- .../models/dynamic_services_scheduler.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 8c12bae5bf9..4940a000d68 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -540,10 +540,8 @@ def from_service_inspect( def as_label_data(self) -> str: # compose_spec needs to be json encoded before encoding it to json # and storing it in the label - return self.model_validate( - self.model_copy( - update={"compose_spec": json.dumps(self.compose_spec)}, deep=True - ) + return self.model_copy( + update={"compose_spec": self.compose_spec}, deep=True ).model_dump_json() model_config = ConfigDict(extra="allow", populate_by_name=True) From 5fe396762eef9e630c4fb352f4f889bfa6abf565 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Mar 2025 14:11:29 +0100 Subject: [PATCH 11/12] null --- .../models/dynamic_services_scheduler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 4940a000d68..471b0875fb7 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -541,7 +541,12 @@ def as_label_data(self) -> str: # compose_spec needs to be json encoded before encoding it to json # and storing it in the label return self.model_copy( - update={"compose_spec": self.compose_spec}, deep=True + update={ + "compose_spec": ( + json.dumps(self.compose_spec) if self.compose_spec else None + ) + }, + deep=True, ).model_dump_json() model_config = ConfigDict(extra="allow", populate_by_name=True) From 2f723f0fb0a58ac208063c8d622c8926476f1da7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Mar 2025 15:53:33 +0100 Subject: [PATCH 12/12] back --- .../models/dynamic_services_scheduler.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 471b0875fb7..5fbe75915c6 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -541,11 +541,7 @@ def as_label_data(self) -> str: # compose_spec needs to be json encoded before encoding it to json # and storing it in the label return self.model_copy( - update={ - "compose_spec": ( - json.dumps(self.compose_spec) if self.compose_spec else None - ) - }, + update={"compose_spec": json.dumps(self.compose_spec)}, deep=True, ).model_dump_json()