Skip to content

Commit 9f60948

Browse files
sandereggsanderegg
committed
new docker label dictionary
1 parent 2858082 commit 9f60948

File tree

2 files changed

+118
-53
lines changed

2 files changed

+118
-53
lines changed

‎packages/models-library/src/models_library/docker.py

Lines changed: 107 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import contextlib
12
import re
23
from typing import Any, Final
34

@@ -6,9 +7,18 @@
67
from models_library.projects import ProjectID
78
from models_library.projects_nodes import NodeID
89
from models_library.users import UserID
9-
from pydantic import BaseModel, ByteSize, ConstrainedStr, Field, root_validator
10+
from pydantic import (
11+
BaseModel,
12+
ByteSize,
13+
ConstrainedStr,
14+
Field,
15+
ValidationError,
16+
parse_obj_as,
17+
root_validator,
18+
)
1019

1120
from .basic_regex import DOCKER_GENERIC_TAG_KEY_RE, DOCKER_LABEL_KEY_REGEX
21+
from .utils.fastapi_encoders import jsonable_encoder
1222

1323

1424
class DockerLabelKey(ConstrainedStr):
@@ -24,14 +34,18 @@ class DockerGenericTag(ConstrainedStr):
2434

2535
_SIMCORE_CONTAINER_PREFIX: Final[str] = "io.simcore.container."
2636
_BACKWARDS_COMPATIBILITY_MAP: Final[dict[str, str]] = {
27-
"user_id": f"{_SIMCORE_CONTAINER_PREFIX}user-id",
28-
"study_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id",
29-
"project_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id",
30-
"uuid": f"{_SIMCORE_CONTAINER_PREFIX}node-id",
3137
"node_id": f"{_SIMCORE_CONTAINER_PREFIX}node-id",
3238
"product_name": f"{_SIMCORE_CONTAINER_PREFIX}product-name",
39+
"project_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id",
3340
"simcore_user_agent": f"{_SIMCORE_CONTAINER_PREFIX}simcore-user-agent",
41+
"study_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id",
42+
"user_id": f"{_SIMCORE_CONTAINER_PREFIX}user-id",
43+
"uuid": f"{_SIMCORE_CONTAINER_PREFIX}node-id",
44+
"mem_limit": f"{_SIMCORE_CONTAINER_PREFIX}memory-limit",
45+
"swarm_stack_name": f"{_SIMCORE_CONTAINER_PREFIX}swarm-stack-name",
3446
}
47+
_UNDEFINED_VALUE_STR: Final[str] = "undefined"
48+
_UNDEFINED_VALUE_INT: Final[str] = "0"
3549

3650

3751
class SimcoreServiceDockerLabelKeys(BaseModel):
@@ -46,36 +60,52 @@ class SimcoreServiceDockerLabelKeys(BaseModel):
4660
..., alias=f"{_SIMCORE_CONTAINER_PREFIX}simcore-user-agent"
4761
)
4862

49-
# None is for backwards compatibility, can be removed in a few sprints
50-
memory_limit: ByteSize | None = Field(
63+
swarm_stack_name: str = Field(
64+
..., alias=f"{_SIMCORE_CONTAINER_PREFIX}swarm-stack-name"
65+
)
66+
67+
memory_limit: ByteSize = Field(
5168
..., alias=f"{_SIMCORE_CONTAINER_PREFIX}memory-limit"
5269
)
53-
cpu_limit: float | None = Field(..., alias=f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit")
70+
cpu_limit: float = Field(..., alias=f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit")
5471

5572
@root_validator(pre=True)
5673
def _backwards_compatibility(cls, values: dict[str, Any]) -> dict[str, Any]:
57-
# NOTE: this is necessary for deployment and legacy service
74+
# NOTE: this is necessary for dy-sidecar and legacy service until they are adjusted
5875
if mapped_values := {
5976
_BACKWARDS_COMPATIBILITY_MAP[k]: v
6077
for k, v in values.items()
6178
if k in _BACKWARDS_COMPATIBILITY_MAP
6279
}:
80+
# these values were sometimes omitted, so let's provide some defaults
81+
for key in ["product-name", "simcore-user-agent", "swarm-stack-name"]:
82+
mapped_values.setdefault(
83+
f"{_SIMCORE_CONTAINER_PREFIX}{key}", _UNDEFINED_VALUE_STR
84+
)
85+
6386
mapped_values.setdefault(
64-
f"{_SIMCORE_CONTAINER_PREFIX}product-name", "osparc"
87+
f"{_SIMCORE_CONTAINER_PREFIX}memory-limit", _UNDEFINED_VALUE_INT
6588
)
89+
90+
def _convert_nano_cpus_to_cpus(nano_cpu: str) -> str:
91+
with contextlib.suppress(ValidationError):
92+
return f"{parse_obj_as(float, nano_cpu) / (1.0*10**9):.2f}"
93+
return _UNDEFINED_VALUE_INT
94+
6695
mapped_values.setdefault(
67-
f"{_SIMCORE_CONTAINER_PREFIX}simcore-user-agent", "undefined"
96+
f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit",
97+
_convert_nano_cpus_to_cpus(
98+
values.get("nano_cpus_limit", _UNDEFINED_VALUE_INT)
99+
),
68100
)
69-
mapped_values.setdefault(f"{_SIMCORE_CONTAINER_PREFIX}memory-limit", "0")
70-
mapped_values.setdefault(f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit", "0")
71101
return mapped_values
72102
return values
73103

74104
def to_docker_labels(self) -> dict[DockerLabelKey, str]:
75105
"""returns a dictionary of strings as required by docker"""
76-
std_export = self.dict(by_alias=True)
106+
std_export = jsonable_encoder(self, by_alias=True)
77107
return {
78-
DockerLabelKey(f"{_SIMCORE_CONTAINER_PREFIX}{k.replace('_', '-')}"): f"{v}"
108+
DockerLabelKey(f"{k.replace('_', '-')}"): f"{v}"
79109
for k, v in sorted(std_export.items())
80110
}
81111

@@ -90,23 +120,80 @@ class Config:
90120
allow_population_by_field_name = True
91121
schema_extra = {
92122
"examples": [
93-
# legacy with no limits (a.k.a all dynamic services)
123+
# legacy service labels
94124
{
125+
"port": "8080", # TODO: go away
126+
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
127+
"swarm_stack_name": "devel-simcore",
128+
"type": "main", # TODO: go away
129+
"user_id": "5",
95130
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
131+
},
132+
# legacy container labels
133+
{
134+
"mem_limit": "1073741824",
135+
"nano_cpus_limit": "4000000000",
136+
"node_id": "1f963626-66e1-43f1-a777-33955c08b909",
137+
"simcore_user_agent": "puppeteer",
96138
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
139+
"swarm_stack_name": "devel-simcore",
97140
"user_id": "5",
141+
},
142+
# dy-sidecar service labels
143+
{
144+
"key": "simcore/services/dynamic/jupyter-math", # TODO: go away
145+
"port": "8888", # TODO: go away
146+
"service_image": "itisfoundation/dynamic-sidecar:master-github-latest", # TODO: go away
147+
"service_port": "8888", # TODO: go away
148+
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
149+
"swarm_stack_name": "devel-simcore",
150+
"type": "main-v2", # TODO: go away
151+
"user_id": "5",
152+
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
153+
"version": "2.0.9", # TODO: go away
154+
# "simcore_user_agent": "???????misssing??"
155+
},
156+
# dy-sidecar container labels
157+
{
158+
"mem_limit": "1073741824",
159+
"nano_cpus_limit": "4000000000",
160+
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
161+
"user_id": "5",
162+
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
163+
},
164+
# dy-proxy service labels
165+
{
166+
"dynamic-type": "dynamic-sidecar",
167+
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
168+
"swarm_stack_name": "devel-simcore",
169+
"type": "dependency-v2",
170+
"user_id": "5",
171+
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
172+
},
173+
# dy-proxy container labels
174+
{
175+
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
176+
"user_id": "5",
177+
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
178+
},
179+
# dy-sidecar user-services labels
180+
{
98181
"product_name": "osparc",
99182
"simcore_user_agent": "puppeteer",
183+
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
184+
"user_id": "5",
185+
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
100186
},
101187
# modern both dynamic-sidecar services and computational services
102188
{
189+
"io.simcore.container.cpu-limit": "2.4",
190+
"io.simcore.container.memory-limit": "1073741824",
103191
"io.simcore.container.node-id": "1f963626-66e1-43f1-a777-33955c08b909",
104-
"io.simcore.container.project-id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
105-
"io.simcore.container.user-id": "5",
106192
"io.simcore.container.product-name": "osparc",
193+
"io.simcore.container.project-id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
107194
"io.simcore.container.simcore-user-agent": "puppeteer",
108-
"io.simcore.container.memory-limit": "1073741824",
109-
"io.simcore.container.cpu-limit": "2.4",
195+
"io.simcore.container.swarm-stack-name": "devel-osparc",
196+
"io.simcore.container.user-id": "5",
110197
},
111198
]
112199
}

‎packages/models-library/tests/test_docker.py

Lines changed: 11 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import pytest
99
from faker import Faker
1010
from models_library.docker import (
11+
_SIMCORE_CONTAINER_PREFIX,
1112
DockerGenericTag,
1213
DockerLabelKey,
1314
SimcoreServiceDockerLabelKeys,
@@ -106,42 +107,19 @@ def test_docker_generic_tag(image_name: str, valid: bool):
106107

107108

108109
@pytest.mark.parametrize(
109-
"obj_data",
110-
[
111-
pytest.param(
112-
{
113-
"user_id": _faker.pyint(),
114-
"project_id": _faker.uuid4(),
115-
"node_id": _faker.uuid4(),
116-
},
117-
id="parse_existing_service_labels-and-legacy",
118-
),
119-
pytest.param(
120-
{
121-
"user_id": _faker.pyint(),
122-
"project_id": _faker.uuid4(),
123-
"node_id": _faker.uuid4(),
124-
"product_name": "test_p",
125-
"simcore_user_agent": "a-test-puppet",
126-
},
127-
id="parse_new_service_labels-soon-to-removed",
128-
),
129-
pytest.param(
130-
{
131-
"io.simcore.container.user_id": _faker.pyint(),
132-
"io.simcore.container.project_id": _faker.uuid4(),
133-
"io.simcore.container.node_id": _faker.uuid4(),
134-
"io.simcore.container.product-name": "test_p",
135-
"io.simcore.container.simcore_user_agent": "a-test-puppet",
136-
},
137-
id="labels following docker rules",
138-
),
139-
],
110+
"obj_data", SimcoreServiceDockerLabelKeys.Config.schema_extra["examples"], ids=str
140111
)
141112
def test_simcore_service_docker_label_keys(obj_data: dict[str, Any]):
142113
simcore_service_docker_label_keys = SimcoreServiceDockerLabelKeys.parse_obj(
143114
obj_data
144115
)
145116
exported_dict = simcore_service_docker_label_keys.to_docker_labels()
146-
assert all(isinstance(v, str) for v in exported_dict.values())
147-
assert parse_obj_as(SimcoreServiceDockerLabelKeys, exported_dict)
117+
assert all(
118+
isinstance(v, str) for v in exported_dict.values()
119+
), "docker labels must be strings!"
120+
assert all(key.startswith(_SIMCORE_CONTAINER_PREFIX) for key in exported_dict)
121+
re_imported_docker_label_keys = parse_obj_as(
122+
SimcoreServiceDockerLabelKeys, exported_dict
123+
)
124+
assert re_imported_docker_label_keys
125+
assert simcore_service_docker_label_keys == re_imported_docker_label_keys

0 commit comments

Comments
 (0)