diff --git a/packages/models-library/src/models_library/utils/specs_substitution.py b/packages/models-library/src/models_library/utils/specs_substitution.py index e593effd962..413b8799da7 100644 --- a/packages/models-library/src/models_library/utils/specs_substitution.py +++ b/packages/models-library/src/models_library/utils/specs_substitution.py @@ -1,14 +1,13 @@ -from typing import Any, TypeAlias +from typing import Any, TypeAlias, cast import yaml -from models_library.utils.string_substitution import ( +from pydantic import StrictBool, StrictFloat, StrictInt + +from .string_substitution import ( SubstitutionsDict, TextTemplate, substitute_all_legacy_identifiers, ) -from pydantic import StrictBool, StrictFloat, StrictInt - -from .string_substitution import SubstitutionsDict, TextTemplate # This constraint on substitution values is to avoid # deserialization issues on the TextTemplate substitution! @@ -72,5 +71,5 @@ def set_substitutions( def run(self) -> dict[str, Any]: new_specs_txt: str = self._template.safe_substitute(self._substitutions) - new_specs: dict = yaml.safe_load(new_specs_txt) - return new_specs + new_specs = yaml.safe_load(new_specs_txt) + return cast(dict[str, Any], new_specs) diff --git a/packages/models-library/src/models_library/utils/string_substitution.py b/packages/models-library/src/models_library/utils/string_substitution.py index 22a2866f90f..6ce86bc55eb 100644 --- a/packages/models-library/src/models_library/utils/string_substitution.py +++ b/packages/models-library/src/models_library/utils/string_substitution.py @@ -7,7 +7,7 @@ from string import Template from typing import Any -OSPARC_IDENTIFIER_PREFIX = "OSPARC_ENVIRONMENT_" +OSPARC_IDENTIFIER_PREFIX = "OSPARC_VARIABLE_" def upgrade_identifier(identifier: str) -> str: @@ -30,7 +30,7 @@ def upgrade_identifier(identifier: str) -> str: def substitute_all_legacy_identifiers(text: str) -> str: """Substitutes all legacy identifiers found in the text by the new format expected in TemplateText - For instance: '%%this-identifier%%' will be substituted by '$OSPARC_ENVIRONMENT_THIS_IDENTIFIER' + For instance: '%%this-identifier%%' will be substituted by '$OSPARC_VARIABLE_THIS_IDENTIFIER' """ def _upgrade(match): diff --git a/packages/models-library/tests/test_service_settings_labels.py b/packages/models-library/tests/test_service_settings_labels.py index 2391267c44b..6bafd236eb7 100644 --- a/packages/models-library/tests/test_service_settings_labels.py +++ b/packages/models-library/tests/test_service_settings_labels.py @@ -393,18 +393,18 @@ def test_not_allowed_in_both_permit_list_and_outgoing_internet(): @pytest.fixture def vendor_environments() -> dict[str, Any]: return { - "OSPARC_ENVIRONMENT_VENDOR_SECRET_DNS_RESOLVER_ADDRESS": "172.0.0.1", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_DNS_RESOLVER_PORT": 1234, - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENCE_HOSTNAME": "hostname", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_TCP_PORTS": [ + "OSPARC_VARIABLE_VENDOR_SECRET_DNS_RESOLVER_ADDRESS": "172.0.0.1", + "OSPARC_VARIABLE_VENDOR_SECRET_DNS_RESOLVER_PORT": 1234, + "OSPARC_VARIABLE_VENDOR_SECRET_LICENCE_HOSTNAME": "hostname", + "OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS": [ 1, 2, 3, 4, ], - "OSPARC_ENVIRONMENT_VENDOR_SECRET_TCP_PORTS_1": 1, - "OSPARC_ENVIRONMENT_VENDOR_SECRET_TCP_PORTS_2": 2, - "OSPARC_ENVIRONMENT_VENDOR_SECRET_TCP_PORTS_3": 3, + "OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS_1": 1, + "OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS_2": 2, + "OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS_3": 3, } @@ -446,13 +446,13 @@ def service_labels() -> dict[str, str]: { "hostname": "license.com", "tcp_ports": [ - "$OSPARC_ENVIRONMENT_VENDOR_SECRET_TCP_PORTS_1", - "$OSPARC_ENVIRONMENT_VENDOR_SECRET_TCP_PORTS_2", + "$OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS_1", + "$OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS_2", 3, ], "dns_resolver": { - "address": "$OSPARC_ENVIRONMENT_VENDOR_SECRET_DNS_RESOLVER_ADDRESS", - "port": "$OSPARC_ENVIRONMENT_VENDOR_SECRET_DNS_RESOLVER_PORT", + "address": "$OSPARC_VARIABLE_VENDOR_SECRET_DNS_RESOLVER_ADDRESS", + "port": "$OSPARC_VARIABLE_VENDOR_SECRET_DNS_RESOLVER_PORT", }, } ] @@ -522,14 +522,14 @@ def service_labels() -> dict[str, str]: def test_can_parse_labels_with_osparc_identifiers( vendor_environments: dict[str, Any], service_labels: dict[str, str] ): - # can load OSPARC_ENVIRONMENT_ identifiers!! + # can load OSPARC_VARIABLE_ identifiers!! service_meta = SimcoreServiceLabels.parse_obj(service_labels) assert service_meta.containers_allowed_outgoing_permit_list["s4l-core"][ 0 ].tcp_ports == [ - "$OSPARC_ENVIRONMENT_VENDOR_SECRET_TCP_PORTS_1", - "$OSPARC_ENVIRONMENT_VENDOR_SECRET_TCP_PORTS_2", + "$OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS_1", + "$OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS_2", 3, ] diff --git a/packages/models-library/tests/test_utils_specs_substitution.py b/packages/models-library/tests/test_utils_specs_substitution.py index 9d578ae5d42..695dfb40e7d 100644 --- a/packages/models-library/tests/test_utils_specs_substitution.py +++ b/packages/models-library/tests/test_utils_specs_substitution.py @@ -25,24 +25,24 @@ def service_version() -> str: @pytest.fixture() -def available_osparc_environments( +def available_osparc_variables( simcore_registry: str, service_version: str, ) -> dict[str, SubstitutionValue]: - osparc_vendor_environments = { - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_SERVER_HOST": "product_a-server", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_SERVER_PRIMARY_PORT": 1, - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_SERVER_SECONDARY_PORT": 2, - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_DNS_RESOLVER_IP": "1.1.1.1", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_DNS_RESOLVER_PORT": "21", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_FILE": "license.txt", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_FILE_PRODUCT1": "license-p1.txt", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_FILE_PRODUCT2": "license-p2.txt", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LIST": "[1, 2, 3]", + osparc_vendor_variables = { + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_HOST": "product_a-server", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_PRIMARY_PORT": 1, + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_SECONDARY_PORT": 2, + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_DNS_RESOLVER_IP": "1.1.1.1", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_DNS_RESOLVER_PORT": "21", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_FILE": "license.txt", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_FILE_PRODUCT1": "license-p1.txt", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_FILE_PRODUCT2": "license-p2.txt", + "OSPARC_VARIABLE_VENDOR_SECRET_LIST": "[1, 2, 3]", } environs = { - **osparc_vendor_environments, + **osparc_vendor_variables, "SIMCORE_REGISTRY": simcore_registry, "SERVICE_VERSION": service_version, "DISPLAY": "True", @@ -74,8 +74,8 @@ def available_osparc_environments( "init": True, "environment": [ "DISPLAY=${DISPLAY}", - "SOME_LIST=$OSPARC_ENVIRONMENT_VENDOR_SECRET_LIST", - "MY_LICENSE=$OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_FILE", + "SOME_LIST=$OSPARC_VARIABLE_VENDOR_SECRET_LIST", + "MY_LICENSE=$OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_FILE", ], "volumes": ["/tmp/.X11-unix:/tmp/.X11-unix"], }, @@ -94,7 +94,7 @@ def available_osparc_environments( ], ) def test_substitutions_in_compose_spec( - available_osparc_environments: dict[str, SubstitutionValue], + available_osparc_variables: dict[str, SubstitutionValue], service_name: str, service_spec: dict[str, Any], expected_service_spec: dict[str, Any], @@ -103,7 +103,7 @@ def test_substitutions_in_compose_spec( identifiers_requested = specs_resolver.get_identifiers() - substitutions = specs_resolver.set_substitutions(available_osparc_environments) + substitutions = specs_resolver.set_substitutions(available_osparc_variables) assert substitutions is specs_resolver.substitutions assert set(identifiers_requested) == set(substitutions.keys()) @@ -133,7 +133,7 @@ def test_nothing_to_substitute(): def test_no_identifier_present( - available_osparc_environments: dict[str, SubstitutionValue] + available_osparc_variables: dict[str, SubstitutionValue] ): original_spec = {"x": 33, "y": {"z": True}, "foo": "$UNREGISTERED_ID"} @@ -141,7 +141,7 @@ def test_no_identifier_present( specs_resolver = SpecsSubstitutionsResolver(original_spec, upgrade=False) assert specs_resolver.get_identifiers() == ["UNREGISTERED_ID"] - assert specs_resolver.set_substitutions(available_osparc_environments) == {} + assert specs_resolver.set_substitutions(available_osparc_variables) == {} # no substitutions assert specs_resolver.run() == original_spec diff --git a/packages/models-library/tests/test_utils_string_substitution.py b/packages/models-library/tests/test_utils_string_substitution.py index fba707535bf..965f292323a 100644 --- a/packages/models-library/tests/test_utils_string_substitution.py +++ b/packages/models-library/tests/test_utils_string_substitution.py @@ -23,15 +23,15 @@ [ ( "%%container_name.sym-server%%", - "OSPARC_ENVIRONMENT_CONTAINER_NAME_SYM_SERVER", + "OSPARC_VARIABLE_CONTAINER_NAME_SYM_SERVER", ), ( "%service_uuid%", - "OSPARC_ENVIRONMENT_SERVICE_UUID", + "OSPARC_VARIABLE_SERVICE_UUID", ), ( "$SERVICE_VERSION", - "OSPARC_ENVIRONMENT_SERVICE_VERSION", + "OSPARC_VARIABLE_SERVICE_VERSION", ), ], ) @@ -50,19 +50,19 @@ def test_substitution_with_new_and_legacy_identifiers(): - SYM_SERVER_HOSTNAME=%%container_name.sym-server%% - APP_HOSTNAME=%%container_name.dsistudio-app%% - APP_HOSTNAME=some-prefix_%service_uuid% - - MY_LICENSE_FILE=${OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_FILE} - - MY_PRODUCT=$OSPARC_ENVIRONMENT_CURRENT_PRODUCT - - MY_EMAIL=$OSPARC_ENVIRONMENT_USER_EMAIL + - MY_LICENSE_FILE=${OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_FILE} + - MY_PRODUCT=$OSPARC_VARIABLE_CURRENT_PRODUCT + - MY_EMAIL=$OSPARC_VARIABLE_USER_EMAIL - AS_VOILA=1 - DISPLAY1=$${KEEP_SINCE_IT_USES_DOLLAR_ESCAPE_SIGN} - DISPLAY2=${KEEP_SINCE_IT_WAS_EXCLUDED_FROM_SUBSTITUTIONS} containers-allowed-outgoing-permit-list: s4l-core: - - hostname: $OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_SERVER_HOST - tcp_ports: [$OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_SERVER_PRIMARY_PORT, $OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_SERVER_SECONDARY_PORT] + - hostname: $OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_HOST + tcp_ports: [$OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_PRIMARY_PORT, $OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_SECONDARY_PORT] dns_resolver: - address: $OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_DNS_RESOLVER_IP - port: $OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_DNS_RESOLVER_PORT + address: $OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_DNS_RESOLVER_IP + port: $OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_DNS_RESOLVER_PORT containers-allowed-outgoing-internet: - s4l-core-stream """ @@ -77,19 +77,19 @@ def test_substitution_with_new_and_legacy_identifiers(): "SIMCORE_REGISTRY", "SERVICE_VERSION", # NOTE: these identifier names were upgraded from legacy - "OSPARC_ENVIRONMENT_CONTAINER_NAME_SYM_SERVER", - "OSPARC_ENVIRONMENT_CONTAINER_NAME_DSISTUDIO_APP", - "OSPARC_ENVIRONMENT_SERVICE_UUID", + "OSPARC_VARIABLE_CONTAINER_NAME_SYM_SERVER", + "OSPARC_VARIABLE_CONTAINER_NAME_DSISTUDIO_APP", + "OSPARC_VARIABLE_SERVICE_UUID", # ----- - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_FILE", - "OSPARC_ENVIRONMENT_CURRENT_PRODUCT", - "OSPARC_ENVIRONMENT_USER_EMAIL", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_FILE", + "OSPARC_VARIABLE_CURRENT_PRODUCT", + "OSPARC_VARIABLE_USER_EMAIL", "KEEP_SINCE_IT_WAS_EXCLUDED_FROM_SUBSTITUTIONS", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_SERVER_HOST", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_SERVER_PRIMARY_PORT", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_SERVER_SECONDARY_PORT", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_DNS_RESOLVER_IP", - "OSPARC_ENVIRONMENT_VENDOR_SECRET_LICENSE_DNS_RESOLVER_PORT", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_HOST", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_PRIMARY_PORT", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_SECONDARY_PORT", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_DNS_RESOLVER_IP", + "OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_DNS_RESOLVER_PORT", ] # prepare substitutions map {id: value, ...} @@ -144,14 +144,14 @@ def test_substitution_with_new_and_legacy_identifiers(): # Some fo the supported identifiers KNOWN_IDENTIFIERS = { "DISPLAY", # NOTE: this might be a mistake! - "OSPARC_ENVIRONMENT_CONTAINER_NAME_DSISTUDIO_APP", - "OSPARC_ENVIRONMENT_CONTAINER_NAME_FSL_APP", - "OSPARC_ENVIRONMENT_CONTAINER_NAME_ISEG_APP", - "OSPARC_ENVIRONMENT_CONTAINER_NAME_S4L_CORE", - "OSPARC_ENVIRONMENT_CONTAINER_NAME_SCT_LABEL_UTILS_APP", - "OSPARC_ENVIRONMENT_CONTAINER_NAME_SPINAL_CORD_TOOLBOX_APP", - "OSPARC_ENVIRONMENT_CONTAINER_NAME_SYM_SERVER", - "OSPARC_ENVIRONMENT_SERVICE_UUID", + "OSPARC_VARIABLE_CONTAINER_NAME_DSISTUDIO_APP", + "OSPARC_VARIABLE_CONTAINER_NAME_FSL_APP", + "OSPARC_VARIABLE_CONTAINER_NAME_ISEG_APP", + "OSPARC_VARIABLE_CONTAINER_NAME_S4L_CORE", + "OSPARC_VARIABLE_CONTAINER_NAME_SCT_LABEL_UTILS_APP", + "OSPARC_VARIABLE_CONTAINER_NAME_SPINAL_CORD_TOOLBOX_APP", + "OSPARC_VARIABLE_CONTAINER_NAME_SYM_SERVER", + "OSPARC_VARIABLE_SERVICE_UUID", "SERVICE_VERSION", "SIMCORE_REGISTRY", } diff --git a/packages/postgres-database/src/simcore_postgres_database/models/services_environments.py b/packages/postgres-database/src/simcore_postgres_database/models/services_environments.py index dd4c568612b..c7d82f5b519 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/services_environments.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/services_environments.py @@ -7,7 +7,7 @@ from .base import metadata # Intentionally includes the term "SECRET" to avoid leaking this value on a public domain -VENDOR_SECRET_PREFIX: Final[str] = "OSPARC_ENVIRONMENT_VENDOR_SECRET_" +VENDOR_SECRET_PREFIX: Final[str] = "OSPARC_VARIABLE_VENDOR_SECRET_" services_vendor_secrets = sa.Table( @@ -16,7 +16,7 @@ # - A secret is an environment value passed to the service at runtime # - A vendor can associate secrets (e.g. a license code) to any of the services it owns # - secrets_map - # - keys should be prefixed with OSPARC_ENVIRONMENT_VENDOR_SECRET_ (can still normalize on read) + # - keys should be prefixed with OSPARC_VARIABLE_VENDOR_SECRET_ (can still normalize on read) # - values might be encrypted # metadata, @@ -35,7 +35,7 @@ JSONB, nullable=False, server_default=sa.text("'{}'::jsonb"), - doc="Maps OSPARC_ENVIRONMENT_VENDOR_SECRET_* identifiers to a secret value (could be encrypted) " + doc="Maps OSPARC_VARIABLE_VENDOR_SECRET_* identifiers to a secret value (could be encrypted) " "that can be replaced at runtime if found in the compose-specs", ), # TIME STAMPS ---- diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 49c6ac57133..38d3ca88141 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -25,7 +25,7 @@ dynamic_services, dynamic_sidecar, node_rights, - oenvs_substitutions, + osparc_variables_substitutions, rabbitmq, remote_debug, storage, @@ -132,7 +132,7 @@ def init_app(settings: AppSettings | None = None) -> FastAPI: settings = app.state.settings assert settings # nosec - oenvs_substitutions.setup(app) + osparc_variables_substitutions.setup(app) if settings.SC_BOOT_MODE == BootModeEnum.DEBUG: remote_debug.setup(app) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/oenvs_substitutions.py b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py similarity index 62% rename from services/director-v2/src/simcore_service_director_v2/modules/oenvs_substitutions.py rename to services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py index 696af98025e..15438579c6b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/oenvs_substitutions.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py @@ -1,6 +1,10 @@ +""" Substitution of osparc variables and secrets + +""" import logging +from collections.abc import Callable, Mapping from copy import deepcopy -from typing import Any, Callable, Mapping +from typing import Any from fastapi import FastAPI from models_library.projects import ProjectID @@ -11,25 +15,24 @@ from pydantic import EmailStr from ..utils.db import get_repository -from ..utils.session_oenvs import ( +from ..utils.osparc_variables import ( ContextDict, - SessionEnvironmentsTable, - resolve_session_environments, + OsparcVariablesTable, + resolve_variables_from_context, ) from .db.repositories.services_environments import ServicesEnvironmentsRepository _logger = logging.getLogger(__name__) -async def substitute_vendor_secrets_oenvs( +async def substitute_vendor_secrets_in_specs( app: FastAPI, specs: dict[str, Any], + *, service_key: ServiceKey, service_version: ServiceVersion, ) -> dict[str, Any]: assert specs # nosec - new_specs: dict[str, Any] - resolver = SpecsSubstitutionsResolver(specs, upgrade=False) repo = get_repository(app, ServicesEnvironmentsRepository) @@ -41,34 +44,33 @@ async def substitute_vendor_secrets_oenvs( # resolve substitutions resolver.set_substitutions(environs=vendor_secrets) - new_specs = resolver.run() + new_specs: dict[str, Any] = resolver.run() return new_specs return deepcopy(specs) -async def substitute_session_oenvs( +async def resolve_and_substitute_session_variables_in_specs( app: FastAPI, specs: dict[str, Any], + *, user_id: UserID, product_name: str, project_id: ProjectID, node_id: NodeID, ) -> dict[str, Any]: - assert specs # nosec - new_specs: dict[str, Any] - table: SessionEnvironmentsTable = app.state.session_environments_table + table: OsparcVariablesTable = app.state.session_variables_table resolver = SpecsSubstitutionsResolver(specs, upgrade=False) if requested := set(resolver.get_identifiers()): - available = set(table.name_keys()) + available = set(table.variables_names()) if identifiers := available.intersection(requested): - environs = await resolve_session_environments( + environs = await resolve_variables_from_context( table.copy(include=identifiers), - session_context=ContextDict( + context=ContextDict( app=app, user_id=user_id, product_name=product_name, @@ -78,18 +80,19 @@ async def substitute_session_oenvs( ) resolver.set_substitutions(environs=environs) - new_specs = resolver.run() - + new_specs: dict[str, Any] = resolver.run() return new_specs + return deepcopy(specs) -async def substitute_lifespan_oenvs( +async def resolve_and_substitute_lifespan_variables_in_specs( _app: FastAPI, _specs: dict[str, Any], + *, _callbacks_registry: Mapping[str, Callable], ): - raise NotImplementedError() + raise NotImplementedError async def _request_user_email(app: FastAPI, user_id: UserID) -> EmailStr: @@ -102,35 +105,35 @@ async def _request_user_role(app: FastAPI, user_id: UserID): return await repo.get_user_role(user_id=user_id) -def _setup_session_oenvs(app: FastAPI): - app.state.session_environments_table = table = SessionEnvironmentsTable() +def _setup_session_osparc_variables(app: FastAPI): + app.state.session_variables_table = table = OsparcVariablesTable() - # Registers some session oenvs + # Registers some session osparc_variables # WARNING: context_name needs to match session_context! for name, context_name in [ - ("OSPARC_ENVIRONMENT_PRODUCT_NAME", "product_name"), - ("OSPARC_ENVIRONMENT_STUDY_UUID", "project_id"), - ("OSPARC_ENVIRONMENT_NODE_ID", "node_id"), + ("OSPARC_VARIABLE_PRODUCT_NAME", "product_name"), + ("OSPARC_VARIABLE_STUDY_UUID", "project_id"), + ("OSPARC_VARIABLE_NODE_ID", "node_id"), ]: table.register_from_context(name, context_name) - table.register_from_handler("OSPARC_ENVIRONMENT_USER_EMAIL")(_request_user_email) - table.register_from_handler("OSPARC_ENVIRONMENT_USER_ROLE")(_request_user_role) + table.register_from_handler("OSPARC_VARIABLE_USER_EMAIL")(_request_user_email) + table.register_from_handler("OSPARC_VARIABLE_USER_ROLE")(_request_user_role) _logger.debug( - "Registered session_environments_table=%s", sorted(list(table.name_keys())) + "Registered session_variables_table=%s", sorted(table.variables_names()) ) def setup(app: FastAPI): """ - **osparc-environments** (*oenvs* in short) are identifiers-value maps that are substituted on the service specs (e.g. docker-compose). + **o2sparc variables and secrets** are identifiers-value maps that are substituted on the service specs (e.g. docker-compose). - **vendor secrets**: information set by a vendor on the platform. e.g. a vendor service license - - **session oenvs**: some session information as "current user email" or the "current product name" - - **lifespan oenvs**: produced before a service is started and cleaned up after it finishes (e.g. API tokens ) + - **session variables**: some session information as "current user email" or the "current product name" + - **lifespan variables**: produced before a service is started and cleaned up after it finishes (e.g. API tokens ) """ def on_startup() -> None: - _setup_session_oenvs(app) + _setup_session_osparc_variables(app) app.add_event_handler("startup", on_startup) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/session_oenvs.py b/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py similarity index 76% rename from services/director-v2/src/simcore_service_director_v2/utils/session_oenvs.py rename to services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py index dfcdb72746b..918a47aad73 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/session_oenvs.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py @@ -1,6 +1,7 @@ import asyncio import inspect -from typing import Any, Callable, Final, NamedTuple, TypeAlias +from collections.abc import Callable +from typing import Any, Final, NamedTuple, TypeAlias from models_library.utils.specs_substitution import SubstitutionValue from pydantic import NonNegativeInt, parse_obj_as @@ -23,9 +24,8 @@ def _get_or_raise(context: ContextDict) -> Any: try: return context[parameter_name] except KeyError as err: - raise CaptureError( - "Parameter {keyname} missing from substitution context" - ) from err + msg = "Parameter {keyname} missing from substitution context" + raise CaptureError(msg) from err # For context["foo"] -> return operator.methodcaller("__getitem__", keyname) # For context.foo -> return operator.attrgetter("project_id") @@ -51,15 +51,15 @@ def _create(context: ContextDict): return _create -class SessionEnvironmentsTable: +class OsparcVariablesTable: def __init__(self): - self._oenv_getters: dict[str, ContextGetter] = {} + self._variables_getters: dict[str, ContextGetter] = {} def register(self, table: dict[str, Callable]): assert all( # nosec - name.startswith("OSPARC_ENVIRONMENT_") for name in table.keys() + name.startswith("OSPARC_VARIABLE_") for name in table ) # nosec - self._oenv_getters.update(table) + self._variables_getters.update(table) def register_from_context(self, name: str, context_name: str): self.register({name: factory_context_getter(context_name)}) @@ -71,13 +71,13 @@ def _decorator(coro: Callable): return _decorator - def name_keys(self): - return self._oenv_getters.keys() + def variables_names(self): + return self._variables_getters.keys() def copy( self, include: set[str] | None = None, exclude: set[str] | None = None ) -> dict[str, ContextGetter]: - all_ = set(self._oenv_getters.keys()) + all_ = set(self._variables_getters.keys()) exclude = exclude or set() include = include or all_ @@ -85,20 +85,20 @@ def copy( assert include.issubset(all_) # nosec selection = include.difference(exclude) - return {k: self._oenv_getters[k] for k in selection} + return {k: self._variables_getters[k] for k in selection} _HANDLERS_TIMEOUT: Final[NonNegativeInt] = parse_obj_as(NonNegativeInt, 4) -async def resolve_session_environments( - oenvs_getters: dict[str, ContextGetter], - session_context: ContextDict, +async def resolve_variables_from_context( + variables_getters: dict[str, ContextGetter], + context: ContextDict, ) -> dict[str, SubstitutionValue]: # evaluate getters from context values pre_environs: dict[str, SubstitutionValue | RequestTuple] = { - key: fun(session_context) for key, fun in oenvs_getters.items() + key: fun(context) for key, fun in variables_getters.items() } environs: dict[str, SubstitutionValue] = {} @@ -115,8 +115,8 @@ async def resolve_session_environments( # evaluates handlers values = await asyncio.gather(*coros.values()) - for key, value in zip(coros.keys(), values): + for key, value in zip(coros.keys(), values, strict=True): environs[key] = value - assert set(environs.keys()) == set(oenvs_getters.keys()) # nosec + assert set(environs.keys()) == set(variables_getters.keys()) # nosec return environs diff --git a/services/director-v2/tests/unit/test_utils_session_oenvs.py b/services/director-v2/tests/unit/test_utils_osparc_variables.py similarity index 58% rename from services/director-v2/tests/unit/test_utils_session_oenvs.py rename to services/director-v2/tests/unit/test_utils_osparc_variables.py index 1bf21c56a43..84bf7ce2e40 100644 --- a/services/director-v2/tests/unit/test_utils_session_oenvs.py +++ b/services/director-v2/tests/unit/test_utils_osparc_variables.py @@ -17,17 +17,17 @@ from pydantic import parse_obj_as from pytest_simcore.helpers.faker_compose_specs import generate_fake_docker_compose from simcore_postgres_database.models.users import UserRole -from simcore_service_director_v2.modules.oenvs_substitutions import ( - substitute_lifespan_oenvs, - substitute_session_oenvs, - substitute_vendor_secrets_oenvs, +from simcore_service_director_v2.modules.osparc_variables_substitutions import ( + resolve_and_substitute_lifespan_variables_in_specs, + resolve_and_substitute_session_variables_in_specs, + substitute_vendor_secrets_in_specs, ) -from simcore_service_director_v2.utils.session_oenvs import ( +from simcore_service_director_v2.utils.osparc_variables import ( ContextDict, - SessionEnvironmentsTable, + OsparcVariablesTable, factory_context_getter, factory_handler, - resolve_session_environments, + resolve_variables_from_context, ) @@ -45,11 +45,11 @@ def session_context(faker: Faker) -> ContextDict: ) -@pytest.mark.acceptance_test +@pytest.mark.acceptance_test() async def test_resolve_session_environs(faker: Faker, session_context: ContextDict): - assert substitute_session_oenvs - assert substitute_vendor_secrets_oenvs - assert substitute_lifespan_oenvs + assert resolve_and_substitute_session_variables_in_specs + assert substitute_vendor_secrets_in_specs + assert resolve_and_substitute_lifespan_variables_in_specs async def _request_user_role(app: FastAPI, user_id: UserID) -> SubstitutionValue: print(app, user_id) @@ -57,22 +57,22 @@ async def _request_user_role(app: FastAPI, user_id: UserID) -> SubstitutionValue return faker.random_element(elements=list(UserRole)).value # REGISTRATION ----- - oenvs_table = SessionEnvironmentsTable() + osparc_variables_table = OsparcVariablesTable() # bulk registration - oenvs_table.register( + osparc_variables_table.register( { - "OSPARC_ENVIRONMENT_PRODUCT_NAME": factory_context_getter("product_name"), - "OSPARC_ENVIRONMENT_STUDY_UUID": factory_context_getter("project_id"), - "OSPARC_ENVIRONMENT_USER_ROLE": factory_handler(_request_user_role), + "OSPARC_VARIABLE_PRODUCT_NAME": factory_context_getter("product_name"), + "OSPARC_VARIABLE_STUDY_UUID": factory_context_getter("project_id"), + "OSPARC_VARIABLE_USER_ROLE": factory_handler(_request_user_role), } ) # single entry - oenvs_table.register_from_context("OSPARC_ENVIRONMENT_NODE_UUID", "node_id") + osparc_variables_table.register_from_context("OSPARC_VARIABLE_NODE_UUID", "node_id") # using decorator - @oenvs_table.register_from_handler("OSPARC_ENVIRONMENT_USER_EMAIL") + @osparc_variables_table.register_from_handler("OSPARC_VARIABLE_USER_EMAIL") async def request_user_email(app: FastAPI, user_id: UserID) -> SubstitutionValue: print(app, user_id) await asyncio.sleep(1) @@ -84,20 +84,22 @@ async def request_user_email(app: FastAPI, user_id: UserID) -> SubstitutionValue # TODO: test validation errors handling # TODO: test timeout error handling - environs = await resolve_session_environments(oenvs_table.copy(), session_context) + environs = await resolve_variables_from_context( + osparc_variables_table.copy(), session_context + ) - assert set(environs.keys()) == set(oenvs_table.name_keys()) + assert set(environs.keys()) == set(osparc_variables_table.variables_names()) # All values extracted from the context MUST be SubstitutionValue assert { key: parse_obj_as(SubstitutionValue, value) for key, value in environs.items() } - for oenv_name, context_name in [ - ("OSPARC_ENVIRONMENT_PRODUCT_NAME", "product_name"), - ("OSPARC_ENVIRONMENT_STUDY_UUID", "project_id"), - ("OSPARC_ENVIRONMENT_NODE_UUID", "node_id"), + for osparc_variable_name, context_name in [ + ("OSPARC_VARIABLE_PRODUCT_NAME", "product_name"), + ("OSPARC_VARIABLE_STUDY_UUID", "project_id"), + ("OSPARC_VARIABLE_NODE_UUID", "node_id"), ]: - assert environs[oenv_name] == session_context[context_name] + assert environs[osparc_variable_name] == session_context[context_name] print(json.dumps(environs, indent=1))