diff --git a/.env-devel b/.env-devel index e367f18c24a..0accea33cd6 100644 --- a/.env-devel +++ b/.env-devel @@ -119,7 +119,6 @@ DIRECTOR_V2_LOGLEVEL=INFO DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH=null DIRECTOR_V2_PORT=8000 DIRECTOR_V2_PROFILING=1 -DIRECTOR_V2_PUBLIC_API_BASE_URL=http://127.0.0.1:8006 DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS=[] DIRECTOR_V2_DOCKER_HUB_REGISTRY=null DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS=False diff --git a/packages/common-library/src/common_library/network.py b/packages/common-library/src/common_library/network.py new file mode 100644 index 00000000000..2842434460e --- /dev/null +++ b/packages/common-library/src/common_library/network.py @@ -0,0 +1,9 @@ +import ipaddress + + +def is_ip_address(host: str) -> bool: + try: + ipaddress.ip_address(host) + return True + except ValueError: + return False diff --git a/packages/common-library/src/common_library/test_network.py b/packages/common-library/src/common_library/test_network.py new file mode 100644 index 00000000000..b7f423df3a1 --- /dev/null +++ b/packages/common-library/src/common_library/test_network.py @@ -0,0 +1,19 @@ +import pytest +from common_library.network import is_ip_address + + +@pytest.mark.parametrize( + "host, expected", + [ + ("127.0.0.1", True), + ("::1", True), + ("192.168.1.1", True), + ("2001:0db8:85a3:0000:0000:8a2e:0370:7334", True), + ("256.256.256.256", False), + ("invalid_host", False), + ("", False), + ("1234:5678:9abc:def0:1234:5678:9abc:defg", False), + ], +) +def test_is_ip_address(host: str, expected: bool): + assert is_ip_address(host) == expected diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/computations.py b/packages/models-library/src/models_library/api_schemas_directorv2/computations.py index 631b220e1cf..3691fdbf6ee 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/computations.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/computations.py @@ -44,6 +44,10 @@ class ComputationCreate(BaseModel): Field(description="if True the computation pipeline will start right away"), ] = False product_name: Annotated[str, Field()] + product_api_base_url: Annotated[ + AnyHttpUrl, + Field(description="Base url of the product"), + ] subgraph: Annotated[ list[NodeID] | None, Field( diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py index d26acac0490..565580b84bd 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py @@ -1,6 +1,7 @@ -from typing import TypeAlias +from typing import Annotated, TypeAlias -from pydantic import BaseModel, ByteSize, ConfigDict, Field +from pydantic import AnyHttpUrl, BaseModel, BeforeValidator, ByteSize, ConfigDict, Field +from pydantic.config import JsonDict from ..resource_tracker import HardwareInfo, PricingInfo from ..services import ServicePortKey @@ -38,39 +39,61 @@ def from_transferred_bytes( class DynamicServiceCreate(ServiceDetails): service_resources: ServiceResourcesDict - product_name: str = Field(..., description="Current product name") - can_save: bool = Field( - ..., description="the service data must be saved when closing" - ) - wallet_info: WalletInfo | None = Field( - default=None, - description="contains information about the wallet used to bill the running service", - ) - pricing_info: PricingInfo | None = Field( - default=None, - description="contains pricing information (ex. pricing plan and unit ids)", - ) - hardware_info: HardwareInfo | None = Field( - default=None, - description="contains harware information (ex. aws_ec2_instances)", - ) - model_config = ConfigDict( - json_schema_extra={ - "example": { - "key": "simcore/services/dynamic/3dviewer", - "version": "2.4.5", - "user_id": 234, - "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", - "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "product_name": "osparc", - "can_save": True, - "service_resources": ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] - "wallet_info": WalletInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] - "pricing_info": PricingInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] - "hardware_info": HardwareInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + product_name: Annotated[str, Field(..., description="Current product name")] + product_api_base_url: Annotated[ + str, + BeforeValidator(lambda v: f"{AnyHttpUrl(v)}"), + Field(..., description="Current product API base URL"), + ] + can_save: Annotated[ + bool, Field(..., description="the service data must be saved when closing") + ] + wallet_info: Annotated[ + WalletInfo | None, + Field( + default=None, + description="contains information about the wallet used to bill the running service", + ), + ] + pricing_info: Annotated[ + PricingInfo | None, + Field( + default=None, + description="contains pricing information (ex. pricing plan and unit ids)", + ), + ] + hardware_info: Annotated[ + HardwareInfo | None, + Field( + default=None, + description="contains hardware information (ex. aws_ec2_instances)", + ), + ] + + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "example": { + "key": "simcore/services/dynamic/3dviewer", + "version": "2.4.5", + "user_id": 234, + "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", + "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "product_name": "osparc", + "product_api_base_url": "https://api.local/", + "can_save": True, + "service_resources": ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "wallet_info": WalletInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "pricing_info": PricingInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "hardware_info": HardwareInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + } } - } + ) + + model_config = ConfigDict( + json_schema_extra=_update_json_schema_extra, ) diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py index 47c4fc69a18..c8324f0bca0 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py @@ -6,6 +6,7 @@ from models_library.users import UserID from models_library.wallets import WalletInfo from pydantic import BaseModel, ConfigDict +from pydantic.config import JsonDict class DynamicServiceStart(DynamicServiceCreate): @@ -13,26 +14,31 @@ class DynamicServiceStart(DynamicServiceCreate): request_scheme: str simcore_user_agent: str - model_config = ConfigDict( - json_schema_extra={ - "example": { - "product_name": "osparc", - "can_save": True, - "user_id": 234, - "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", - "service_key": "simcore/services/dynamic/3dviewer", - "service_version": "2.4.5", - "service_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "request_dns": "some.local", - "request_scheme": "http", - "simcore_user_agent": "", - "service_resources": ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] - "wallet_info": WalletInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] - "pricing_info": PricingInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] - "hardware_info": HardwareInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "example": { + "product_name": "osparc", + "product_api_base_url": "https://api.local", + "can_save": True, + "user_id": 234, + "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", + "service_key": "simcore/services/dynamic/3dviewer", + "service_version": "2.4.5", + "service_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "request_dns": "some.local", + "request_scheme": "http", + "simcore_user_agent": "", + "service_resources": ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "wallet_info": WalletInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "pricing_info": PricingInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "hardware_info": HardwareInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + } } - } - ) + ) + + model_config = ConfigDict(json_schema_extra=_update_json_schema_extra) class DynamicServiceStop(BaseModel): diff --git a/packages/models-library/src/models_library/api_schemas_webserver/auth.py b/packages/models-library/src/models_library/api_schemas_webserver/auth.py index 6fa33b3fdc4..697867d93b8 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/auth.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/auth.py @@ -55,10 +55,13 @@ class UnregisterCheck(InputSchema): class ApiKeyCreateRequest(InputSchema): display_name: Annotated[str, Field(..., min_length=3)] - expiration: timedelta | None = Field( - None, - description="Time delta from creation time to expiration. If None, then it does not expire.", - ) + expiration: Annotated[ + timedelta | None, + Field( + None, + description="Time delta from creation time to expiration. If None, then it does not expire.", + ), + ] model_config = ConfigDict( alias_generator=AliasGenerator( @@ -86,11 +89,14 @@ class ApiKeyCreateRequest(InputSchema): class ApiKeyCreateResponse(OutputSchema): id: IDStr display_name: Annotated[str, Field(..., min_length=3)] - expiration: timedelta | None = Field( - None, - description="Time delta from creation time to expiration. If None, then it does not expire.", - ) - api_base_url: HttpUrl + expiration: Annotated[ + timedelta | None, + Field( + None, + description="Time delta from creation time to expiration. If None, then it does not expire.", + ), + ] + api_base_url: HttpUrl | None = None api_key: str api_secret: str diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel index 02493e4c86a..33425caf303 100644 --- a/services/director-v2/.env-devel +++ b/services/director-v2/.env-devel @@ -25,8 +25,6 @@ DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_ID=1234 DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_NAME=1234 DIRECTOR_V2_SELF_SIGNED_SSL_FILENAME=filename -DIRECTOR_V2_PUBLIC_API_BASE_URL=http://127.0.0.1:8006 - DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}' LOG_LEVEL=DEBUG diff --git a/services/director-v2/openapi.json b/services/director-v2/openapi.json index 856d5d29721..144d9f92bed 100644 --- a/services/director-v2/openapi.json +++ b/services/director-v2/openapi.json @@ -1291,6 +1291,13 @@ "type": "string", "title": "Product Name" }, + "product_api_base_url": { + "type": "string", + "minLength": 1, + "format": "uri", + "title": "Product Api Base Url", + "description": "Base url of the product" + }, "subgraph": { "anyOf": [ { @@ -1347,7 +1354,8 @@ "required": [ "user_id", "project_id", - "product_name" + "product_name", + "product_api_base_url" ], "title": "ComputationCreate" }, @@ -1789,6 +1797,11 @@ "title": "Product Name", "description": "Current product name" }, + "product_api_base_url": { + "type": "string", + "title": "Product Api Base Url", + "description": "Current product API base URL" + }, "can_save": { "type": "boolean", "title": "Can Save", @@ -1825,7 +1838,7 @@ "type": "null" } ], - "description": "contains harware information (ex. aws_ec2_instances)" + "description": "contains hardware information (ex. aws_ec2_instances)" } }, "type": "object", @@ -1837,6 +1850,7 @@ "service_uuid", "service_resources", "product_name", + "product_api_base_url", "can_save" ], "title": "DynamicServiceCreate", @@ -1855,6 +1869,7 @@ "pricing_unit_cost_id": 1, "pricing_unit_id": 1 }, + "product_api_base_url": "https://api.local/", "product_name": "osparc", "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", "service_resources": { @@ -3002,6 +3017,18 @@ ], "title": "Product Name", "description": "Current product upon which this service is scheduledIf set to None, the current product is undefined. Mostly for backwards compatibility" + }, + "product_api_base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Product Api Base Url", + "description": "Base URL for the current product's API." } }, "additionalProperties": true, diff --git a/services/director-v2/src/simcore_service_director_v2/constants.py b/services/director-v2/src/simcore_service_director_v2/constants.py index 194425d0328..d4a5690d9bb 100644 --- a/services/director-v2/src/simcore_service_director_v2/constants.py +++ b/services/director-v2/src/simcore_service_director_v2/constants.py @@ -24,3 +24,4 @@ UNDEFINED_STR_METADATA = "undefined-metadata" UNDEFINED_DOCKER_LABEL = "undefined-label" +UNDEFINED_API_BASE_URL = "https://api.local" diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index 66bc9857c02..03f256b01b0 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -16,9 +16,7 @@ NoAuthentication, ) from pydantic import ( - AfterValidator, AliasChoices, - AnyHttpUrl, AnyUrl, Field, NonNegativeInt, @@ -233,14 +231,6 @@ class AppSettings(BaseApplicationSettings, MixinLoggingSettings): description="resource usage tracker service client's plugin", ) - DIRECTOR_V2_PUBLIC_API_BASE_URL: Annotated[ - AnyHttpUrl | str, - AfterValidator(lambda v: f"{v}".rstrip("/")), - Field( - ..., - description="Base URL used to access the public api e.g. http://127.0.0.1:6000 for development or https://api.osparc.io", - ), - ] DIRECTOR_V2_TRACING: TracingSettings | None = Field( json_schema_extra={"auto_default_from_env": True}, description="settings for opentelemetry tracing", diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py index 6f0f7bf7986..ab17131186d 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py @@ -30,6 +30,7 @@ class RunMetadataDict(TypedDict, total=False): node_id_names_map: dict[NodeID, str] project_name: str product_name: str + product_api_base_url: str simcore_user_agent: str user_email: str wallet_id: int | None diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 39dfda3bc98..d0888dd1acf 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -35,6 +35,7 @@ from pydantic import ( AnyHttpUrl, BaseModel, + BeforeValidator, ConfigDict, Field, StringConstraints, @@ -475,6 +476,14 @@ def get_proxy_endpoint(self) -> AnyHttpUrl: ), ] = None + product_api_base_url: Annotated[ + str | None, + BeforeValidator(lambda v: f"{AnyHttpUrl(v)}"), + Field( + description="Base URL for the current product's API.", + ), + ] = None + @classmethod def from_http_request( # pylint: disable=too-many-arguments @@ -502,6 +511,7 @@ def from_http_request( "version": service.version, "service_resources": service.service_resources, "product_name": service.product_name, + "product_api_base_url": service.product_api_base_url, "paths_mapping": simcore_service_labels.paths_mapping, "callbacks_mapping": simcore_service_labels.callbacks_mapping, "compose_spec": json_dumps(simcore_service_labels.compose_spec), diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py index f72287b0082..7ed0736d3cd 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py @@ -277,6 +277,7 @@ async def assemble_spec( # pylint: disable=too-many-arguments # noqa: PLR0913 simcore_service_labels: SimcoreServiceLabels, allow_internet_access: bool, product_name: ProductName, + product_api_base_url: str, user_id: UserID, project_id: ProjectID, node_id: NodeID, @@ -354,6 +355,7 @@ async def assemble_spec( # pylint: disable=too-many-arguments # noqa: PLR0913 safe=False, user_id=user_id, product_name=product_name, + product_api_base_url=product_api_base_url, project_id=project_id, node_id=node_id, service_run_id=service_run_id, @@ -394,6 +396,7 @@ async def assemble_spec( # pylint: disable=too-many-arguments # noqa: PLR0913 user_id=user_id, safe=True, product_name=product_name, + product_api_base_url=product_api_base_url, project_id=project_id, node_id=node_id, service_run_id=service_run_id, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py index 4f41391c638..d7dd034134b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py @@ -76,7 +76,7 @@ async def submit_compose_sepc(app: FastAPI, scheduler_data: SchedulerData) -> No allow_internet_access: bool = await groups_extra_properties.has_internet_access( user_id=scheduler_data.user_id, product_name=scheduler_data.product_name ) - + assert scheduler_data.product_api_base_url is not None # nosec dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings = ( app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER ) @@ -95,6 +95,7 @@ async def submit_compose_sepc(app: FastAPI, scheduler_data: SchedulerData) -> No simcore_service_labels=simcore_service_labels, allow_internet_access=allow_internet_access, product_name=scheduler_data.product_name, + product_api_base_url=scheduler_data.product_api_base_url, user_id=scheduler_data.user_id, project_id=scheduler_data.project_id, node_id=scheduler_data.node_uuid, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables/substitutions.py b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables/substitutions.py index 00fc4c5ba0e..0d458749ba3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables/substitutions.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables/substitutions.py @@ -23,7 +23,6 @@ from pydantic import BaseModel from servicelib.fastapi.app_state import SingletonInAppStateMixin from servicelib.logging_utils import log_context -from simcore_service_director_v2.core.settings import get_application_settings from ...utils.db import get_repository from ...utils.osparc_variables import ( @@ -180,6 +179,7 @@ async def resolve_and_substitute_session_variables_in_model( safe: bool = True, user_id: UserID, product_name: str, + product_api_base_url: str | None, project_id: ProjectID, node_id: NodeID, service_run_id: ServiceRunID, @@ -194,7 +194,6 @@ async def resolve_and_substitute_session_variables_in_model( # if it raises an error vars need replacement raise_if_unresolved_osparc_variable_identifier_found(model) except UnresolvedOsparcVariableIdentifierError: - app_settings = get_application_settings(app) table = OsparcSessionVariablesTable.get_from_app_state(app) identifiers = await resolve_variables_from_context( table.copy(), @@ -206,7 +205,7 @@ async def resolve_and_substitute_session_variables_in_model( node_id=node_id, run_id=service_run_id, wallet_id=wallet_id, - api_server_base_url=app_settings.DIRECTOR_V2_PUBLIC_API_BASE_URL, + api_server_base_url=product_api_base_url, ), ) _logger.debug("replacing with the identifiers=%s", identifiers) @@ -225,6 +224,7 @@ async def resolve_and_substitute_session_variables_in_specs( safe: bool = True, user_id: UserID, product_name: str, + product_api_base_url: str, project_id: ProjectID, node_id: NodeID, service_run_id: ServiceRunID, @@ -241,7 +241,6 @@ async def resolve_and_substitute_session_variables_in_specs( identifiers_to_replace, ) if identifiers_to_replace: - app_settings = get_application_settings(app) environs = await resolve_variables_from_context( table.copy(include=identifiers_to_replace), context=ContextDict( @@ -252,7 +251,7 @@ async def resolve_and_substitute_session_variables_in_specs( node_id=node_id, run_id=service_run_id, wallet_id=wallet_id, - api_server_base_url=app_settings.DIRECTOR_V2_PUBLIC_API_BASE_URL, + api_server_base_url=product_api_base_url, ), ) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index 18e67f2dfc1..e774cdb7cc9 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -41,7 +41,7 @@ from simcore_sdk.node_ports_v2.links import ItemValue as _NPItemValue from sqlalchemy.ext.asyncio import AsyncEngine -from ..constants import UNDEFINED_DOCKER_LABEL +from ..constants import UNDEFINED_API_BASE_URL, UNDEFINED_DOCKER_LABEL from ..core.errors import ( ComputationalBackendNotConnectedError, ComputationalSchedulerChangedError, @@ -318,6 +318,7 @@ async def compute_task_envs( wallet_id: WalletID | None, ) -> ContainerEnvsDict: product_name = metadata.get("product_name", UNDEFINED_DOCKER_LABEL) + product_api_base_url = metadata.get("product_api_base_url", UNDEFINED_API_BASE_URL) task_envs = node_image.envs if task_envs: vendor_substituted_envs = await substitute_vendor_secrets_in_specs( @@ -332,6 +333,7 @@ async def compute_task_envs( vendor_substituted_envs, user_id=user_id, product_name=product_name, + product_api_base_url=product_api_base_url, project_id=project_id, node_id=node_id, service_run_id=resource_tracking_run_id, diff --git a/services/director-v2/tests/conftest.py b/services/director-v2/tests/conftest.py index 22a36ee0f5c..8335706ad7b 100644 --- a/services/director-v2/tests/conftest.py +++ b/services/director-v2/tests/conftest.py @@ -171,7 +171,6 @@ def mock_env( "COMPUTATIONAL_BACKEND_ENABLED": "false", "DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED": "false", "DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED": "0", - "DIRECTOR_V2_PUBLIC_API_BASE_URL": "http://127.0.0.1:8006", "DYNAMIC_SIDECAR_IMAGE": f"{dynamic_sidecar_docker_image_name}", "DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS": "{}", "POSTGRES_DB": "test", diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 1c1afe2f5fb..f16977bc1cf 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -150,6 +150,7 @@ def fake_workbench_computational_pipeline_details_not_started( "user_id": "some invalid id", "project_id": "not a uuid", "product_name": "not a product", + "product_api_base_url": "http://invalid", }, status.HTTP_422_UNPROCESSABLE_ENTITY, ), @@ -158,6 +159,7 @@ def fake_workbench_computational_pipeline_details_not_started( "user_id": 2, "project_id": "not a uuid", "product_name": "not a product", + "product_api_base_url": "http://invalid", }, status.HTTP_422_UNPROCESSABLE_ENTITY, ), @@ -166,6 +168,7 @@ def fake_workbench_computational_pipeline_details_not_started( "user_id": 3, "project_id": "16e60a5d-834e-4267-b44d-3af49171bf21", "product_name": "not a product", + "product_api_base_url": "http://invalid", }, status.HTTP_404_NOT_FOUND, ), @@ -191,6 +194,7 @@ async def test_start_empty_computation_is_refused( create_registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], osparc_product_name: str, + osparc_product_api_base_url: str, create_pipeline: Callable[..., Awaitable[ComputationGet]], ): user = create_registered_user() @@ -204,6 +208,7 @@ async def test_start_empty_computation_is_refused( user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) @@ -397,6 +402,7 @@ async def test_run_partial_computation( fake_workbench_without_outputs: dict[str, Any], params: PartialComputationParams, osparc_product_name: str, + osparc_product_api_base_url: str, create_pipeline: Callable[..., Awaitable[ComputationGet]], ): user = create_registered_user() @@ -449,6 +455,7 @@ def _convert_to_pipeline_details( user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, subgraph=[ str(node_id) for index, node_id in enumerate(sleepers_project.workbench) @@ -492,6 +499,7 @@ def _convert_to_pipeline_details( user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, expected_response_status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, subgraph=[ str(node_id) @@ -513,6 +521,7 @@ def _convert_to_pipeline_details( user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, expected_response_status_code=status.HTTP_201_CREATED, subgraph=[ str(node_id) @@ -546,6 +555,7 @@ async def test_run_computation( fake_workbench_computational_pipeline_details: PipelineDetails, fake_workbench_computational_pipeline_details_completed: PipelineDetails, osparc_product_name: str, + osparc_product_api_base_url: str, create_pipeline: Callable[..., Awaitable[ComputationGet]], ): user = create_registered_user() @@ -558,6 +568,7 @@ async def test_run_computation( user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, expected_response_status_code=status.HTTP_201_CREATED, ) @@ -604,6 +615,7 @@ async def test_run_computation( user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) # now force run again @@ -627,6 +639,7 @@ async def test_run_computation( user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, force_restart=True, ) # check the contents is correct @@ -658,6 +671,7 @@ async def test_abort_computation( fake_workbench_without_outputs: dict[str, Any], fake_workbench_computational_pipeline_details: PipelineDetails, osparc_product_name: str, + osparc_product_api_base_url: str, create_pipeline: Callable[..., Awaitable[ComputationGet]], ): user = create_registered_user() @@ -675,6 +689,7 @@ async def test_abort_computation( user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) # check the contents is correctb @@ -736,6 +751,7 @@ async def test_update_and_delete_computation( fake_workbench_computational_pipeline_details_not_started: PipelineDetails, fake_workbench_computational_pipeline_details: PipelineDetails, osparc_product_name: str, + osparc_product_api_base_url: str, create_pipeline: Callable[..., Awaitable[ComputationGet]], ): user = create_registered_user() @@ -747,6 +763,7 @@ async def test_update_and_delete_computation( user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) # check the contents is correctb @@ -765,6 +782,7 @@ async def test_update_and_delete_computation( user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) # check the contents is correctb @@ -783,6 +801,7 @@ async def test_update_and_delete_computation( user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) # check the contents is correctb @@ -801,6 +820,7 @@ async def test_update_and_delete_computation( user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) # check the contents is correctb await assert_computation_task_out_obj( @@ -831,6 +851,7 @@ async def test_update_and_delete_computation( user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) # try to delete the pipeline, is expected to be forbidden if force parameter is false (default) @@ -856,6 +877,7 @@ async def test_pipeline_with_no_computational_services_still_create_correct_comp project: Callable[..., Awaitable[ProjectAtDB]], jupyter_service: dict[str, Any], osparc_product_name: str, + osparc_product_api_base_url: str, create_pipeline: Callable[..., Awaitable[ComputationGet]], ): user = create_registered_user() @@ -881,6 +903,7 @@ async def test_pipeline_with_no_computational_services_still_create_correct_comp user_id=user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) # still this pipeline shall be createable if we do not want to start it @@ -890,6 +913,7 @@ async def test_pipeline_with_no_computational_services_still_create_correct_comp user_id=user["id"], start_pipeline=False, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) @@ -899,6 +923,7 @@ async def test_pipeline_with_control_loop_made_of_dynamic_services_is_allowed( project: Callable[..., Awaitable[ProjectAtDB]], jupyter_service: dict[str, Any], osparc_product_name: str, + osparc_product_api_base_url: str, ): user = create_registered_user() # create a workbench with just 2 dynamic service in a cycle @@ -940,6 +965,7 @@ async def test_pipeline_with_control_loop_made_of_dynamic_services_is_allowed( "project_id": str(project_with_dynamic_node.uuid), "start_pipeline": True, "product_name": osparc_product_name, + "product_api_base_url": osparc_product_api_base_url, }, ) assert ( @@ -954,6 +980,7 @@ async def test_pipeline_with_control_loop_made_of_dynamic_services_is_allowed( "project_id": str(project_with_dynamic_node.uuid), "start_pipeline": False, "product_name": osparc_product_name, + "product_api_base_url": osparc_product_api_base_url, }, ) assert ( @@ -968,6 +995,7 @@ async def test_pipeline_with_cycle_containing_a_computational_service_is_forbidd sleeper_service: dict[str, Any], jupyter_service: dict[str, Any], osparc_product_name: str, + osparc_product_api_base_url: str, ): user = create_registered_user() # create a workbench with just 2 dynamic service in a cycle @@ -1021,6 +1049,7 @@ async def test_pipeline_with_cycle_containing_a_computational_service_is_forbidd "project_id": str(project_with_cycly_and_comp_service.uuid), "start_pipeline": True, "product_name": osparc_product_name, + "product_api_base_url": osparc_product_api_base_url, }, ) assert ( @@ -1035,6 +1064,7 @@ async def test_pipeline_with_cycle_containing_a_computational_service_is_forbidd "project_id": str(project_with_cycly_and_comp_service.uuid), "start_pipeline": False, "product_name": osparc_product_name, + "product_api_base_url": osparc_product_api_base_url, }, ) assert ( @@ -1051,6 +1081,7 @@ async def test_burst_create_computations( fake_workbench_computational_pipeline_details: PipelineDetails, fake_workbench_computational_pipeline_details_completed: PipelineDetails, osparc_product_name: str, + osparc_product_api_base_url: str, create_pipeline: Callable[..., Awaitable[ComputationGet]], ): user = create_registered_user() @@ -1069,6 +1100,7 @@ async def test_burst_create_computations( project=sleepers_project, user_id=user["id"], product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, start_pipeline=False, ) for _ in range(NUMBER_OF_CALLS) @@ -1079,6 +1111,7 @@ async def test_burst_create_computations( project=sleepers_project2, user_id=user["id"], product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, start_pipeline=False, ) ] @@ -1096,6 +1129,7 @@ async def test_burst_create_computations( project=sleepers_project, user_id=user["id"], product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, start_pipeline=True, ) for _ in range(NUMBER_OF_CALLS) @@ -1106,6 +1140,7 @@ async def test_burst_create_computations( project=sleepers_project2, user_id=user["id"], product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, start_pipeline=False, ) ] diff --git a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py index 25aaf0de8ed..e4a5cc39047 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py +++ b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py @@ -117,11 +117,13 @@ def start_request_data( service_resources: ServiceResourcesDict, ensure_swarm_and_networks: None, osparc_product_name: str, + osparc_product_api_base_url: str, ) -> dict[str, Any]: return { "user_id": user_id, "project_id": project_id, "product_name": osparc_product_name, + "product_api_base_url": osparc_product_api_base_url, "service_uuid": node_uuid, "service_key": dy_static_file_server_dynamic_sidecar_service["image"]["name"], "service_version": dy_static_file_server_dynamic_sidecar_service["image"][ diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 85ecce3211e..30563157d6d 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -265,6 +265,7 @@ async def current_study( fake_dy_workbench: dict[str, Any], async_client: httpx.AsyncClient, osparc_product_name: str, + osparc_product_api_base_url: str, create_pipeline: Callable[..., Awaitable[ComputationGet]], ) -> ProjectAtDB: project_at_db = await project(current_user, workbench=fake_dy_workbench) @@ -276,6 +277,7 @@ async def current_study( user_id=current_user["id"], start_pipeline=False, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) return project_at_db @@ -710,6 +712,7 @@ async def _fetch_data_via_aioboto( async def _start_and_wait_for_dynamic_services_ready( director_v2_client: httpx.AsyncClient, product_name: str, + product_api_base_url: str, user_id: UserID, workbench_dynamic_services: dict[str, Node], current_study: ProjectAtDB, @@ -721,6 +724,7 @@ async def _start_and_wait_for_dynamic_services_ready( assert_start_service( director_v2_client=director_v2_client, product_name=product_name, + product_api_base_url=product_api_base_url, user_id=user_id, project_id=str(current_study.uuid), service_key=node.key, @@ -902,6 +906,7 @@ async def test_nodeports_integration( fake_dy_success: dict[str, Any], tmp_path: Path, osparc_product_name: str, + osparc_product_api_base_url: str, create_pipeline: Callable[..., Awaitable[ComputationGet]], mock_io_log_redirect_cb: LogRedirectCB, faker: Faker, @@ -938,6 +943,7 @@ async def test_nodeports_integration( await _start_and_wait_for_dynamic_services_ready( director_v2_client=async_client, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, user_id=current_user["id"], workbench_dynamic_services=workbench_dynamic_services, current_study=current_study, @@ -952,6 +958,7 @@ async def test_nodeports_integration( user_id=current_user["id"], start_pipeline=True, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, ) # wait for the computation to finish (either by failing, success or abort) @@ -1132,6 +1139,7 @@ async def test_nodeports_integration( await _start_and_wait_for_dynamic_services_ready( director_v2_client=async_client, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, user_id=current_user["id"], workbench_dynamic_services=workbench_dynamic_services, current_study=current_study, diff --git a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py index 0f6a0ec3165..61af3dd5823 100644 --- a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py +++ b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py @@ -240,6 +240,7 @@ async def test_legacy_and_dynamic_sidecar_run( services_endpoint: dict[str, URL], async_client: httpx.AsyncClient, osparc_product_name: str, + osparc_product_api_base_url: str, ensure_services_stopped: None, mock_projects_networks_repository: None, mock_sidecars_client: mock.Mock, @@ -264,6 +265,7 @@ async def test_legacy_and_dynamic_sidecar_run( director_v2_client=async_client, # context product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, user_id=user_dict["id"], project_id=str(dy_static_file_server_project.uuid), # service diff --git a/services/director-v2/tests/integration/02/utils.py b/services/director-v2/tests/integration/02/utils.py index 15c6ea87e22..69c7ca81d4d 100644 --- a/services/director-v2/tests/integration/02/utils.py +++ b/services/director-v2/tests/integration/02/utils.py @@ -328,6 +328,7 @@ async def _handle_redirection( async def assert_start_service( director_v2_client: httpx.AsyncClient, product_name: str, + product_api_base_url: str, user_id: UserID, project_id: str, service_key: str, @@ -353,6 +354,7 @@ async def assert_start_service( service_resources ), "product_name": product_name, + "product_api_base_url": product_api_base_url, } headers = { X_DYNAMIC_SIDECAR_REQUEST_DNS: director_v2_client.base_url.host, diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index 0bacfeafb85..cc4c32899ae 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -67,6 +67,11 @@ def osparc_product_name() -> str: return "osparc" +@pytest.fixture(scope="session") +def osparc_product_api_base_url() -> str: + return "https://api.osparc.io" + + COMPUTATION_URL: str = "v2/computations" @@ -82,6 +87,7 @@ async def _creator( project: ProjectAtDB, user_id: UserID, product_name: str, + product_api_base_url: str, start_pipeline: bool, **kwargs, ) -> ComputationGet: @@ -92,6 +98,7 @@ async def _creator( "project_id": str(project.uuid), "start_pipeline": start_pipeline, "product_name": product_name, + "product_api_base_url": product_api_base_url, **kwargs, }, ) diff --git a/services/director-v2/tests/unit/conftest.py b/services/director-v2/tests/unit/conftest.py index 0a1f327c0ee..eb22767b34a 100644 --- a/services/director-v2/tests/unit/conftest.py +++ b/services/director-v2/tests/unit/conftest.py @@ -76,13 +76,13 @@ def simcore_service_labels() -> SimcoreServiceLabels: @pytest.fixture def dynamic_service_create() -> DynamicServiceCreate: return DynamicServiceCreate.model_validate( - DynamicServiceCreate.model_config["json_schema_extra"]["example"] + DynamicServiceCreate.model_json_schema()["example"] ) @pytest.fixture def dynamic_sidecar_port() -> PortInt: - return PortInt(1222) + return 1222 @pytest.fixture diff --git a/services/director-v2/tests/unit/test_modules_osparc_variables.py b/services/director-v2/tests/unit/test_modules_osparc_variables.py index 606712f3c20..605cb32cb83 100644 --- a/services/director-v2/tests/unit/test_modules_osparc_variables.py +++ b/services/director-v2/tests/unit/test_modules_osparc_variables.py @@ -151,7 +151,6 @@ async def fake_app(faker: Faker) -> AsyncIterable[FastAPI]: app.state.engine = AsyncMock() mock_settings = Mock() - mock_settings.DIRECTOR_V2_PUBLIC_API_BASE_URL = faker.url() app.state.settings = mock_settings substitutions.setup(app) @@ -187,6 +186,7 @@ async def test_resolve_and_substitute_session_variables_in_specs( specs=specs, user_id=1, product_name="a_product", + product_api_base_url=faker.url(), project_id=faker.uuid4(cast_to=None), node_id=faker.uuid4(cast_to=None), service_run_id=ServiceRunID("some_run_id"), @@ -223,8 +223,10 @@ async def test_substitute_vendor_secrets_in_specs( fake_app, specs=specs, product_name="a_product", - service_key=ServiceKey("simcore/services/dynamic/fake"), - service_version=ServiceVersion("0.0.1"), + service_key=TypeAdapter(ServiceKey).validate_python( + "simcore/services/dynamic/fake" + ), + service_version=TypeAdapter(ServiceVersion).validate_python("0.0.1"), ) print("REPLACED SPECS\n", replaced_specs) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py index 5d64fcd8765..633e8cd2a44 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py @@ -368,10 +368,17 @@ def product_name(faker: Faker) -> str: return faker.name() +@pytest.fixture +def product_api_base_url(faker: Faker) -> AnyHttpUrl: + return TypeAdapter(AnyHttpUrl).validate_python(faker.url()) + + async def test_computation_create_validators( create_registered_user: Callable[..., dict[str, Any]], project: Callable[..., Awaitable[ProjectAtDB]], fake_workbench_without_outputs: dict[str, Any], + product_name: str, + product_api_base_url: AnyHttpUrl, faker: Faker, ): user = create_registered_user() @@ -379,13 +386,15 @@ async def test_computation_create_validators( ComputationCreate( user_id=user["id"], project_id=proj.uuid, - product_name=faker.pystr(), + product_name=product_name, + product_api_base_url=product_api_base_url, use_on_demand_clusters=True, ) ComputationCreate( user_id=user["id"], project_id=proj.uuid, - product_name=faker.pystr(), + product_name=product_name, + product_api_base_url=product_api_base_url, use_on_demand_clusters=False, ) @@ -395,6 +404,7 @@ async def test_create_computation( mocked_director_service_fcts: respx.MockRouter, mocked_catalog_service_fcts: respx.MockRouter, product_name: str, + product_api_base_url: AnyHttpUrl, fake_workbench_without_outputs: dict[str, Any], create_registered_user: Callable[..., dict[str, Any]], project: Callable[..., Awaitable[ProjectAtDB]], @@ -407,7 +417,10 @@ async def test_create_computation( create_computation_url, json=jsonable_encoder( ComputationCreate( - user_id=user["id"], project_id=proj.uuid, product_name=product_name + user_id=user["id"], + project_id=proj.uuid, + product_name=product_name, + product_api_base_url=product_api_base_url, ) ), ) @@ -492,6 +505,7 @@ async def test_create_computation_with_wallet( mocked_resource_usage_tracker_service_fcts: respx.MockRouter, mocked_clusters_keeper_service_get_instance_type_details: mock.Mock, product_name: str, + product_api_base_url: AnyHttpUrl, fake_workbench_without_outputs: dict[str, Any], create_registered_user: Callable[..., dict[str, Any]], project: Callable[..., Awaitable[ProjectAtDB]], @@ -521,6 +535,7 @@ async def test_create_computation_with_wallet( user_id=user["id"], project_id=proj.uuid, product_name=product_name, + product_api_base_url=product_api_base_url, wallet_info=wallet_info, ) ), @@ -602,6 +617,7 @@ async def test_create_computation_with_wallet_with_invalid_pricing_unit_name_rai mocked_resource_usage_tracker_service_fcts: respx.MockRouter, mocked_clusters_keeper_service_get_instance_type_details_with_invalid_name: mock.Mock, product_name: str, + product_api_base_url: AnyHttpUrl, fake_workbench_without_outputs: dict[str, Any], create_registered_user: Callable[..., dict[str, Any]], project: Callable[..., Awaitable[ProjectAtDB]], @@ -621,6 +637,7 @@ async def test_create_computation_with_wallet_with_invalid_pricing_unit_name_rai user_id=user["id"], project_id=proj.uuid, product_name=product_name, + product_api_base_url=product_api_base_url, wallet_info=wallet_info, ) ), @@ -643,6 +660,7 @@ async def test_create_computation_with_wallet_with_no_clusters_keeper_raises_503 mocked_catalog_service_fcts: respx.MockRouter, mocked_resource_usage_tracker_service_fcts: respx.MockRouter, product_name: str, + product_api_base_url: AnyHttpUrl, fake_workbench_without_outputs: dict[str, Any], create_registered_user: Callable[..., dict[str, Any]], project: Callable[..., Awaitable[ProjectAtDB]], @@ -659,6 +677,7 @@ async def test_create_computation_with_wallet_with_no_clusters_keeper_raises_503 user_id=user["id"], project_id=proj.uuid, product_name=product_name, + product_api_base_url=product_api_base_url, wallet_info=wallet_info, ) ), @@ -695,6 +714,7 @@ async def test_start_computation( mocked_director_service_fcts: respx.MockRouter, mocked_catalog_service_fcts: respx.MockRouter, product_name: str, + product_api_base_url: AnyHttpUrl, fake_workbench_without_outputs: dict[str, Any], create_registered_user: Callable[..., dict[str, Any]], project: Callable[..., Awaitable[ProjectAtDB]], @@ -711,6 +731,7 @@ async def test_start_computation( project_id=proj.uuid, start_pipeline=True, product_name=product_name, + product_api_base_url=product_api_base_url, ) ), ) @@ -727,6 +748,7 @@ async def test_start_computation_with_project_node_resources_defined( mocked_director_service_fcts: respx.MockRouter, mocked_catalog_service_fcts: respx.MockRouter, product_name: str, + product_api_base_url: AnyHttpUrl, fake_workbench_without_outputs: dict[str, Any], create_registered_user: Callable[..., dict[str, Any]], project: Callable[..., Awaitable[ProjectAtDB]], @@ -758,6 +780,7 @@ async def test_start_computation_with_project_node_resources_defined( project_id=proj.uuid, start_pipeline=True, product_name=product_name, + product_api_base_url=product_api_base_url, ) ), ) @@ -772,6 +795,7 @@ async def test_start_computation_with_deprecated_services_raises_406( mocked_director_service_fcts: respx.MockRouter, mocked_catalog_service_fcts_deprecated: respx.MockRouter, product_name: str, + product_api_base_url: AnyHttpUrl, fake_workbench_without_outputs: dict[str, Any], fake_workbench_adjacency: dict[str, Any], create_registered_user: Callable[..., dict[str, Any]], @@ -789,6 +813,7 @@ async def test_start_computation_with_deprecated_services_raises_406( project_id=proj.uuid, start_pipeline=True, product_name=product_name, + product_api_base_url=product_api_base_url, ) ), ) diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index 6d62f1ca952..8f26a8bd297 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -187,6 +187,7 @@ def expected_dynamic_sidecar_spec( "examples" ][3], "product_name": osparc_product_name, + "product_api_base_url": "https://api.local/", "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", "proxy_service_name": "dy-proxy_75c7f3f4-18f9-4678-8610-54a2ade78eaa", "request_dns": "test-endpoint", @@ -588,12 +589,12 @@ async def test_merge_dynamic_sidecar_specs_with_user_specific_specs( assert dynamic_sidecar_spec_dict == expected_dynamic_sidecar_spec_dict catalog_client = CatalogClient.instance(minimal_app) - user_service_specs: dict[ - str, Any - ] = await catalog_client.get_service_specifications( - scheduler_data.user_id, - mock_service_key_version.key, - mock_service_key_version.version, + user_service_specs: dict[str, Any] = ( + await catalog_client.get_service_specifications( + scheduler_data.user_id, + mock_service_key_version.key, + mock_service_key_version.version, + ) ) assert user_service_specs assert "sidecar" in user_service_specs diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 45756dfdd9d..931891df54e 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -317,7 +317,6 @@ services: DIRECTOR_V2_DEV_FEATURES_ENABLED: ${DIRECTOR_V2_DEV_FEATURES_ENABLED} DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED: ${DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED} - DIRECTOR_V2_PUBLIC_API_BASE_URL: ${DIRECTOR_V2_PUBLIC_API_BASE_URL} DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: ${DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS} DIRECTOR_V2_PROFILING: ${DIRECTOR_V2_PROFILING} DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL: ${DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL} diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py index d4c2ed67b21..c3afea52818 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py @@ -58,6 +58,7 @@ async def post_dynamic_service( ) -> Response: post_data = { "product_name": dynamic_service_start.product_name, + "product_api_base_url": dynamic_service_start.product_api_base_url, "can_save": dynamic_service_start.can_save, "user_id": dynamic_service_start.user_id, "project_id": dynamic_service_start.project_id, diff --git a/services/dynamic-scheduler/tests/unit/conftest.py b/services/dynamic-scheduler/tests/unit/conftest.py index a25596bd4f2..dd59a127201 100644 --- a/services/dynamic-scheduler/tests/unit/conftest.py +++ b/services/dynamic-scheduler/tests/unit/conftest.py @@ -13,9 +13,7 @@ @pytest.fixture def get_dynamic_service_start() -> Callable[[NodeID], DynamicServiceStart]: def _(node_id: NodeID) -> DynamicServiceStart: - dict_data = deepcopy( - DynamicServiceStart.model_config["json_schema_extra"]["example"] - ) + dict_data = deepcopy(DynamicServiceStart.model_json_schema()["example"]) dict_data["service_uuid"] = f"{node_id}" return TypeAdapter(DynamicServiceStart).validate_python(dict_data) diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__models.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__models.py index b07a41ed3fe..92d3b701522 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__models.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__models.py @@ -53,7 +53,7 @@ def test_serialization( [ None, TypeAdapter(DynamicServiceStart).validate_python( - DynamicServiceStart.model_config["json_schema_extra"]["example"] + DynamicServiceStart.model_json_schema()["example"] ), ], ) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 6f96b9a22f5..c7efcd6aa11 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -8368,7 +8368,9 @@ components: description: Time delta from creation time to expiration. If None, then it does not expire. apiBaseUrl: - type: string + anyOf: + - type: string + - type: 'null' title: Apibaseurl apiKey: type: string @@ -8380,7 +8382,6 @@ components: required: - id - displayName - - apiBaseUrl - apiKey - apiSecret title: ApiKeyCreateResponse diff --git a/services/web/server/src/simcore_service_webserver/api_keys/_controller/rest.py b/services/web/server/src/simcore_service_webserver/api_keys/_controller/rest.py index b7db0f0068d..69483f4ecd0 100644 --- a/services/web/server/src/simcore_service_webserver/api_keys/_controller/rest.py +++ b/services/web/server/src/simcore_service_webserver/api_keys/_controller/rest.py @@ -21,7 +21,7 @@ from ...login.decorators import login_required from ...models import RequestContext from ...security.decorators import permission_required -from ...utils_aiohttp import envelope_json_response +from ...utils_aiohttp import envelope_json_response, get_api_base_url from .. import _service from ..models import ApiKey from .rest_exceptions import handle_plugin_requests_exceptions @@ -55,8 +55,8 @@ async def create_api_key(request: web.Request): api_key = ApiKeyCreateResponse.model_validate( { **asdict(created_api_key), - "api_base_url": "http://localhost:8000", - } # TODO: https://github.com/ITISFoundation/osparc-simcore/issues/6340 # @pcrespov + "api_base_url": get_api_base_url(request), + } ) return envelope_json_response(api_key) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_client.py b/services/web/server/src/simcore_service_webserver/director_v2/_client.py index c8863a0d169..ac3a61b726e 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_client.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_client.py @@ -16,6 +16,7 @@ ) from models_library.projects import ProjectID from models_library.users import UserID +from pydantic import AnyHttpUrl, TypeAdapter from ._client_base import request_director_v2 from .settings import DirectorV2Settings, get_client_session, get_plugin_settings @@ -62,7 +63,12 @@ async def get_computation( return DirectorV2ComputationGet.model_validate(computation_task_out) async def start_computation( - self, project_id: ProjectID, user_id: UserID, product_name: str, **options + self, + project_id: ProjectID, + user_id: UserID, + product_name: str, + product_api_base_url: str, + **options, ) -> str: computation_task_out = await request_director_v2( self._app, @@ -73,6 +79,9 @@ async def start_computation( user_id=user_id, project_id=project_id, product_name=product_name, + product_api_base_url=TypeAdapter(AnyHttpUrl).validate_python( + product_api_base_url + ), **options, ).model_dump(mode="json", exclude_unset=True), ) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_controller/rest.py b/services/web/server/src/simcore_service_webserver/director_v2/_controller/rest.py index c9d5e9f1345..12ab71bd96f 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_controller/rest.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_controller/rest.py @@ -26,7 +26,7 @@ from ...models import RequestContext from ...products import products_web from ...security.decorators import permission_required -from ...utils_aiohttp import envelope_json_response +from ...utils_aiohttp import envelope_json_response, get_api_base_url from .. import _director_v2_service from .._client import DirectorV2RestClient from .._director_v2_abc_service import get_project_run_policy @@ -111,7 +111,11 @@ async def start_computation(request: web.Request) -> web.Response: _started_pipelines_ids = await asyncio.gather( *[ computations.start_computation( - pid, req_ctx.user_id, req_ctx.product_name, **options + pid, + req_ctx.user_id, + req_ctx.product_name, + get_api_base_url(request), + **options, ) for pid in running_project_ids ] diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py index 10bad99a0c4..006bf4133a1 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py @@ -48,6 +48,7 @@ async def create_or_update_pipeline( user_id: UserID, project_id: ProjectID, product_name: ProductName, + product_api_base_url: str, ) -> DataType | None: # NOTE https://github.com/ITISFoundation/osparc-simcore/issues/7527 settings: DirectorV2Settings = get_plugin_settings(app) @@ -57,6 +58,7 @@ async def create_or_update_pipeline( "user_id": user_id, "project_id": f"{project_id}", "product_name": product_name, + "product_api_base_url": product_api_base_url, "wallet_info": await get_wallet_info( app, product=products_service.get_product(app, product_name), diff --git a/services/web/server/src/simcore_service_webserver/products/_web_middlewares.py b/services/web/server/src/simcore_service_webserver/products/_web_middlewares.py index e82a1a54f5b..a13d324ea70 100644 --- a/services/web/server/src/simcore_service_webserver/products/_web_middlewares.py +++ b/services/web/server/src/simcore_service_webserver/products/_web_middlewares.py @@ -8,6 +8,7 @@ from .._meta import API_VTAG from ..constants import APP_PRODUCTS_KEY, RQ_PRODUCT_KEY +from ..utils_aiohttp import iter_origins from .models import Product _logger = logging.getLogger(__name__) @@ -20,16 +21,9 @@ def _get_default_product_name(app: web.Application) -> str: def _discover_product_by_hostname(request: web.Request) -> str | None: products: OrderedDict[str, Product] = request.app[APP_PRODUCTS_KEY] - # - # SEE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Host - # SEE https://doc.traefik.io/traefik/getting-started/faq/#what-are-the-forwarded-headers-when-proxying-http-requests - originating_hosts = [ - request.headers.get("X-Forwarded-Host"), - request.host, - ] for product in products.values(): - for host in originating_hosts: - if host and product.host_regex.search(host): + for _, host in iter_origins(request): + if product.host_regex.search(host): product_name: str = product.name return product_name return None diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py index 9642ba581d3..2c02ca01959 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py @@ -62,7 +62,7 @@ from ...login.decorators import login_required from ...security.decorators import permission_required from ...users.api import get_user_id_from_gid, get_user_role -from ...utils_aiohttp import envelope_json_response +from ...utils_aiohttp import envelope_json_response, get_api_base_url from .. import _access_rights_service as access_rights_service from .. import _nodes_service, _projects_service, nodes_utils from .._nodes_service import NodeScreenshot, get_node_screenshots @@ -121,6 +121,7 @@ async def create_node(request: web.Request) -> web.Response: project_data, req_ctx.user_id, req_ctx.product_name, + get_api_base_url(request), body.service_key, body.service_version, body.service_id, @@ -182,6 +183,7 @@ async def patch_project_node(request: web.Request) -> web.Response: await _projects_service.patch_project_node( request.app, product_name=req_ctx.product_name, + product_api_base_url=get_api_base_url(request), user_id=req_ctx.user_id, project_id=path_params.project_id, node_id=path_params.node_id, @@ -211,6 +213,7 @@ async def delete_node(request: web.Request) -> web.Response: req_ctx.user_id, NodeIDStr(path_params.node_id), req_ctx.product_name, + product_api_base_url=get_api_base_url(request), ) return web.json_response(status=status.HTTP_204_NO_CONTENT) @@ -278,6 +281,7 @@ async def start_node(request: web.Request) -> web.Response: await _projects_service.start_project_node( request, product_name=req_ctx.product_name, + product_api_base_url=get_api_base_url(request), user_id=req_ctx.user_id, project_id=path_params.project_id, node_id=path_params.node_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index 0caa06d993b..86ecd1d2185 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -35,7 +35,7 @@ from ...security.api import check_user_permission from ...security.decorators import permission_required from ...users.api import get_user_fullname -from ...utils_aiohttp import envelope_json_response +from ...utils_aiohttp import envelope_json_response, get_api_base_url from .. import _crud_api_create, _crud_api_read, _projects_service from .._permalink_service import update_or_pop_permalink_in_project from ..models import ProjectDict @@ -116,6 +116,7 @@ async def create_project(request: web.Request): copy_data=query_params.copy_data, user_id=req_ctx.user_id, product_name=req_ctx.product_name, + product_api_base_url=get_api_base_url(request), simcore_user_agent=header_params.simcore_user_agent, predefined_project=predefined_project, parent_project_uuid=header_params.parent_project_uuid, @@ -452,6 +453,7 @@ async def clone_project(request: web.Request): copy_data=True, user_id=req_ctx.user_id, product_name=req_ctx.product_name, + product_api_base_url=get_api_base_url(request), simcore_user_agent=request.headers.get( X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE ), diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_states_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_states_rest.py index 2587982f90b..3085674e303 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_states_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_states_rest.py @@ -27,7 +27,7 @@ from ...products.models import Product from ...security.decorators import permission_required from ...users import api -from ...utils_aiohttp import envelope_json_response +from ...utils_aiohttp import envelope_json_response, get_api_base_url from .. import _projects_service, projects_wallets_service from ..exceptions import ProjectStartsTooManyDynamicNodesError from ._rest_exceptions import handle_plugin_requests_exceptions @@ -118,7 +118,11 @@ async def open_project(request: web.Request) -> web.Response: # services in the project is highter than the maximum allowed per project # the project shall still open though. await _projects_service.run_project_dynamic_services( - request, project, req_ctx.user_id, req_ctx.product_name + request, + project, + req_ctx.user_id, + req_ctx.product_name, + get_api_base_url(request), ) # and let's update the project last change timestamp diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index e287438d0b2..35f5ccb9e1f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -258,6 +258,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche copy_data: bool, user_id: UserID, product_name: str, + product_api_base_url: str, predefined_project: ProjectDict | None, simcore_user_agent: str, parent_project_uuid: ProjectID | None, @@ -411,7 +412,11 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche # This is a new project and every new graph needs to be reflected in the pipeline tables await director_v2_service.create_or_update_pipeline( - request.app, user_id, new_project["uuid"], product_name + request.app, + user_id, + new_project["uuid"], + product_name, + product_api_base_url, ) # get the latest state of the project (lastChangeDate for instance) new_project, _ = await _projects_repository.get_project_dict_and_type( diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py index 9535381ca82..40ccff58a41 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py @@ -617,6 +617,7 @@ async def _start_dynamic_service( # noqa: C901 service_key: ServiceKey, service_version: ServiceVersion, product_name: str, + product_api_base_url: str, user_id: UserID, project_uuid: ProjectID, node_uuid: NodeID, @@ -788,6 +789,7 @@ async def _() -> None: app=request.app, dynamic_service_start=DynamicServiceStart( product_name=product_name, + product_api_base_url=product_api_base_url, can_save=save_state, project_id=project_uuid, user_id=user_id, @@ -816,6 +818,7 @@ async def add_project_node( project: dict[str, Any], user_id: UserID, product_name: str, + product_api_base_url: str, service_key: ServiceKey, service_version: ServiceVersion, service_id: str | None, @@ -866,7 +869,11 @@ async def add_project_node( # also ensure the project is updated by director-v2 since services # are due to access comp_tasks at some point see [https://github.com/ITISFoundation/osparc-simcore/issues/3216] await director_v2_service.create_or_update_pipeline( - request.app, user_id, project["uuid"], product_name + request.app, + user_id, + project["uuid"], + product_name, + product_api_base_url, ) await dynamic_scheduler_service.update_projects_networks( request.app, project_id=ProjectID(project["uuid"]) @@ -880,6 +887,7 @@ async def add_project_node( service_key=service_key, service_version=service_version, product_name=product_name, + product_api_base_url=product_api_base_url, user_id=user_id, project_uuid=ProjectID(project["uuid"]), node_uuid=node_uuid, @@ -890,7 +898,8 @@ async def add_project_node( async def start_project_node( request: web.Request, - product_name: str, + product_name: ProductName, + product_api_base_url: str, user_id: UserID, project_id: ProjectID, node_id: NodeID, @@ -906,6 +915,7 @@ async def start_project_node( service_key=node_details.key, service_version=node_details.version, product_name=product_name, + product_api_base_url=product_api_base_url, user_id=user_id, project_uuid=project_id, node_uuid=node_id, @@ -946,6 +956,7 @@ async def delete_project_node( user_id: UserID, node_uuid: NodeIDStr, product_name: ProductName, + product_api_base_url: str, ) -> None: log.debug( "deleting node %s in project %s for user %s", node_uuid, project_uuid, user_id @@ -991,7 +1002,7 @@ async def delete_project_node( # also ensure the project is updated by director-v2 since services product_name = products_web.get_product_name(request) await director_v2_service.create_or_update_pipeline( - request.app, user_id, project_uuid, product_name + request.app, user_id, project_uuid, product_name, product_api_base_url ) await dynamic_scheduler_service.update_projects_networks( request.app, project_id=project_uuid @@ -1063,6 +1074,7 @@ async def patch_project_node( app: web.Application, *, product_name: ProductName, + product_api_base_url: str, user_id: UserID, project_id: ProjectID, node_id: NodeID, @@ -1121,7 +1133,11 @@ async def patch_project_node( # 4. Make calls to director-v2 to keep data in sync (ex. comp_tasks DB table) await director_v2_service.create_or_update_pipeline( - app, user_id, project_id, product_name=product_name + app, + user_id, + project_id, + product_name=product_name, + product_api_base_url=product_api_base_url, ) if _node_patch_exclude_unset.get("label"): await dynamic_scheduler_service.update_projects_networks( @@ -1203,7 +1219,9 @@ async def update_project_node_outputs( # changed entries come in the form of {node_uuid: {outputs: {changed_key1: value1, changed_key2: value2}}} # we do want only the key names changed_keys = ( - changed_entries.get(NodeIDStr(f"{node_id}"), {}).get("outputs", {}).keys() + changed_entries.get(TypeAdapter(NodeIDStr).validate_python(f"{node_id}"), {}) + .get("outputs", {}) + .keys() ) return updated_project, changed_keys @@ -1755,6 +1773,7 @@ async def run_project_dynamic_services( project: dict, user_id: UserID, product_name: str, + product_api_base_url: str, ) -> None: # first get the services if they already exist project_settings: ProjectsSettings = get_plugin_settings(request.app) @@ -1803,6 +1822,7 @@ async def run_project_dynamic_services( service_key=services_to_start_uuids[service_uuid]["key"], service_version=services_to_start_uuids[service_uuid]["version"], product_name=product_name, + product_api_base_url=product_api_base_url, user_id=user_id, project_uuid=project["uuid"], node_uuid=NodeID(service_uuid), diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py index a0509275a12..ca2dce6dd60 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py @@ -183,7 +183,12 @@ def _create_project_with_filepicker_and_service( async def _add_new_project( - app: web.Application, project: Project, user: UserInfo, *, product_name: str + app: web.Application, + project: Project, + user: UserInfo, + *, + product_name: str, + product_api_base_url: str, ): # TODO: move this to projects_api # TODO: this piece was taken from the end of projects.projects_handlers.create_projects @@ -212,7 +217,9 @@ async def _add_new_project( # # TODO: Ensure this user has access to these services! # - await create_or_update_pipeline(app, user.id, project.uuid, product_name) + await create_or_update_pipeline( + app, user.id, project.uuid, product_name, product_api_base_url + ) async def _project_exists( @@ -245,6 +252,7 @@ async def get_or_create_project_with_file_and_service( download_link: HttpUrl, *, product_name: str, + product_api_base_url: str, ) -> ProjectNodePair: # # Generate one project per user + download_link + viewer @@ -292,7 +300,13 @@ async def get_or_create_project_with_file_and_service( viewer, ) - await _add_new_project(app, project, user, product_name=product_name) + await _add_new_project( + app, + project, + user, + product_name=product_name, + product_api_base_url=product_api_base_url, + ) return ProjectNodePair(project_uid=project_uid, node_uid=service_id) @@ -304,6 +318,7 @@ async def get_or_create_project_with_service( service_info: ServiceInfo, *, product_name: str, + product_api_base_url: str, ) -> ProjectNodePair: project_uid: ProjectID = compose_uuid_from(user.id, service_info.footprint) _, service_id = _generate_nodeids(project_uid) @@ -322,7 +337,13 @@ async def get_or_create_project_with_service( owner=user, service_info=service_info, ) - await _add_new_project(app, project, user, product_name=product_name) + await _add_new_project( + app, + project, + user, + product_name=product_name, + product_api_base_url=product_api_base_url, + ) return ProjectNodePair(project_uid=project_uid, node_uid=service_id) @@ -335,6 +356,7 @@ async def get_or_create_project_with_file( *, project_thumbnail: HttpUrl, product_name: str, + product_api_base_url: str, ) -> ProjectNodePair: project_uid: ProjectID = compose_uuid_from(user.id, file_params.footprint) file_picker_id, _ = _generate_nodeids(project_uid) @@ -364,6 +386,12 @@ async def get_or_create_project_with_file( }, ) - await _add_new_project(app, project, user, product_name=product_name) + await _add_new_project( + app, + project, + user, + product_name=product_name, + product_api_base_url=product_api_base_url, + ) return ProjectNodePair(project_uid=project_uid, node_uid=file_picker_id) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py index 0a0d37ef17b..605b00f623d 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py @@ -19,7 +19,7 @@ from ..dynamic_scheduler import api as dynamic_scheduler_service from ..products import products_web from ..utils import compose_support_error_msg -from ..utils_aiohttp import create_redirect_to_page_response +from ..utils_aiohttp import create_redirect_to_page_response, get_api_base_url from ._catalog import ValidService, validate_requested_service from ._constants import MSG_UNEXPECTED_DISPATCH_ERROR from ._core import validate_requested_file, validate_requested_viewer @@ -249,6 +249,7 @@ async def get_redirection_to_viewer(request: web.Request): viewer, file_params.download_link, product_name=products_web.get_product_name(request), + product_api_base_url=get_api_base_url(request), ) await dynamic_scheduler_service.update_projects_networks( request.app, project_id=project_id @@ -280,6 +281,7 @@ async def get_redirection_to_viewer(request: web.Request): user, service_info=_create_service_info_from(valid_service), product_name=products_web.get_product_name(request), + product_api_base_url=get_api_base_url(request), ) await dynamic_scheduler_service.update_projects_networks( request.app, project_id=project_id @@ -318,6 +320,7 @@ async def get_redirection_to_viewer(request: web.Request): app=request.app ).STUDIES_DEFAULT_FILE_THUMBNAIL, product_name=products_web.get_product_name(request), + product_api_base_url=get_api_base_url(request), ) await dynamic_scheduler_service.update_projects_networks( request.app, project_id=project_id diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py index fa5eabc0124..f9012c67d4f 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py @@ -42,7 +42,7 @@ from ..security.api import is_anonymous, remember_identity from ..storage.api import copy_data_folders_from_project from ..utils import compose_support_error_msg -from ..utils_aiohttp import create_redirect_to_page_response +from ..utils_aiohttp import create_redirect_to_page_response, get_api_base_url from ._constants import ( MSG_PROJECT_NOT_FOUND, MSG_PROJECT_NOT_PUBLISHED, @@ -211,7 +211,11 @@ async def copy_study_to_account( if lr_task.done: await lr_task.result() await director_v2_service.create_or_update_pipeline( - request.app, user["id"], project["uuid"], product_name + request.app, + user["id"], + project["uuid"], + product_name, + get_api_base_url(request), ) await dynamic_scheduler_service.update_projects_networks( request.app, project_id=ProjectID(project["uuid"]) diff --git a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py index 5a13e108201..10f28669c8a 100644 --- a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py +++ b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py @@ -1,12 +1,13 @@ import io import logging -from collections.abc import Callable +from collections.abc import Callable, Iterator from typing import Any, Generic, Literal, TypeAlias, TypeVar from aiohttp import web from aiohttp.web_exceptions import HTTPError, HTTPException from aiohttp.web_routedef import RouteDef, RouteTableDef from common_library.json_serialization import json_dumps +from common_library.network import is_ip_address from models_library.generics import Envelope from pydantic import BaseModel, Field from servicelib.common_headers import X_FORWARDED_PROTO @@ -128,3 +129,38 @@ class NextPage(BaseModel, Generic[PageParameters]): ..., description="Code name to the front-end page. Ideally a PageStr" ) parameters: PageParameters | None = None + + +def iter_origins(request: web.Request) -> Iterator[tuple[str, str]]: + # + # SEE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Host + # SEE https://doc.traefik.io/traefik/getting-started/faq/#what-are-the-forwarded-headers-when-proxying-http-requests + seen = set() + + # X-Forwarded-Proto and X-Forwarded-Host can contain a comma-separated list of protocols and hosts + # (when the request passes through multiple proxies) + fwd_protos = [ + p.strip() + for p in request.headers.get("X-Forwarded-Proto", "").split(",") + if p.strip() + ] + fwd_hosts = [ + h.strip() + for h in request.headers.get("X-Forwarded-Host", "").split(",") + if h.strip() + ] + + if fwd_protos and fwd_hosts: + for proto, host in zip(fwd_protos, fwd_hosts, strict=False): + if (proto, host) not in seen: + seen.add((proto, host)) + yield (proto, host.partition(":")[0]) # strip port + + # fallback to request scheme/host + yield request.scheme, f"{request.host.partition(':')[0]}" + + +def get_api_base_url(request: web.Request) -> str: + scheme, host = next(iter_origins(request)) + api_host = api_host = host if is_ip_address(host) else f"api.{host}" + return f"{scheme}://{api_host}" diff --git a/services/web/server/tests/unit/isolated/test_dynamic_scheduler.py b/services/web/server/tests/unit/isolated/test_dynamic_scheduler.py index 944a958baf2..1810e48493a 100644 --- a/services/web/server/tests/unit/isolated/test_dynamic_scheduler.py +++ b/services/web/server/tests/unit/isolated/test_dynamic_scheduler.py @@ -48,7 +48,7 @@ def mock_rpc_client( @pytest.fixture def dynamic_service_start() -> DynamicServiceStart: return DynamicServiceStart.model_validate( - DynamicServiceStart.model_config["json_schema_extra"]["example"] + DynamicServiceStart.model_json_schema()["example"] ) diff --git a/services/web/server/tests/unit/isolated/test_utils_aiohttp.py b/services/web/server/tests/unit/isolated/test_utils_aiohttp.py index e07914929bd..52c3cc6089b 100644 --- a/services/web/server/tests/unit/isolated/test_utils_aiohttp.py +++ b/services/web/server/tests/unit/isolated/test_utils_aiohttp.py @@ -5,13 +5,18 @@ import json from typing import Any +from unittest.mock import Mock, patch from uuid import UUID import pytest from aiohttp import web from faker import Faker from pydantic import BaseModel -from simcore_service_webserver.utils_aiohttp import envelope_json_response +from simcore_service_webserver.utils_aiohttp import ( + envelope_json_response, + get_api_base_url, + iter_origins, +) @pytest.fixture @@ -30,6 +35,18 @@ class Point(BaseModel): } +@pytest.fixture +def make_request(): + def _make_request(headers=None, scheme="http", host="example.com"): + req = Mock(spec=web.Request) + req.headers = headers or {} + req.scheme = scheme + req.host = host + return req + + return _make_request + + def test_enveloped_successful_response(data: dict): resp = envelope_json_response(data, web.HTTPCreated) assert resp.text is not None @@ -46,3 +63,116 @@ def test_enveloped_failing_response(): assert resp.text is not None assert {"error"} == set(json.loads(resp.text).keys()) + + +@pytest.mark.parametrize( + "headers, expected_output", + [ + # No headers - fallback to request + ( + {}, + [("http", "localhost")], + ), + # Single entry + ( + {"X-Forwarded-Proto": "https", "X-Forwarded-Host": "example.com"}, + [("https", "example.com"), ("http", "localhost")], + ), + # Multiple entries with ports + ( + { + "X-Forwarded-Proto": "https, http", + "X-Forwarded-Host": "api.example.com:443, 192.168.1.1:8080", + }, + [ + ("https", "api.example.com"), + ("http", "192.168.1.1"), + ("http", "localhost"), + ], + ), + # Unequal list lengths (proto longer) + ( + { + "X-Forwarded-Proto": "http, https, ftp", + "X-Forwarded-Host": "site.com, cdn.site.com", + }, + [("http", "site.com"), ("https", "cdn.site.com"), ("http", "localhost")], + ), + # With whitespace + ( + { + "X-Forwarded-Proto": " https , http ", + "X-Forwarded-Host": " example.com , proxy.com ", + }, + [("https", "example.com"), ("http", "proxy.com"), ("http", "localhost")], + ), + # Duplicate entries + ( + { + "X-Forwarded-Proto": "https, https", + "X-Forwarded-Host": "example.com, example.com", + }, + [("https", "example.com"), ("http", "localhost")], + ), + ], +) +def test_iter_origins(headers, expected_output): + request = Mock( + spec=web.Request, + headers=headers, + scheme="http", + host="localhost:8080", # Port should be stripped + ) + + results = list(iter_origins(request)) + + assert results == expected_output + + +def test_no_forwarded_headers_regular_host(make_request): + req = make_request() + with patch("common_library.network.is_ip_address", return_value=False): + url = get_api_base_url(req) + assert url == "http://api.example.com" + + +def test_no_forwarded_headers_ip_host(make_request): + req = make_request(host="192.168.1.2") + with patch("common_library.network.is_ip_address", return_value=True): + url = get_api_base_url(req) + assert url == "http://192.168.1.2" + + +def test_with_forwarded_headers(make_request): + headers = {"X-Forwarded-Proto": "https", "X-Forwarded-Host": "mydomain.com"} + req = make_request(headers=headers, scheme="http", host="example.com") + with patch("common_library.network.is_ip_address", return_value=False): + url = get_api_base_url(req) + assert url == "https://api.mydomain.com" + + +def test_with_multiple_forwarded_headers(make_request): + headers = { + "X-Forwarded-Proto": "https, http", + "X-Forwarded-Host": "api1.com, api2.com", + } + req = make_request(headers=headers, scheme="http", host="example.com") + with patch("common_library.network.is_ip_address", side_effect=[False, False]): + url = get_api_base_url(req) + assert url == "https://api.api1.com" + + +def test_forwarded_host_with_port(make_request): + headers = {"X-Forwarded-Proto": "https", "X-Forwarded-Host": "mydomain.com:8080"} + req = make_request(headers=headers, scheme="http", host="example.com:8080") + with patch("common_library.network.is_ip_address", return_value=False): + url = get_api_base_url(req) + assert url == "https://api.mydomain.com" + + +def test_empty_forwarded_headers_fallback(make_request): + headers = {"X-Forwarded-Proto": "", "X-Forwarded-Host": ""} + req = make_request(headers=headers, scheme="https", host="example.com") + with patch("common_library.network.is_ip_address", return_value=False): + url = get_api_base_url(req) + assert url == "https://api.example.com" diff --git a/services/web/server/tests/unit/with_dbs/01/test_api_keys.py b/services/web/server/tests/unit/with_dbs/01/test_api_keys.py index a486dd85603..0793762fada 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_api_keys.py +++ b/services/web/server/tests/unit/with_dbs/01/test_api_keys.py @@ -109,7 +109,11 @@ async def test_create_api_key( expected: HTTPStatus, ): display_name = "foo" - resp = await client.post("/v0/auth/api-keys", json={"displayName": display_name}) + resp = await client.post( + "/v0/auth/api-keys", + json={"displayName": display_name}, + headers={"X-Forwarded-Proto": "https", "X-Forwarded-Host": "osparc.io"}, + ) data, errors = await assert_status(resp, expected) @@ -117,6 +121,8 @@ async def test_create_api_key( assert data["displayName"] == display_name assert "apiKey" in data assert "apiSecret" in data + assert "apiBaseUrl" in data + assert data["apiBaseUrl"] == "https://api.osparc.io/" resp = await client.get("/v0/auth/api-keys") data, _ = await assert_status(resp, expected) diff --git a/services/web/server/tests/unit/with_dbs/01/test_director_v2.py b/services/web/server/tests/unit/with_dbs/01/test_director_v2.py index be0d045e06f..11468d1257f 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_director_v2.py +++ b/services/web/server/tests/unit/with_dbs/01/test_director_v2.py @@ -32,11 +32,16 @@ async def test_create_pipeline( user_id: UserID, project_id: ProjectID, osparc_product_name: str, + osparc_product_api_base_url: str, ): assert client.app task_out = await director_v2_service.create_or_update_pipeline( - client.app, user_id, project_id, osparc_product_name + client.app, + user_id, + project_id, + osparc_product_name, + osparc_product_api_base_url, ) assert task_out assert isinstance(task_out, dict) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index 3d4b26894b8..8795f8cec62 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -360,6 +360,7 @@ async def test_open_project( mock_orphaned_services: mock.Mock, mock_catalog_api: dict[str, mock.Mock], osparc_product_name: str, + osparc_product_api_base_url: str, mocked_notifications_plugin: dict[str, mock.Mock], ): # POST /v0/projects/{project_id}:open @@ -399,6 +400,7 @@ async def test_open_project( request_dns=request_dns, can_save=save_state, product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, service_resources=ServiceResourcesDictHelpers.create_jsonable( mock_service_resources ), @@ -499,6 +501,7 @@ async def test_open_template_project_for_edition( mock_orphaned_services: mock.Mock, mock_catalog_api: dict[str, mock.Mock], osparc_product_name: str, + osparc_product_api_base_url: str, mocked_notifications_plugin: dict[str, mock.Mock], ): # POST /v0/projects/{project_id}:open @@ -544,6 +547,7 @@ async def test_open_template_project_for_edition( mock_service_resources ), product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, wallet_info=None, pricing_info=None, hardware_info=None, diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py index 2ae68f22182..b05757d70bc 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py @@ -86,6 +86,7 @@ async def test_add_new_project_from_model_instance( client: TestClient, mocker: MockerFixture, osparc_product_name: str, + osparc_product_api_base_url: str, user: UserInfo, project_id: ProjectID, file_picker_id: NodeID, @@ -117,7 +118,13 @@ async def test_add_new_project_from_model_instance( viewer_info=viewer_info, ) - await _add_new_project(client.app, project, user, product_name=osparc_product_name) + await _add_new_project( + client.app, + project, + user, + product_name=osparc_product_name, + product_api_base_url=osparc_product_api_base_url, + ) assert mock_directorv2_api.called diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index aadf58c6101..652144bb57d 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -258,6 +258,11 @@ def osparc_product_name() -> str: return "osparc" +@pytest.fixture(scope="session") +def osparc_product_api_base_url() -> str: + return "http://127.0.0.1/" + + @pytest.fixture async def default_product_name(client: TestClient) -> ProductName: assert client.app