diff --git a/api/specs/web-server/_catalog.py b/api/specs/web-server/_catalog.py index 90dd187c55b..153c2d8b968 100644 --- a/api/specs/web-server/_catalog.py +++ b/api/specs/web-server/_catalog.py @@ -4,6 +4,7 @@ from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet from models_library.api_schemas_webserver.catalog import ( CatalogServiceGet, + CatalogServiceListItem, CatalogServiceUpdate, ServiceInputGet, ServiceInputKey, @@ -31,16 +32,11 @@ ) -# -# /catalog/services/* COLLECTION -# - - @router.get( "/catalog/services/-/latest", - response_model=Page[CatalogServiceGet], + response_model=Page[CatalogServiceListItem], ) -def list_services_latest(_query_params: Annotated[ListServiceParams, Depends()]): +def list_services_latest(_query: Annotated[ListServiceParams, Depends()]): pass @@ -48,8 +44,7 @@ def list_services_latest(_query_params: Annotated[ListServiceParams, Depends()]) "/catalog/services/{service_key}/{service_version}", response_model=Envelope[CatalogServiceGet], ) -def get_service(_path_params: Annotated[ServicePathParams, Depends()]): - ... +def get_service(_path: Annotated[ServicePathParams, Depends()]): ... @router.patch( @@ -57,10 +52,9 @@ def get_service(_path_params: Annotated[ServicePathParams, Depends()]): response_model=Envelope[CatalogServiceGet], ) def update_service( - _path_params: Annotated[ServicePathParams, Depends()], - _update: CatalogServiceUpdate, -): - ... + _path: Annotated[ServicePathParams, Depends()], + _body: CatalogServiceUpdate, +): ... @router.get( @@ -68,9 +62,8 @@ def update_service( response_model=Envelope[list[ServiceInputGet]], ) def list_service_inputs( - _path_params: Annotated[ServicePathParams, Depends()], -): - ... + _path: Annotated[ServicePathParams, Depends()], +): ... @router.get( @@ -78,9 +71,8 @@ def list_service_inputs( response_model=Envelope[ServiceInputGet], ) def get_service_input( - _path_params: Annotated[_ServiceInputsPathParams, Depends()], -): - ... + _path: Annotated[_ServiceInputsPathParams, Depends()], +): ... @router.get( @@ -88,10 +80,9 @@ def get_service_input( response_model=Envelope[list[ServiceInputKey]], ) def get_compatible_inputs_given_source_output( - _path_params: Annotated[ServicePathParams, Depends()], - _query_params: Annotated[_FromServiceOutputParams, Depends()], -): - ... + _path: Annotated[ServicePathParams, Depends()], + _query: Annotated[_FromServiceOutputParams, Depends()], +): ... @router.get( @@ -99,9 +90,8 @@ def get_compatible_inputs_given_source_output( response_model=Envelope[list[ServiceOutputKey]], ) def list_service_outputs( - _path_params: Annotated[ServicePathParams, Depends()], -): - ... + _path: Annotated[ServicePathParams, Depends()], +): ... @router.get( @@ -109,9 +99,8 @@ def list_service_outputs( response_model=Envelope[list[ServiceOutputGet]], ) def get_service_output( - _path_params: Annotated[_ServiceOutputsPathParams, Depends()], -): - ... + _path: Annotated[_ServiceOutputsPathParams, Depends()], +): ... @router.get( @@ -119,10 +108,9 @@ def get_service_output( response_model=Envelope[list[ServiceOutputKey]], ) def get_compatible_outputs_given_target_input( - _path_params: Annotated[ServicePathParams, Depends()], - _query_params: Annotated[_ToServiceInputsParams, Depends()], -): - ... + _path: Annotated[ServicePathParams, Depends()], + _query: Annotated[_ToServiceInputsParams, Depends()], +): ... @router.get( @@ -130,9 +118,8 @@ def get_compatible_outputs_given_target_input( response_model=Envelope[ServiceResourcesGet], ) def get_service_resources( - _params: Annotated[ServicePathParams, Depends()], -): - ... + _path: Annotated[ServicePathParams, Depends()], +): ... @router.get( @@ -142,6 +129,5 @@ def get_service_resources( tags=["pricing-plans"], ) async def get_service_pricing_plan( - _params: Annotated[ServicePathParams, Depends()], -): - ... + _path: Annotated[ServicePathParams, Depends()], +): ... diff --git a/api/specs/web-server/_projects_nodes.py b/api/specs/web-server/_projects_nodes.py index 06189fa7cc3..61650993ab7 100644 --- a/api/specs/web-server/_projects_nodes.py +++ b/api/specs/web-server/_projects_nodes.py @@ -18,6 +18,7 @@ NodePatch, NodeRetrieve, NodeRetrieved, + ProjectNodeServicesGet, ServiceResourcesDict, ) from models_library.generics import Envelope @@ -76,8 +77,7 @@ def delete_node(project_id: str, node_id: str): # noqa: ARG001 ) def retrieve_node( project_id: str, node_id: str, _retrieve: NodeRetrieve # noqa: ARG001 -): - ... +): ... @router.post( @@ -147,8 +147,7 @@ def get_node_resources(project_id: str, node_id: str): # noqa: ARG001 ) def replace_node_resources( project_id: str, node_id: str, _new: ServiceResourcesDict # noqa: ARG001 -): - ... +): ... # @@ -156,6 +155,13 @@ def replace_node_resources( # +@router.get( + "/projects/{project_id}/nodes/-/services", + response_model=Envelope[ProjectNodeServicesGet], +) +async def get_project_services(project_id: ProjectID): ... + + @router.get( "/projects/{project_id}/nodes/-/services:access", response_model=Envelope[_ProjectGroupAccess], @@ -163,8 +169,7 @@ def replace_node_resources( ) async def get_project_services_access_for_gid( project_id: ProjectID, for_gid: GroupID # noqa: ARG001 -): - ... +): ... assert_handler_signature_against_model( @@ -197,8 +202,7 @@ async def list_project_nodes_previews(project_id: ProjectID): # noqa: ARG001 ) async def get_project_node_preview( project_id: ProjectID, node_id: NodeID # noqa: ARG001 -): - ... +): ... assert_handler_signature_against_model(get_project_node_preview, NodePathParams) diff --git a/packages/models-library/src/models_library/access_rights.py b/packages/models-library/src/models_library/access_rights.py index a6cea15a946..a78ba105ac8 100644 --- a/packages/models-library/src/models_library/access_rights.py +++ b/packages/models-library/src/models_library/access_rights.py @@ -1,9 +1,18 @@ +from typing import Annotated + from pydantic import BaseModel, ConfigDict, Field class AccessRights(BaseModel): - read: bool = Field(..., description="has read access") - write: bool = Field(..., description="has write access") - delete: bool = Field(..., description="has deletion rights") + read: Annotated[bool, Field(description="has read access")] + write: Annotated[bool, Field(description="has write access")] + delete: Annotated[bool, Field(description="has deletion rights")] + + model_config = ConfigDict(extra="forbid") + + +class ExecutableAccessRights(BaseModel): + write: Annotated[bool, Field(description="can change executable settings")] + execute: Annotated[bool, Field(description="can run executable")] model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index e4c39c4c158..9e57924808b 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -1,6 +1,7 @@ from datetime import datetime -from typing import Any, TypeAlias +from typing import Annotated, Any, TypeAlias +from common_library.basic_types import DEFAULT_FACTORY from models_library.rpc_pagination import PageRpc from pydantic import ConfigDict, Field, HttpUrl, NonNegativeInt from pydantic.config import JsonDict @@ -154,9 +155,10 @@ class ServiceGet( ServiceMetaDataPublished, ServiceAccessRights, ServiceMetaDataEditable ): # pylint: disable=too-many-ancestors - owner: LowerCaseEmailStr | None = Field( - description="None when the owner email cannot be found in the database" - ) + owner: Annotated[ + LowerCaseEmailStr | None, + Field(description="None when the owner email cannot be found in the database"), + ] @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: @@ -169,7 +171,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None: ) -class ServiceGetV2(CatalogOutputSchema): +class _BaseServiceGetV2(CatalogOutputSchema): # Model used in catalog's rpc and rest interfaces key: ServiceKey version: ServiceVersion @@ -183,13 +185,14 @@ class ServiceGetV2(CatalogOutputSchema): version_display: str | None = None - service_type: ServiceType = Field(default=..., alias="type") + service_type: Annotated[ServiceType, Field(alias="type")] contact: LowerCaseEmailStr | None - authors: list[Author] = Field(..., min_length=1) - owner: LowerCaseEmailStr | None = Field( - description="None when the owner email cannot be found in the database" - ) + authors: Annotated[list[Author], Field(min_length=1)] + owner: Annotated[ + LowerCaseEmailStr | None, + Field(description="None when the owner email cannot be found in the database"), + ] inputs: ServiceInputsDict outputs: ServiceOutputsDict @@ -202,13 +205,25 @@ class ServiceGetV2(CatalogOutputSchema): classifiers: list[str] | None = [] quality: dict[str, Any] = {} - history: list[ServiceRelease] = Field( - default_factory=list, - description="history of releases for this service at this point in time, starting from the newest to the oldest." - " It includes current release.", - json_schema_extra={"default": []}, + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + alias_generator=snake_to_camel, ) + +class ServiceGetV2(_BaseServiceGetV2): + # Model used in catalog's rpc and rest interfaces + history: Annotated[ + list[ServiceRelease], + Field( + default_factory=list, + description="history of releases for this service at this point in time, starting from the newest to the oldest." + " It includes current release.", + json_schema_extra={"default": []}, + ), + ] = DEFAULT_FACTORY + @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: schema.update( @@ -269,16 +284,25 @@ def _update_json_schema_extra(schema: JsonDict) -> None: ) model_config = ConfigDict( - extra="forbid", - populate_by_name=True, - alias_generator=snake_to_camel, json_schema_extra=_update_json_schema_extra, ) +class ServiceListItem(_BaseServiceGetV2): + history: Annotated[ + list[ServiceRelease], + Field( + default_factory=list, + deprecated=True, + description="History will be replaced by current 'release' instead", + json_schema_extra={"default": []}, + ), + ] = DEFAULT_FACTORY + + PageRpcServicesGetV2: TypeAlias = PageRpc[ # WARNING: keep this definition in models_library and not in the RPC interface - ServiceGetV2 + ServiceListItem ] ServiceResourcesGet: TypeAlias = ServiceResourcesDict @@ -310,3 +334,11 @@ class ServiceUpdateV2(CatalogInputSchema): assert set(ServiceUpdateV2.model_fields.keys()) - set( # nosec ServiceGetV2.model_fields.keys() ) == {"deprecated"} + + +class MyServiceGet(CatalogOutputSchema): + key: ServiceKey + release: ServiceRelease + + owner: GroupID | None + my_access_rights: ServiceGroupAccessRightsV2 diff --git a/packages/models-library/src/models_library/api_schemas_webserver/catalog.py b/packages/models-library/src/models_library/api_schemas_webserver/catalog.py index 1391ce19ce3..76fdf4ac06e 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/catalog.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/catalog.py @@ -1,4 +1,4 @@ -from typing import Any, TypeAlias +from typing import Annotated, Any, TypeAlias from pydantic import ConfigDict, Field from pydantic.config import JsonDict @@ -32,9 +32,9 @@ class _BaseCommonApiExtension(BaseModel): class ServiceInputGet(ServiceInput, _BaseCommonApiExtension): """Extends fields of api_schemas_catalog.services.ServiceGet.outputs[*]""" - key_id: ServiceInputKey = Field( - ..., description="Unique name identifier for this input" - ) + key_id: Annotated[ + ServiceInputKey, Field(description="Unique name identifier for this input") + ] @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: @@ -78,9 +78,9 @@ def _update_json_schema_extra(schema: JsonDict) -> None: class ServiceOutputGet(ServiceOutput, _BaseCommonApiExtension): """Extends fields of api_schemas_catalog.services.ServiceGet.outputs[*]""" - key_id: ServiceOutputKey = Field( - ..., description="Unique name identifier for this input" - ) + key_id: Annotated[ + ServiceOutputKey, Field(description="Unique name identifier for this input") + ] @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: @@ -225,38 +225,22 @@ def _update_json_schema_extra(schema: JsonDict) -> None: } -class ServiceGet(api_schemas_catalog_services.ServiceGet): - # pylint: disable=too-many-ancestors - inputs: ServiceInputsGetDict = Field( # type: ignore[assignment] - ..., description="inputs with extended information" - ) - outputs: ServiceOutputsGetDict = Field( # type: ignore[assignment] - ..., description="outputs with extended information" - ) - - @staticmethod - def _update_json_schema_extra(schema: JsonDict) -> None: - schema.update({"examples": [_EXAMPLE_FILEPICKER, _EXAMPLE_SLEEPER]}) - - model_config = ConfigDict( - **OutputSchema.model_config, - json_schema_extra=_update_json_schema_extra, - ) +ServiceResourcesGet: TypeAlias = api_schemas_catalog_services.ServiceResourcesGet -ServiceResourcesGet: TypeAlias = api_schemas_catalog_services.ServiceResourcesGet +class CatalogServiceListItem(api_schemas_catalog_services.ServiceListItem): + inputs: ServiceInputsGetDict # type: ignore[assignment] + outputs: ServiceOutputsGetDict # type: ignore[assignment] class CatalogServiceGet(api_schemas_catalog_services.ServiceGetV2): - # NOTE: will replace ServiceGet! - # pylint: disable=too-many-ancestors - inputs: ServiceInputsGetDict = Field( # type: ignore[assignment] - ..., description="inputs with extended information" - ) - outputs: ServiceOutputsGetDict = Field( # type: ignore[assignment] - ..., description="outputs with extended information" - ) + inputs: Annotated[ # type: ignore[assignment] + ServiceInputsGetDict, Field(description="inputs with extended information") + ] + outputs: Annotated[ # type: ignore[assignment] + ServiceOutputsGetDict, Field(description="outputs with extended information") + ] @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 69aafba0962..a8932553201 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -1,8 +1,12 @@ # mypy: disable-error-code=truthy-function from typing import Annotated, Any, Literal, TypeAlias +from models_library.groups import GroupID +from models_library.projects import ProjectID +from models_library.services_history import ServiceRelease from pydantic import ConfigDict, Field +from ..access_rights import ExecutableAccessRights from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut from ..basic_types import PortInt from ..projects_nodes import InputID, InputsDict, PartialNode @@ -40,7 +44,7 @@ class NodePatch(InputSchemaWithoutCamelCase): ] inputs_required: Annotated[ list[InputID] | None, - Field(alias="inputsRequired"), + Field(alias="inputsRequired"), ] = None input_nodes: Annotated[ list[NodeID] | None, @@ -55,9 +59,9 @@ class NodePatch(InputSchemaWithoutCamelCase): ), ] = None boot_options: Annotated[BootOptions | None, Field(alias="bootOptions")] = None - outputs: dict[ - str, Any - ] | None = None # NOTE: it is used by frontend for File Picker + outputs: dict[str, Any] | None = ( + None # NOTE: it is used by frontend for File Picker + ) def to_domain_model(self) -> PartialNode: data = self.model_dump( @@ -197,3 +201,20 @@ class NodeRetrieve(InputSchemaWithoutCamelCase): class NodeRetrieved(RetrieveDataOut): model_config = OutputSchema.model_config + + +class NodeServiceGet(OutputSchema): + key: ServiceKey + release: ServiceRelease + owner: Annotated[ + GroupID | None, + Field( + description="Service owner primary group id or None if ownership still not defined" + ), + ] + my_access_rights: ExecutableAccessRights + + +class ProjectNodeServicesGet(OutputSchema): + project_uuid: ProjectID + services: list[NodeServiceGet] diff --git a/packages/models-library/src/models_library/services_history.py b/packages/models-library/src/models_library/services_history.py index b38f5f2e783..91ed08fbe4b 100644 --- a/packages/models-library/src/models_library/services_history.py +++ b/packages/models-library/src/models_library/services_history.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import TypeAlias +from typing import Annotated, TypeAlias from pydantic import BaseModel, ConfigDict, Field @@ -8,41 +8,42 @@ class CompatibleService(BaseModel): - key: ServiceKey | None = Field( - default=None, - description="If None, it refer to current service. Used only for inter-service compatibility", - ) + key: Annotated[ + ServiceKey | None, + Field( + description="If None, it refer to current service. Used only for inter-service compatibility" + ), + ] = None version: ServiceVersion class Compatibility(BaseModel): - # NOTE: as an object it is more maintainable than a list - can_update_to: CompatibleService = Field( - ..., description="Latest compatible service at this moment" - ) + can_update_to: Annotated[ + CompatibleService, Field(description="Latest compatible service at this moment") + ] model_config = ConfigDict(alias_generator=snake_to_camel, populate_by_name=True) class ServiceRelease(BaseModel): - # from ServiceMetaDataPublished version: ServiceVersion - version_display: str | None = Field( - default=None, description="If None, then display `version`" - ) - released: datetime | None = Field( - default=None, description="When provided, it indicates the release timestamp" - ) - retired: datetime | None = Field( - default=None, - description="whether this service is planned to be retired. " - "If None, the service is still active. " - "If now tuple[dict[str, Any], ...]: # type: ignore + """ + Returns a fake factory that creates catalog DATA that can be used to fill + both services_meta_data and services_access_rights tables + + + Example: + fake_service, *fake_access_rights = create_fake_service_data( + "simcore/services/dynamic/jupyterlab", + "0.0.1", + team_access="xw", + everyone_access="x", + product=target_product, + ), + + owner_access, team_access, everyone_access = fake_access_rights + + """ diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py index d278bb350ba..be5e7c6c4e4 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py @@ -11,3 +11,7 @@ class CatalogItemNotFoundError(CatalogApiBaseError): class CatalogForbiddenError(CatalogApiBaseError): msg_template = "Insufficient access rights for {name}" + + +class CatalogNotAvailableError(CatalogApiBaseError): + msg_template = "Catalog service failed unexpectedly" diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py index ed7d4662a00..ae137dcd1b9 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py @@ -4,7 +4,12 @@ from typing import Any, cast from models_library.api_schemas_catalog import CATALOG_RPC_NAMESPACE -from models_library.api_schemas_catalog.services import ServiceGetV2, ServiceUpdateV2 +from models_library.api_schemas_catalog.services import ( + MyServiceGet, + ServiceGetV2, + ServiceListItem, + ServiceUpdateV2, +) from models_library.products import ProductName from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rpc_pagination import ( @@ -30,7 +35,7 @@ async def list_services_paginated( # pylint: disable=too-many-arguments user_id: UserID, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, offset: NonNegativeInt = 0, -) -> PageRpc[ServiceGetV2]: +) -> PageRpc[ServiceListItem]: """ Raises: ValidationError: on invalid arguments @@ -57,10 +62,10 @@ async def _call( result = await _call( product_name=product_name, user_id=user_id, limit=limit, offset=offset ) - assert ( - TypeAdapter(PageRpc[ServiceGetV2]).validate_python(result) is not None - ) # nosec - return cast(PageRpc[ServiceGetV2], result) + assert ( # nosec + TypeAdapter(PageRpc[ServiceListItem]).validate_python(result) is not None + ) + return cast(PageRpc[ServiceListItem], result) @log_decorator(_logger, level=logging.DEBUG) @@ -191,3 +196,42 @@ async def _call( service_key=service_key, service_version=service_version, ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def batch_get_my_services( + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + ids: list[ + tuple[ + ServiceKey, + ServiceVersion, + ] + ], +) -> list[MyServiceGet]: + """ + Raises: + ValidationError: on invalid arguments + CatalogForbiddenError: no access-rights to list services + """ + + @validate_call() + async def _call( + product_name: ProductName, + user_id: UserID, + ids: list[tuple[ServiceKey, ServiceVersion]], + ): + return await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("batch_get_my_services"), + product_name=product_name, + user_id=user_id, + ids=ids, + timeout_s=40 * RPC_REQUEST_DEFAULT_TIMEOUT_S, + ) + + result = await _call(product_name=product_name, user_id=user_id, ids=ids) + assert TypeAdapter(list[MyServiceGet]).validate_python(result) is not None # nosec + return cast(list[MyServiceGet], result) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index ec1b5dca6b1..4f77c0f8a49 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -152,7 +152,7 @@ async def cached_registry_services() -> dict[str, Any]: services_owner_emails, ) = await asyncio.gather( cached_registry_services(), - services_repo.list_services_access_rights( + services_repo.batch_get_services_access_rights( key_versions=services_in_db, product_name=x_simcore_products_name, ), diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 0cd2870fb0f..6d95fbeb962 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -4,6 +4,7 @@ from fastapi import FastAPI from models_library.api_schemas_catalog.services import ( + MyServiceGet, PageRpcServicesGetV2, ServiceGetV2, ServiceUpdateV2, @@ -12,7 +13,7 @@ from models_library.rpc_pagination import DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, PageLimitInt from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import NonNegativeInt +from pydantic import NonNegativeInt, ValidationError, validate_call from pyinstrument import Profiler from servicelib.logging_utils import log_decorator from servicelib.rabbitmq import RPCRouter @@ -20,6 +21,7 @@ CatalogForbiddenError, CatalogItemNotFoundError, ) +from simcore_service_catalog.db.repositories.groups import GroupsRepository from ...db.repositories.services import ServicesRepository from ...services import services_api @@ -51,8 +53,9 @@ async def _wrapper(app: FastAPI, **kwargs): return _wrapper -@router.expose(reraise_if_error_type=(CatalogForbiddenError,)) +@router.expose(reraise_if_error_type=(CatalogForbiddenError, ValidationError)) @_profile_rpc_call +@validate_call(config={"arbitrary_types_allowed": True}) async def list_services_paginated( app: FastAPI, *, @@ -86,9 +89,16 @@ async def list_services_paginated( ) -@router.expose(reraise_if_error_type=(CatalogItemNotFoundError, CatalogForbiddenError)) +@router.expose( + reraise_if_error_type=( + CatalogItemNotFoundError, + CatalogForbiddenError, + ValidationError, + ) +) @log_decorator(_logger, level=logging.DEBUG) @_profile_rpc_call +@validate_call(config={"arbitrary_types_allowed": True}) async def get_service( app: FastAPI, *, @@ -114,8 +124,15 @@ async def get_service( return service -@router.expose(reraise_if_error_type=(CatalogItemNotFoundError, CatalogForbiddenError)) +@router.expose( + reraise_if_error_type=( + CatalogItemNotFoundError, + CatalogForbiddenError, + ValidationError, + ) +) @log_decorator(_logger, level=logging.DEBUG) +@validate_call(config={"arbitrary_types_allowed": True}) async def update_service( app: FastAPI, *, @@ -145,8 +162,15 @@ async def update_service( return service -@router.expose(reraise_if_error_type=(CatalogItemNotFoundError, CatalogForbiddenError)) +@router.expose( + reraise_if_error_type=( + CatalogItemNotFoundError, + CatalogForbiddenError, + ValidationError, + ) +) @log_decorator(_logger, level=logging.DEBUG) +@validate_call(config={"arbitrary_types_allowed": True}) async def check_for_service( app: FastAPI, *, @@ -165,3 +189,33 @@ async def check_for_service( service_key=service_key, service_version=service_version, ) + + +@router.expose(reraise_if_error_type=(CatalogForbiddenError, ValidationError)) +@log_decorator(_logger, level=logging.DEBUG) +@validate_call(config={"arbitrary_types_allowed": True}) +async def batch_get_my_services( + app: FastAPI, + *, + product_name: ProductName, + user_id: UserID, + ids: list[ + tuple[ + ServiceKey, + ServiceVersion, + ] + ], +) -> list[MyServiceGet]: + assert app.state.engine # nosec + + services = await services_api.batch_get_my_services( + repo=ServicesRepository(app.state.engine), + groups_repo=GroupsRepository(app.state.engine), + product_name=product_name, + user_id=user_id, + ids=ids, + ) + + assert [(sv.key, sv.release.version) for sv in services] == ids # nosec + + return services diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/db/repositories/services.py index 1cae7e1c43b..96433c83dd1 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/services.py @@ -179,7 +179,7 @@ async def get_service( product_name: str | None = None, ) -> ServiceMetaDataDBGet | None: - query = sa.select(SERVICES_META_DATA_COLS) + query = sa.select(*SERVICES_META_DATA_COLS) if gids or execute_access or write_access: conditions = [ @@ -481,7 +481,7 @@ async def get_service_access_rights( async for row in await conn.stream(query) ] - async def list_services_access_rights( + async def batch_get_services_access_rights( self, key_versions: Iterable[tuple[str, str]], product_name: str | None = None, diff --git a/services/catalog/src/simcore_service_catalog/models/services_db.py b/services/catalog/src/simcore_service_catalog/models/services_db.py index 902aae32993..5ddcbe239ef 100644 --- a/services/catalog/src/simcore_service_catalog/models/services_db.py +++ b/services/catalog/src/simcore_service_catalog/models/services_db.py @@ -3,6 +3,7 @@ from common_library.basic_types import DEFAULT_FACTORY from models_library.basic_types import IdInt +from models_library.groups import GroupID from models_library.products import ProductName from models_library.services_access import ServiceGroupAccessRights from models_library.services_base import ServiceKeyVersion @@ -10,7 +11,6 @@ from models_library.utils.common_validators import empty_str_to_none_pre_validator from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic.config import JsonDict -from pydantic.types import PositiveInt from simcore_postgres_database.models.services_compatibility import CompatiblePolicyDict @@ -20,7 +20,7 @@ class ServiceMetaDataDBGet(BaseModel): version: ServiceVersion # ownership - owner: IdInt | None + owner: GroupID | None # display name: str @@ -208,7 +208,7 @@ class ServiceWithHistoryDBGet(BaseModel): class ServiceAccessRightsAtDB(ServiceKeyVersion, ServiceGroupAccessRights): - gid: PositiveInt + gid: GroupID product_name: ProductName @staticmethod diff --git a/services/catalog/src/simcore_service_catalog/services/compatibility.py b/services/catalog/src/simcore_service_catalog/services/compatibility.py index f3d9b6c680a..db8483e11c9 100644 --- a/services/catalog/src/simcore_service_catalog/services/compatibility.py +++ b/services/catalog/src/simcore_service_catalog/services/compatibility.py @@ -96,9 +96,13 @@ async def evaluate_service_compatibility_map( user_id: UserID, service_release_history: list[ReleaseDBGet], ) -> dict[ServiceVersion, Compatibility | None]: - released_versions = _convert_to_versions(service_release_history) - result: dict[ServiceVersion, Compatibility | None] = {} + """ + Evaluates the compatibility among a list of service releases for a given product and user. + """ + compatibility_map: dict[ServiceVersion, Compatibility | None] = {} + + released_versions = _convert_to_versions(service_release_history) for release in service_release_history: compatibility = None if release.compatibility_policy: @@ -108,7 +112,7 @@ async def evaluate_service_compatibility_map( repo=repo, target_version=release.version, released_versions=released_versions, - compatibility_policy={**release.compatibility_policy}, + compatibility_policy=dict(release.compatibility_policy), ) elif latest_version := _get_latest_compatible_version( release.version, @@ -117,6 +121,6 @@ async def evaluate_service_compatibility_map( compatibility = Compatibility( can_update_to=CompatibleService(version=f"{latest_version}") ) - result[release.version] = compatibility + compatibility_map[release.version] = compatibility - return result + return compatibility_map diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index 27368a7565f..11f8d57644a 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -1,6 +1,13 @@ import logging +from contextlib import suppress -from models_library.api_schemas_catalog.services import ServiceGetV2, ServiceUpdateV2 +from models_library.api_schemas_catalog.services import ( + MyServiceGet, + ServiceGetV2, + ServiceListItem, + ServiceUpdateV2, +) +from models_library.groups import GroupID from models_library.products import ProductName from models_library.rest_pagination import PageLimitInt from models_library.services_access import ServiceGroupAccessRightsV2 @@ -13,6 +20,7 @@ CatalogForbiddenError, CatalogItemNotFoundError, ) +from simcore_service_catalog.db.repositories.groups import GroupsRepository from ..db.repositories.services import ServicesRepository from ..models.services_db import ( @@ -45,7 +53,7 @@ def _db_to_api_model( description=service_db.description, description_ui=service_db.description_ui, version_display=service_db.version_display, - type=service_manifest.service_type, + service_type=service_manifest.service_type, contact=service_manifest.contact, authors=service_manifest.authors, owner=(service_db.owner_email if service_db.owner_email else None), @@ -82,7 +90,7 @@ async def list_services_paginated( user_id: UserID, limit: PageLimitInt | None, offset: NonNegativeInt = 0, -) -> tuple[NonNegativeInt, list[ServiceGetV2]]: +) -> tuple[NonNegativeInt, list[ServiceListItem]]: # defines the order total_count, services = await repo.list_latest_services( @@ -91,10 +99,10 @@ async def list_services_paginated( if services: # injects access-rights - access_rights: dict[ - tuple[str, str], list[ServiceAccessRightsAtDB] - ] = await repo.list_services_access_rights( - ((s.key, s.version) for s in services), product_name=product_name + access_rights: dict[tuple[str, str], list[ServiceAccessRightsAtDB]] = ( + await repo.batch_get_services_access_rights( + ((s.key, s.version) for s in services), product_name=product_name + ) ) if not access_rights: raise CatalogForbiddenError( @@ -114,12 +122,15 @@ async def list_services_paginated( items = [ _db_to_api_model( - service_db=s, access_rights_db=ar, service_manifest=sm, compatibility_map=cm + service_db=sc, + access_rights_db=ar, + service_manifest=sm, + compatibility_map=cm, ) - for s in services + for sc in services if ( - (ar := access_rights.get((s.key, s.version))) - and (sm := service_manifest.get((s.key, s.version))) + (ar := access_rights.get((sc.key, sc.version))) + and (sm := service_manifest.get((sc.key, sc.version))) and ( # NOTE: This operation might be resource-intensive. # It is temporarily implemented on a trial basis. @@ -127,13 +138,20 @@ async def list_services_paginated( repo, product_name=product_name, user_id=user_id, - service_release_history=s.history, + service_release_history=sc.history, ) ) ) ] - return total_count, items + return total_count, [ + ServiceListItem.model_validate( + { + **it.model_dump(exclude_unset=True, by_alias=True), + } + ) + for it in items + ] async def get_service( @@ -333,3 +351,93 @@ async def check_for_service( user_id=user_id, product_name=product_name, ) + + +async def batch_get_my_services( + repo: ServicesRepository, + groups_repo: GroupsRepository, + *, + product_name: ProductName, + user_id: UserID, + ids: list[ + tuple[ + ServiceKey, + ServiceVersion, + ] + ], +) -> list[MyServiceGet]: + + services_access_rights = await repo.batch_get_services_access_rights( + key_versions=ids, product_name=product_name + ) + + user_groups = await groups_repo.list_user_groups(user_id=user_id) + my_group_ids = {g.gid for g in user_groups} + + my_services = [] + for service_key, service_version in ids: + + # Evaluate user's access-rights to this service key:version + access_rights = services_access_rights.get((service_key, service_version), []) + my_access_rights = ServiceGroupAccessRightsV2(execute=False, write=False) + for ar in access_rights: + if ar.gid in my_group_ids: + my_access_rights.execute |= ar.execute_access + my_access_rights.write |= ar.write_access + + # Get service metadata + service_db = await repo.get_service( + product_name=product_name, + key=service_key, + version=service_version, + ) + assert service_db # nosec + + # Find service owner (if defined!) + owner: GroupID | None = service_db.owner + if not owner: + # NOTE can be more than one. Just get first. + with suppress(StopIteration): + owner = next( + ar.gid + for ar in access_rights + if ar.write_access and ar.execute_access + ) + + # Evaluate `compatibility` + compatibility: Compatibility | None = None + if my_access_rights.execute or my_access_rights.write: + history = await repo.get_service_history( + # NOTE: that the service history might be different for each user + # since access rights are defined on a k:v basis + product_name=product_name, + user_id=user_id, + key=service_key, + ) + assert history # nosec + + compatibility_map = await evaluate_service_compatibility_map( + repo, + product_name=product_name, + user_id=user_id, + service_release_history=history, + ) + + compatibility = compatibility_map.get(service_db.version) + + my_services.append( + MyServiceGet( + key=service_db.key, + release=ServiceRelease( + version=service_db.version, + version_display=service_db.version_display, + released=service_db.created, + retired=service_db.deprecated, + compatibility=compatibility, + ), + owner=owner, + my_access_rights=my_access_rights, + ) + ) + + return my_services diff --git a/services/catalog/tests/unit/with_dbs/conftest.py b/services/catalog/tests/unit/with_dbs/conftest.py index 1bd0bb27e50..74daa97804d 100644 --- a/services/catalog/tests/unit/with_dbs/conftest.py +++ b/services/catalog/tests/unit/with_dbs/conftest.py @@ -18,9 +18,11 @@ from models_library.services import ServiceMetaDataPublished from models_library.users import UserID from pydantic import ConfigDict, TypeAdapter +from pytest_simcore.helpers.catalog_services import CreateFakeServiceDataCallable from pytest_simcore.helpers.faker_factories import ( random_service_access_rights, random_service_meta_data, + random_user, ) from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.postgres_tools import ( @@ -159,6 +161,24 @@ async def user( yield row +@pytest.fixture +async def other_user( + user_id: UserID, + sqlalchemy_async_engine: AsyncEngine, + faker: Faker, +) -> AsyncIterator[dict[str, Any]]: + + _user = random_user(fake=faker, id=user_id + 1) + async with insert_and_get_row_lifespan( # pylint:disable=contextmanager-generator-missing-cleanup + sqlalchemy_async_engine, + table=users, + values=_user, + pk_col=users.c.id, + pk_value=_user["id"], + ) as row: + yield row + + @pytest.fixture() async def user_groups_ids( sqlalchemy_async_engine: AsyncEngine, user: dict[str, Any] @@ -354,12 +374,12 @@ def _fake_factory(**overrides): return _fake_factory -@pytest.fixture() +@pytest.fixture async def create_fake_service_data( user_groups_ids: list[int], products_names: list[str], faker: Faker, -) -> Callable: +) -> CreateFakeServiceDataCallable: """Returns a fake factory that creates catalog DATA that can be used to fill both services_meta_data and services_access_rights tables @@ -376,11 +396,11 @@ async def create_fake_service_data( owner_access, team_access, everyone_access = fake_access_rights """ - everyone_gid, user_gid, team_gid = user_groups_ids + everyone_gid, user_primary_gid, team_standard_gid = user_groups_ids def _random_service(**overrides) -> dict[str, Any]: return random_service_meta_data( - owner_primary_gid=user_gid, + owner_primary_gid=user_primary_gid, fake=faker, **overrides, ) @@ -396,9 +416,9 @@ def _random_access(service, **overrides) -> dict[str, Any]: def _fake_factory( key, version, - team_access=None, - everyone_access=None, - product=products_names[0], + team_access: str | None = None, + everyone_access: str | None = None, + product: ProductName = products_names[0], deprecated: datetime | None = None, ) -> tuple[dict[str, Any], ...]: service = _random_service(key=key, version=version, deprecated=deprecated) @@ -420,7 +440,7 @@ def _fake_factory( fakes.append( _random_access( service, - gid=team_gid, + gid=team_standard_gid, execute_access="x" in team_access, write_access="w" in team_access, product_name=product, diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py index 301c64bf7be..732dda730c1 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py @@ -240,7 +240,9 @@ async def test_list_services_that_are_deprecated( ): # injects fake data in db - deprecation_date = datetime.utcnow() + timedelta(days=1) + deprecation_date = datetime.utcnow() + timedelta( # NOTE: old offset-naive column + days=1 + ) deprecated_service = create_fake_service_data( "simcore/services/dynamic/jupyterlab", "1.0.1", diff --git a/services/catalog/tests/unit/with_dbs/test_api_rpc.py b/services/catalog/tests/unit/with_dbs/test_api_rpc.py index 3605eeae7f0..65b6dd70b11 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rpc.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rpc.py @@ -5,7 +5,7 @@ # pylint: disable=unused-variable -from collections.abc import AsyncIterator, Callable +from collections.abc import Callable from typing import Any import pytest @@ -16,9 +16,8 @@ from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID from pydantic import ValidationError -from pytest_simcore.helpers.faker_factories import random_icon_url, random_user +from pytest_simcore.helpers.faker_factories import random_icon_url from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict -from pytest_simcore.helpers.postgres_tools import insert_and_get_row_lifespan from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter from servicelib.rabbitmq import RabbitMQRPCClient @@ -27,13 +26,12 @@ CatalogItemNotFoundError, ) from servicelib.rabbitmq.rpc_interfaces.catalog.services import ( + batch_get_my_services, check_for_service, get_service, list_services_paginated, update_service, ) -from simcore_postgres_database.models.users import users -from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = [ "rabbit", @@ -164,8 +162,8 @@ async def test_rpc_catalog_client( assert got.key == service_key assert got.version == service_version - assert got == next( - item + assert got.model_dump() == next( + item.model_dump() for item in page.data if (item.key == service_key and item.version == service_version) ) @@ -260,24 +258,6 @@ async def test_rpc_check_for_service( ) -@pytest.fixture -async def other_user( - user_id: UserID, - sqlalchemy_async_engine: AsyncEngine, - faker: Faker, -) -> AsyncIterator[dict[str, Any]]: - - _user = random_user(fake=faker, id=user_id + 1) - async with insert_and_get_row_lifespan( # pylint:disable=contextmanager-generator-missing-cleanup - sqlalchemy_async_engine, - table=users, - values=_user, - pk_col=users.c.id, - pk_value=_user["id"], - ) as row: - yield row - - async def test_rpc_get_service_access_rights( background_sync_task_mocked: None, mocked_director_service_api: MockRouter, @@ -418,3 +398,82 @@ async def test_rpc_get_service_access_rights( "name": "foo", "description": "bar", } + + +async def test_rpc_batch_get_my_services( + background_sync_task_mocked: None, + mocked_director_service_api: MockRouter, + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + user: dict[str, Any], + user_id: UserID, + app: FastAPI, + create_fake_service_data: Callable, + services_db_tables_injector: Callable, +): + # Create fake services data + service_key = "simcore/services/comp/test-batch-service" + service_version_1 = "1.0.0" + service_version_2 = "1.0.5" + + other_service_key = "simcore/services/comp/other-batch-service" + other_service_version = "1.0.0" + + fake_service_1 = create_fake_service_data( + service_key, + service_version_1, + team_access=None, + everyone_access=None, + product=product_name, + ) + fake_service_2 = create_fake_service_data( + service_key, + service_version_2, + team_access="x", + everyone_access=None, + product=product_name, + ) + fake_service_3 = create_fake_service_data( + other_service_key, + other_service_version, + team_access=None, + everyone_access=None, + product=product_name, + ) + + # Inject fake services into the database + await services_db_tables_injector([fake_service_1, fake_service_2, fake_service_3]) + + # Batch get my services: project with two, not three + ids = [ + (service_key, service_version_1), + (other_service_key, other_service_version), + ] + + my_services = await batch_get_my_services( + rpc_client, + product_name=product_name, + user_id=user_id, + ids=ids, + ) + + assert len(my_services) == 2 + + # Check access rights to all of them + assert my_services[0].my_access_rights.model_dump() == { + "execute": True, + "write": True, + } + assert my_services[0].owner == user["primary_gid"] + assert my_services[0].key == service_key + assert my_services[0].release.version == service_version_1 + assert my_services[0].release.compatibility + assert ( + my_services[0].release.compatibility.can_update_to.version == service_version_2 + ) + + assert my_services[1].my_access_rights.model_dump() == { + "execute": True, + "write": True, + } + assert my_services[1].owner == user["primary_gid"] diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py index bcfae48d319..a2d3c2551c7 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -1,16 +1,23 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable +# pylint: disable=too-many-arguments + from collections.abc import Callable +from datetime import datetime, timedelta from typing import Any import pytest from fastapi import FastAPI +from models_library.api_schemas_catalog.services import MyServiceGet from models_library.products import ProductName from models_library.users import UserID +from pydantic import TypeAdapter +from pytest_simcore.helpers.catalog_services import CreateFakeServiceDataCallable from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api +from simcore_service_catalog.db.repositories.groups import GroupsRepository from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.services import manifest, services_api from simcore_service_catalog.services.director import DirectorApi @@ -29,6 +36,11 @@ def services_repo(sqlalchemy_async_engine: AsyncEngine): return ServicesRepository(sqlalchemy_async_engine) +@pytest.fixture +def groups_repo(sqlalchemy_async_engine: AsyncEngine): + return GroupsRepository(sqlalchemy_async_engine) + + @pytest.fixture def num_services() -> int: return 5 @@ -78,7 +90,10 @@ async def background_sync_task_mocked( services_db_tables_injector: Callable, fake_services_data: list, ) -> None: - # inject db services (typically done by the sync background task) + """ + Emulates a sync backgroundtask that injects + some services in the db + """ await services_db_tables_injector(fake_services_data) @@ -139,7 +154,120 @@ async def test_list_services_paginated( service_version=item.version, ) - assert got == item + assert got.model_dump() == item.model_dump() # since it is cached, it should only call it `limit` times assert mocked_director_service_api["get_service"].call_count == limit + + +async def test_batch_get_my_services( + background_tasks_setup_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_service_api: MockRouter, + target_product: ProductName, + services_repo: ServicesRepository, + groups_repo: GroupsRepository, + user_id: UserID, + user: dict[str, Any], + other_user: dict[str, Any], + create_fake_service_data: CreateFakeServiceDataCallable, + services_db_tables_injector: Callable, +): + # catalog + service_key = "simcore/services/comp/some-service" + service_version_1 = "1.0.0" # can upgrade to 1.0.1 + service_version_2 = "1.0.10" # latest + + other_service_key = "simcore/services/comp/other-service" + other_service_version = "2.1.2" + + expected_retirement = datetime.utcnow() + timedelta( + days=1 + ) # NOTE: old offset-naive column + + # Owned by user + fake_service_1 = create_fake_service_data( + service_key, + service_version_1, + team_access=None, + everyone_access=None, + product=target_product, + deprecated=expected_retirement, + ) + fake_service_2 = create_fake_service_data( + service_key, + service_version_2, + team_access="x", + everyone_access=None, + product=target_product, + ) + + # Owned by other-user + fake_service_3 = create_fake_service_data( + other_service_key, + other_service_version, + team_access=None, + everyone_access=None, + product=target_product, + ) + _service, _owner_access = fake_service_3 + _service["owner"] = other_user["primary_gid"] + _owner_access["gid"] = other_user["primary_gid"] + + # Inject fake services into the database + await services_db_tables_injector([fake_service_1, fake_service_2, fake_service_3]) + + # UNDER TEST ------------------------------- + + # Batch get services e.g. services in a project + services_ids = [ + (service_key, service_version_1), + (other_service_key, other_service_version), + ] + + my_services = await services_api.batch_get_my_services( + services_repo, + groups_repo, + product_name=target_product, + user_id=user_id, + ids=services_ids, + ) + + # CHECKS ------------------------------- + + # assert returned order and length as ids + assert services_ids == [(sc.key, sc.release.version) for sc in my_services] + + assert my_services == TypeAdapter(list[MyServiceGet]).validate_python( + [ + { + "key": "simcore/services/comp/some-service", + "release": { + "version": service_version_1, + "version_display": None, + "released": my_services[0].release.released, + "retired": expected_retirement, + "compatibility": { + "can_update_to": {"version": service_version_2} + }, # can be updated + }, + "owner": user["primary_gid"], + "my_access_rights": {"execute": True, "write": True}, # full access + }, + { + "key": "simcore/services/comp/other-service", + "release": { + "version": other_service_version, + "version_display": None, + "released": my_services[1].release.released, + "retired": None, + "compatibility": None, # cannot be updated + }, + "owner": other_user["primary_gid"], # needs to request access + "my_access_rights": { + "execute": False, + "write": False, + }, + }, + ] + ) diff --git a/services/web/server/VERSION b/services/web/server/VERSION index 7e750b4ebf3..0b09455034e 100644 --- a/services/web/server/VERSION +++ b/services/web/server/VERSION @@ -1 +1 @@ -0.60.0 +0.61.0 diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 6012cf501a1..65b8a29b46b 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.60.0 +current_version = 0.61.0 commit = True message = services/webserver api version: {current_version} → {new_version} tag = False diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 1cd96ab7b28..1fc7d0cad53 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.1.0 info: title: simcore-service-webserver description: Main service with an interface (http-API & websockets) to the web front-end - version: 0.60.0 + version: 0.61.0 servers: - url: '' description: webserver @@ -2027,7 +2027,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Page_CatalogServiceGet_' + $ref: '#/components/schemas/Page_CatalogServiceListItem_' /v0/catalog/services/{service_key}/{service_version}: get: tags: @@ -4927,6 +4927,28 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_dict_Annotated_str__StringConstraints___ImageResources__' + /v0/projects/{project_id}/nodes/-/services: + get: + tags: + - projects + - nodes + summary: Get Project Services + operationId: get_project_services + parameters: + - name: project_id + in: path + required: true + schema: + type: string + format: uuid + title: Project Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_ProjectNodeServicesGet_' /v0/projects/{project_id}/nodes/-/services:access: get: tags: @@ -8034,6 +8056,121 @@ components: type: computational version: 2.2.1 version_display: 2 Xtreme + CatalogServiceListItem: + properties: + key: + type: string + pattern: ^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$ + title: Key + version: + type: string + pattern: ^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$ + title: Version + name: + type: string + title: Name + thumbnail: + anyOf: + - type: string + - type: 'null' + title: Thumbnail + icon: + anyOf: + - type: string + - type: 'null' + title: Icon + description: + type: string + title: Description + descriptionUi: + type: boolean + title: Descriptionui + default: false + versionDisplay: + anyOf: + - type: string + - type: 'null' + title: Versiondisplay + type: + $ref: '#/components/schemas/ServiceType' + contact: + anyOf: + - type: string + format: email + - type: 'null' + title: Contact + authors: + items: + $ref: '#/components/schemas/Author' + type: array + minItems: 1 + title: Authors + owner: + anyOf: + - type: string + format: email + - type: 'null' + title: Owner + description: None when the owner email cannot be found in the database + inputs: + type: object + title: Inputs + outputs: + type: object + title: Outputs + bootOptions: + anyOf: + - type: object + - type: 'null' + title: Bootoptions + minVisibleInputs: + anyOf: + - type: integer + minimum: 0 + - type: 'null' + title: Minvisibleinputs + accessRights: + anyOf: + - additionalProperties: + $ref: '#/components/schemas/ServiceGroupAccessRightsV2' + type: object + - type: 'null' + title: Accessrights + classifiers: + anyOf: + - items: + type: string + type: array + - type: 'null' + title: Classifiers + default: [] + quality: + type: object + title: Quality + default: {} + history: + items: + $ref: '#/components/schemas/ServiceRelease' + type: array + title: History + description: History will be replaced by current 'release' instead + default: [] + deprecated: true + additionalProperties: false + type: object + required: + - key + - version + - name + - description + - type + - contact + - authors + - owner + - inputs + - outputs + - accessRights + title: CatalogServiceListItem CatalogServiceUpdate: properties: name: @@ -9107,6 +9244,19 @@ components: title: Error type: object title: Envelope[ProjectMetadataGet] + Envelope_ProjectNodeServicesGet_: + properties: + data: + anyOf: + - $ref: '#/components/schemas/ProjectNodeServicesGet' + - type: 'null' + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[ProjectNodeServicesGet] Envelope_ProjectState_: properties: data: @@ -10151,6 +10301,22 @@ components: - resource - field title: ErrorItemType + ExecutableAccessRights: + properties: + write: + type: boolean + title: Write + description: can change executable settings + execute: + type: boolean + title: Execute + description: can run executable + additionalProperties: false + type: object + required: + - write + - execute + title: ExecutableAccessRights FeaturesDict: properties: age: @@ -12062,6 +12228,32 @@ components: - thumbnail_url - file_url title: NodeScreenshot + NodeServiceGet: + properties: + key: + type: string + pattern: ^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$ + title: Key + release: + $ref: '#/components/schemas/ServiceRelease' + owner: + anyOf: + - type: integer + exclusiveMinimum: true + minimum: 0 + - type: 'null' + title: Owner + description: Service owner primary group id or None if ownership still not + defined + myAccessRights: + $ref: '#/components/schemas/ExecutableAccessRights' + type: object + required: + - key + - release + - owner + - myAccessRights + title: NodeServiceGet NodeState: properties: modified: @@ -12208,7 +12400,7 @@ components: - total - count title: PageMetaInfoLimitOffset - Page_CatalogServiceGet_: + Page_CatalogServiceListItem_: properties: _meta: $ref: '#/components/schemas/PageMetaInfoLimitOffset' @@ -12216,7 +12408,7 @@ components: $ref: '#/components/schemas/PageLinks' data: items: - $ref: '#/components/schemas/CatalogServiceGet' + $ref: '#/components/schemas/CatalogServiceListItem' type: array title: Data additionalProperties: false @@ -12225,7 +12417,7 @@ components: - _meta - _links - data - title: Page[CatalogServiceGet] + title: Page[CatalogServiceListItem] Page_LicensedItemPurchaseGet_: properties: _meta: @@ -13550,6 +13742,22 @@ components: required: - custom title: ProjectMetadataUpdate + ProjectNodeServicesGet: + properties: + projectUuid: + type: string + format: uuid + title: Projectuuid + services: + items: + $ref: '#/components/schemas/NodeServiceGet' + type: array + title: Services + type: object + required: + - projectUuid + - services + title: ProjectNodeServicesGet ProjectOutputGet: properties: key: @@ -14402,9 +14610,9 @@ components: format: date-time - type: 'null' title: Retired - description: 'whether this service is planned to be retired. If None, the + description: whether this service is planned to be retired. If None, the service is still active. If now list[MyServiceGet]: + try: + + return await catalog_rpc.batch_get_my_services( + get_rabbitmq_rpc_client(app), + user_id=user_id, + product_name=product_name, + ids=services_ids, + ) + except RPCServerError as err: + raise CatalogNotAvailableError( + user_id=user_id, + product_name=product_name, + ) from err + + async def get_service_v2( app: web.Application, *, diff --git a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py new file mode 100644 index 00000000000..e0dab71aaf2 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py @@ -0,0 +1,5 @@ +from ._api import batch_get_my_services + +__all__: tuple[str, ...] = ("batch_get_my_services",) + +# nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/projects/_common/exceptions_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_common/exceptions_handlers.py index d9116350236..ad2db124670 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_common/exceptions_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_common/exceptions_handlers.py @@ -6,6 +6,7 @@ from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( CatalogForbiddenError, CatalogItemNotFoundError, + CatalogNotAvailableError, ) from ...exception_handling import ( @@ -163,6 +164,10 @@ _OTHER_ERRORS: ExceptionToHttpErrorMap = { + CatalogNotAvailableError: HttpErrorInfo( + status.HTTP_503_SERVICE_UNAVAILABLE, + "This service is currently not available", + ), ClustersKeeperNotAvailableError: HttpErrorInfo( status.HTTP_503_SERVICE_UNAVAILABLE, "Clusters-keeper service is not available", diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py index 6b4b8df3f90..0206e1315cc 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py @@ -12,6 +12,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID, SimCoreFileLink +from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID from pydantic import ( BaseModel, @@ -26,6 +27,7 @@ from ..application_settings import get_application_settings from ..storage.api import get_download_link, get_files_in_node_folder +from . import _nodes_repository from .exceptions import ProjectStartsTooManyDynamicNodesError _logger = logging.getLogger(__name__) @@ -71,6 +73,14 @@ def get_total_project_dynamic_nodes_creation_interval( return max_nodes * _NODE_START_INTERVAL_S.total_seconds() +async def get_project_nodes_services( + app: web.Application, *, project_uuid: ProjectID +) -> list[tuple[ServiceKey, ServiceVersion]]: + return await _nodes_repository.get_project_nodes_services( + app, project_uuid=project_uuid + ) + + # # PREVIEWS # diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 28fcb974511..986090a224a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -1,6 +1,4 @@ -""" Handlers for CRUD operations on /projects/{*}/nodes/{*} - -""" +"""Handlers for CRUD operations on /projects/{*}/nodes/{*}""" import asyncio import logging @@ -10,6 +8,7 @@ from models_library.api_schemas_catalog.service_access_rights import ( ServiceAccessRightsGet, ) +from models_library.api_schemas_catalog.services import MyServiceGet from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStop, @@ -23,12 +22,15 @@ NodeOutputs, NodePatch, NodeRetrieve, + NodeServiceGet, + ProjectNodeServicesGet, ) from models_library.groups import EVERYONE_GROUP_ID, Group, GroupID, GroupType from models_library.projects import Project, ProjectID from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.services import ServiceKeyVersion from models_library.services_resources import ServiceResourcesDict +from models_library.services_types import ServiceKey, ServiceVersion from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import BaseModel, Field from servicelib.aiohttp import status @@ -55,6 +57,7 @@ from simcore_postgres_database.models.users import UserRole from .._meta import API_VTAG as VTAG +from ..catalog import catalog_service from ..catalog import client as catalog_client from ..dynamic_scheduler import api as dynamic_scheduler_api from ..groups.api import get_group_from_gid, list_all_user_groups_ids @@ -64,6 +67,8 @@ from ..security.decorators import permission_required from ..users.api import get_user_id_from_gid, get_user_role from ..utils_aiohttp import envelope_json_response +from . import _access_rights_api as access_rights_service +from . import _nodes_api as _nodes_service from . import nodes_utils, projects_service from ._common.exceptions_handlers import handle_plugin_requests_exceptions from ._common.models import ProjectPathParams, RequestContext @@ -464,6 +469,49 @@ class _ProjectGroupAccess(BaseModel): inaccessible_services: list[ServiceKeyVersion] | None = Field(default=None) +@routes.get( + f"/{VTAG}/projects/{{project_id}}/nodes/-/services", + name="get_project_services", +) +@login_required +@permission_required("project.read") +@handle_plugin_requests_exceptions +async def get_project_services(request: web.Request) -> web.Response: + req_ctx = RequestContext.model_validate(request) + path_params = parse_request_path_parameters_as(ProjectPathParams, request) + + await access_rights_service.check_user_project_permission( + request.app, + product_name=req_ctx.product_name, + user_id=req_ctx.user_id, + project_id=path_params.project_id, + permission="read", + ) + + services_in_project: list[tuple[ServiceKey, ServiceVersion]] = ( + await _nodes_service.get_project_nodes_services( + request.app, project_uuid=path_params.project_id + ) + ) + + services: list[MyServiceGet] = await catalog_service.batch_get_my_services( + request.app, + product_name=req_ctx.product_name, + user_id=req_ctx.user_id, + services_ids=services_in_project, + ) + + return envelope_json_response( + ProjectNodeServicesGet( + project_uuid=path_params.project_id, + services=[ + NodeServiceGet.model_validate(sv, from_attributes=True) + for sv in services + ], + ) + ) + + @routes.get( f"/{VTAG}/projects/{{project_id}}/nodes/-/services:access", name="get_project_services_access_for_gid", @@ -471,9 +519,7 @@ class _ProjectGroupAccess(BaseModel): @login_required @permission_required("project.read") @handle_plugin_requests_exceptions -async def get_project_services_access_for_gid( - request: web.Request, -) -> web.Response: +async def get_project_services_access_for_gid(request: web.Request) -> web.Response: req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) query_params: _ServicesAccessQuery = parse_request_query_parameters_as( diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py new file mode 100644 index 00000000000..e5060360265 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py @@ -0,0 +1,18 @@ +from aiohttp import web +from models_library.projects import ProjectID +from models_library.services_types import ServiceKey, ServiceVersion +from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo + +from ..db.plugin import get_database_engine + + +async def get_project_nodes_services( + app: web.Application, *, project_uuid: ProjectID +) -> list[tuple[ServiceKey, ServiceVersion]]: + repo = ProjectNodesRepo(project_uuid=project_uuid) + + async with get_database_engine(app).acquire() as conn: + nodes = await repo.list(conn) + + # removes duplicates by preserving order + return list(dict.fromkeys((node.key, node.version) for node in nodes)) diff --git a/services/web/server/src/simcore_service_webserver/tags/errors.py b/services/web/server/src/simcore_service_webserver/tags/errors.py index 95fa3185972..579ed5ef125 100644 --- a/services/web/server/src/simcore_service_webserver/tags/errors.py +++ b/services/web/server/src/simcore_service_webserver/tags/errors.py @@ -1,8 +1,9 @@ +# pylint: disable=too-many-ancestors + from ..errors import WebServerBaseError -class TagsPermissionError(WebServerBaseError, PermissionError): - ... +class TagsPermissionError(WebServerBaseError, PermissionError): ... class ShareTagWithEveryoneNotAllowedError(TagsPermissionError): diff --git a/services/web/server/tests/unit/with_dbs/02/conftest.py b/services/web/server/tests/unit/with_dbs/02/conftest.py index 25be7db87c8..2bd29316680 100644 --- a/services/web/server/tests/unit/with_dbs/02/conftest.py +++ b/services/web/server/tests/unit/with_dbs/02/conftest.py @@ -107,8 +107,8 @@ def mock_catalog_api( @pytest.fixture async def user_project( client: TestClient, - fake_project, - logged_user, + fake_project: ProjectDict, + logged_user: UserInfoDict, tests_data_dir: Path, osparc_product_name: str, ) -> AsyncIterator[ProjectDict]: @@ -223,7 +223,7 @@ async def _creator(**prj_kwargs) -> ProjectDict: @pytest.fixture def fake_services( - create_dynamic_service_mock: Callable[..., Awaitable[DynamicServiceGet]] + create_dynamic_service_mock: Callable[..., Awaitable[DynamicServiceGet]], ) -> Callable[..., Awaitable[list[DynamicServiceGet]]]: async def create_fakes(number_services: int) -> list[DynamicServiceGet]: return [await create_dynamic_service_mock() for _ in range(number_services)] diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py index 3e8a4d9e2b4..cedb64451bb 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py @@ -13,9 +13,13 @@ from models_library.api_schemas_catalog.service_access_rights import ( ServiceAccessRightsGet, ) +from models_library.api_schemas_catalog.services import MyServiceGet +from models_library.services_history import ServiceRelease from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status +from servicelib.rabbitmq import RPCServerError from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects.models import ProjectDict from yarl import URL @@ -95,9 +99,9 @@ def mock_catalog_api_get_service_access_rights_response(mocker: MockerFixture): async def test_user_role_access( client: TestClient, user_project: ProjectDict, - logged_user: dict, + logged_user: UserInfoDict, expected: HTTPStatus, - mock_catalog_api_get_service_access_rights_response, + mock_catalog_api_get_service_access_rights_response: None, ): assert client.app @@ -123,7 +127,7 @@ async def test_accessible_thanks_to_everyone_group_id( client: TestClient, user_project: ProjectDict, mocker: MockerFixture, - logged_user: dict, + logged_user: UserInfoDict, ): mocker.patch( "simcore_service_webserver.projects._nodes_handlers.catalog_client.get_service_access_rights", @@ -176,7 +180,7 @@ async def test_accessible_thanks_to_concrete_group_id( client: TestClient, user_project: ProjectDict, mocker: MockerFixture, - logged_user: dict, + logged_user: UserInfoDict, ): for_gid = logged_user["primary_gid"] @@ -229,7 +233,7 @@ async def test_accessible_through_product_group( client: TestClient, user_project: ProjectDict, mocker: MockerFixture, - logged_user: dict, + logged_user: UserInfoDict, ): for_gid = logged_user["primary_gid"] @@ -288,7 +292,7 @@ async def test_accessible_for_one_service( client: TestClient, user_project: ProjectDict, mocker: MockerFixture, - logged_user: dict, + logged_user: UserInfoDict, ): for_gid = logged_user["primary_gid"] @@ -348,7 +352,7 @@ async def test_not_accessible_for_more_services( client: TestClient, user_project: ProjectDict, mocker: MockerFixture, - logged_user: dict, + logged_user: UserInfoDict, ): mocker.patch( "simcore_service_webserver.projects._nodes_handlers.catalog_client.get_service_access_rights", @@ -412,7 +416,7 @@ async def test_not_accessible_for_service_because_of_execute_access_false( client: TestClient, user_project: ProjectDict, mocker: MockerFixture, - logged_user: dict, + logged_user: UserInfoDict, ): for_gid = logged_user["primary_gid"] @@ -461,3 +465,122 @@ async def test_not_accessible_for_service_because_of_execute_access_false( {"key": "simcore/services/comp/itis/sleeper", "version": "2.1.4"} ], } + + +@pytest.mark.parametrize("user_role", [UserRole.USER]) +async def test_get_project_services( + client: TestClient, + user_project: ProjectDict, + mocker: MockerFixture, + logged_user: UserInfoDict, +): + fake_services_in_project = [ + (sv["key"], sv["version"]) for sv in user_project["workbench"].values() + ] + + mocker.patch( + "simcore_service_webserver.catalog._api.catalog_rpc.batch_get_my_services", + spec=True, + return_value=[ + MyServiceGet( + key=service_key, + release=ServiceRelease( + version=service_version, + version_display=f"v{service_version}", + released="2023-01-01T00:00:00Z", + retired=None, + compatibility=None, + ), + owner=logged_user["primary_gid"], + my_access_rights={"execute": True, "write": False}, + ) + for service_key, service_version in fake_services_in_project + ], + ) + + assert client.app + + project_id = user_project["uuid"] + + expected_url = client.app.router["get_project_services"].url_for( + project_id=project_id + ) + assert URL(f"/v0/projects/{project_id}/nodes/-/services") == expected_url + + resp = await client.get(f"/v0/projects/{project_id}/nodes/-/services") + data, _ = await assert_status(resp, status.HTTP_200_OK) + + assert data == { + "projectUuid": project_id, + "services": [ + { + "key": "simcore/services/comp/itis/sleeper", + "myAccessRights": {"execute": True, "write": False}, + "owner": logged_user["primary_gid"], + "release": { + "compatibility": None, + "released": "2023-01-01T00:00:00+00:00", + "retired": None, + "version": "2.1.4", + "versionDisplay": "v2.1.4", + }, + }, + { + "key": "simcore/services/frontend/parameter/integer", + "myAccessRights": {"execute": True, "write": False}, + "owner": logged_user["primary_gid"], + "release": { + "compatibility": None, + "released": "2023-01-01T00:00:00+00:00", + "retired": None, + "version": "1.0.0", + "versionDisplay": "v1.0.0", + }, + }, + { + "key": "simcore/services/comp/itis/sleeper", + "myAccessRights": {"execute": True, "write": False}, + "owner": logged_user["primary_gid"], + "release": { + "compatibility": None, + "released": "2023-01-01T00:00:00+00:00", + "retired": None, + "version": "2.1.5", + "versionDisplay": "v2.1.5", + }, + }, + ], + } + + +@pytest.mark.parametrize("user_role", [UserRole.USER]) +async def test_get_project_services_service_unavailable( + client: TestClient, + user_project: ProjectDict, + mocker: MockerFixture, + logged_user: UserInfoDict, +): + mocker.patch( + "simcore_service_webserver.catalog._api.catalog_rpc.batch_get_my_services", + spec=True, + side_effect=RPCServerError( + exc_message="Service Unavailable", + method_name="batch_get_my_services", + exc_type="Exception", + ), + ) + + assert client.app + + project_id = user_project["uuid"] + + expected_url = client.app.router["get_project_services"].url_for( + project_id=project_id + ) + assert URL(f"/v0/projects/{project_id}/nodes/-/services") == expected_url + + resp = await client.get(f"/v0/projects/{project_id}/nodes/-/services") + data, error = await assert_status(resp, status.HTTP_503_SERVICE_UNAVAILABLE) + + assert error + assert not data