diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py index aa6db574410..fe112184b0e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py @@ -12,7 +12,12 @@ from models_library.products import ProductName from models_library.rest_pagination import PageOffsetInt from models_library.rpc_pagination import PageLimitInt, PageRpc +from models_library.services_enums import ServiceType from models_library.services_history import ServiceRelease +from models_library.services_regex import ( + COMPUTATIONAL_SERVICE_KEY_RE, + DYNAMIC_SERVICE_KEY_RE, +) from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID from pydantic import NonNegativeInt, TypeAdapter @@ -65,6 +70,14 @@ async def get_service( got.version = service_version got.key = service_key + if DYNAMIC_SERVICE_KEY_RE.match(got.key): + got.service_type = ServiceType.DYNAMIC + elif COMPUTATIONAL_SERVICE_KEY_RE.match(got.key): + got.service_type = ServiceType.COMPUTATIONAL + else: + msg = "Service type not recognized. Please extend the mock yourself" + raise RuntimeError(msg) + return got async def update_service( diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py index 3e44b62c3b8..a58544a59f0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py @@ -7,7 +7,9 @@ class CapturedParameterSchema(BaseModel): title: str | None = None - type_: Literal["str", "int", "float", "bool"] | None = Field(None, alias="type") + type_: Literal["str", "int", "float", "bool", "null"] | None = Field( + None, alias="type" + ) pattern: str | None = None format_: Literal["uuid"] | None = Field(None, alias="format") exclusiveMinimum: bool | None = None diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 78309bd37eb..9c0d63ed8e9 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -223,7 +223,7 @@ "files" ], "summary": "List Files", - "description": "Lists all files stored in the system\n\nSEE `get_files_page` for a paginated version of this function", + "description": "\ud83d\udea8 **Deprecated**: This endpoint is deprecated and will be removed in a future release.\nPlease use `GET /v0/files/page` instead.\n\n\n\nLists all files stored in the system\n\nAdded in *version 0.5*: \n\nRemoved in *version 0.7*: This endpoint is deprecated and will be removed in a future version", "operationId": "list_files", "responses": { "200": { @@ -1414,7 +1414,7 @@ "solvers" ], "summary": "List Solvers", - "description": "Lists all available solvers (latest version)\n\nSEE get_solvers_page for paginated version of this function", + "description": "\ud83d\udea8 **Deprecated**: This endpoint is deprecated and will be removed in a future release.\nPlease use `GET /v0/solvers/page` instead.\n\n\n\nLists all available solvers (latest version)\n\nNew in *version 0.5.0*\n\nRemoved in *version 0.7*: This endpoint is deprecated and will be removed in a future version", "operationId": "list_solvers", "responses": { "200": { @@ -2356,7 +2356,7 @@ "solvers" ], "summary": "List Jobs", - "description": "List of jobs in a specific released solver (limited to 20 jobs)\n\n- DEPRECATION: This implementation and returned values are deprecated and the will be replaced by that of get_jobs_page\n- SEE `get_jobs_page` for paginated version of this function", + "description": "\ud83d\udea8 **Deprecated**: This endpoint is deprecated and will be removed in a future release.\nPlease use `GET /{solver_key}/releases/{version}/jobs/page` instead.\n\n\n\nList of jobs in a specific released solver\n\nNew in *version 0.5*\n\nRemoved in *version 0.7*: This endpoint is deprecated and will be removed in a future version", "operationId": "list_jobs", "security": [ { diff --git a/services/api-server/src/simcore_service_api_server/_service.py b/services/api-server/src/simcore_service_api_server/_service.py deleted file mode 100644 index bc826b1a310..00000000000 --- a/services/api-server/src/simcore_service_api_server/_service.py +++ /dev/null @@ -1,58 +0,0 @@ -import logging -from collections.abc import Callable - -from models_library.api_schemas_webserver.projects import ProjectCreateNew, ProjectGet -from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID -from pydantic import HttpUrl -from simcore_service_api_server.models.schemas.jobs import Job, JobInputs -from simcore_service_api_server.models.schemas.programs import Program -from simcore_service_api_server.models.schemas.solvers import Solver -from simcore_service_api_server.services_http.solver_job_models_converters import ( - create_job_from_project, - create_new_project_for_job, -) -from simcore_service_api_server.services_http.webserver import AuthSession - -_logger = logging.getLogger(__name__) - - -async def create_solver_or_program_job( - *, - webserver_api: AuthSession, - solver_or_program: Solver | Program, - inputs: JobInputs, - parent_project_uuid: ProjectID | None, - parent_node_id: NodeID | None, - url_for: Callable[..., HttpUrl], - hidden: bool -) -> tuple[Job, ProjectGet]: - # creates NEW job as prototype - pre_job = Job.create_job_from_solver_or_program( - solver_or_program_name=solver_or_program.name, inputs=inputs - ) - _logger.debug("Creating Job '%s'", pre_job.name) - - project_in: ProjectCreateNew = create_new_project_for_job( - solver_or_program, pre_job, inputs - ) - new_project: ProjectGet = await webserver_api.create_project( - project_in, - is_hidden=hidden, - parent_project_uuid=parent_project_uuid, - parent_node_id=parent_node_id, - ) - assert new_project # nosec - assert new_project.uuid == pre_job.id # nosec - - # for consistency, it rebuild job - job = create_job_from_project( - solver_or_program=solver_or_program, project=new_project, url_for=url_for - ) - assert job.id == pre_job.id # nosec - assert job.name == pre_job.name # nosec - assert job.name == Job.compose_resource_name( - parent_name=solver_or_program.resource_name, - job_id=job.id, - ) - return job, new_project diff --git a/services/api-server/src/simcore_service_api_server/_service_job.py b/services/api-server/src/simcore_service_api_server/_service_job.py new file mode 100644 index 00000000000..1b7cd79a317 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/_service_job.py @@ -0,0 +1,75 @@ +import logging +from collections.abc import Callable +from typing import Annotated + +from fastapi import Depends +from models_library.api_schemas_webserver.projects import ProjectCreateNew, ProjectGet +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID +from pydantic import HttpUrl +from servicelib.fastapi.app_state import SingletonInAppStateMixin +from servicelib.logging_utils import log_context + +from .api.dependencies.webserver_http import get_webserver_session +from .models.schemas.jobs import Job, JobInputs +from .models.schemas.programs import Program +from .models.schemas.solvers import Solver +from .services_http.solver_job_models_converters import ( + create_job_from_project, + create_new_project_for_job, +) +from .services_http.webserver import AuthSession + +_logger = logging.getLogger(__name__) + + +class JobService(SingletonInAppStateMixin): + app_state_name = "JobService" + _web_rest_api: AuthSession + + def __init__( + self, web_rest_api: Annotated[AuthSession, Depends(get_webserver_session)] + ): + self._web_rest_api = web_rest_api + + async def create_job( + self, + *, + solver_or_program: Solver | Program, + inputs: JobInputs, + parent_project_uuid: ProjectID | None, + parent_node_id: NodeID | None, + url_for: Callable[..., HttpUrl], + hidden: bool, + ) -> tuple[Job, ProjectGet]: + # creates NEW job as prototype + pre_job = Job.create_job_from_solver_or_program( + solver_or_program_name=solver_or_program.name, inputs=inputs + ) + with log_context( + logger=_logger, level=logging.DEBUG, msg=f"Creating job {pre_job.name}" + ): + project_in: ProjectCreateNew = create_new_project_for_job( + solver_or_program, pre_job, inputs + ) + new_project: ProjectGet = await self._web_rest_api.create_project( + project_in, + is_hidden=hidden, + parent_project_uuid=parent_project_uuid, + parent_node_id=parent_node_id, + ) + + assert new_project # nosec + assert new_project.uuid == pre_job.id # nosec + + # for consistency, it rebuild job + job = create_job_from_project( + solver_or_program=solver_or_program, project=new_project, url_for=url_for + ) + assert job.id == pre_job.id # nosec + assert job.name == pre_job.name # nosec + assert job.name == Job.compose_resource_name( + parent_name=solver_or_program.resource_name, + job_id=job.id, + ) + return job, new_project diff --git a/services/api-server/src/simcore_service_api_server/_service_programs.py b/services/api-server/src/simcore_service_api_server/_service_programs.py new file mode 100644 index 00000000000..9f55d1379a8 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/_service_programs.py @@ -0,0 +1,33 @@ +from typing import Annotated + +from fastapi import Depends +from models_library.basic_types import VersionStr +from models_library.services_enums import ServiceType + +from .models.schemas.programs import Program, ProgramKeyId +from .services_rpc.catalog import CatalogService + + +class ProgramService: + _catalog_service: CatalogService + + def __init__(self, _catalog_service: Annotated[CatalogService, Depends()]): + self._catalog_service = _catalog_service + + async def get_program( + self, + *, + user_id: int, + name: ProgramKeyId, + version: VersionStr, + product_name: str, + ) -> Program: + service = await self._catalog_service.get( + user_id=user_id, + name=name, + version=version, + product_name=product_name, + ) + assert service.service_type == ServiceType.DYNAMIC # nosec + + return Program.create_from_service(service) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py new file mode 100644 index 00000000000..5aa76086396 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -0,0 +1,72 @@ +from typing import Annotated + +from common_library.pagination_tools import iter_pagination_params +from fastapi import Depends +from models_library.basic_types import VersionStr +from models_library.products import ProductName +from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE +from models_library.services_enums import ServiceType +from models_library.services_history import ServiceRelease +from models_library.users import UserID +from packaging.version import Version + +from .models.schemas.solvers import Solver, SolverKeyId +from .services_rpc.catalog import CatalogService + +DEFAULT_PAGINATION_LIMIT = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE - 1 + + +class SolverService: + _catalog_service: CatalogService + + def __init__(self, catalog_service: Annotated[CatalogService, Depends()]): + self._catalog_service = catalog_service + + async def get_solver( + self, + *, + user_id: UserID, + name: SolverKeyId, + version: VersionStr, + product_name: ProductName, + ) -> Solver: + service = await self._catalog_service.get( + user_id=user_id, + name=name, + version=version, + product_name=product_name, + ) + assert ( # nosec + service.service_type == ServiceType.COMPUTATIONAL + ), "Expected by SolverName regex" + + return Solver.create_from_service(service) + + async def get_latest_release( + self, + *, + user_id: int, + solver_key: SolverKeyId, + product_name: str, + ) -> Solver: + service_releases: list[ServiceRelease] = [] + for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): + releases, page_meta = await self._catalog_service.list_release_history( + user_id=user_id, + service_key=solver_key, + product_name=product_name, + offset=page_params.offset, + limit=page_params.limit, + ) + page_params.total_number_of_items = page_meta.total + service_releases.extend(releases) + + release = sorted(service_releases, key=lambda s: Version(s.version))[-1] + service = await self._catalog_service.get( + user_id=user_id, + name=solver_key, + version=release.version, + product_name=product_name, + ) + + return Solver.create_from_service(service) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/_constants.py b/services/api-server/src/simcore_service_api_server/api/routes/_constants.py index 5962551d341..5e9f4eb73b6 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/_constants.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/_constants.py @@ -20,3 +20,41 @@ # removed on inputs/outputs in routes FMSG_CHANGELOG_REMOVED_IN_VERSION_FORMAT: Final[str] = "Removed in *version {}*: {}\n" + +FMSG_DEPRECATED_ROUTE_NOTICE: Final[str] = ( + "🚨 **Deprecated**: This endpoint is deprecated and will be removed in a future release.\n" + "Please use `{}` instead.\n\n" +) + + +def create_route_description( + *, + base: str = "", + deprecated: bool = False, + alternative: str | None = None, # alternative + changelog: list[str] | None = None +) -> str: + """ + Builds a consistent route description with optional deprecation and changelog information. + + Args: + base (str): Main route description. + deprecated (tuple): (retirement_date, alternative_route) if deprecated. + changelog (List[str]): List of formatted changelog strings. + + Returns: + str: Final description string. + """ + parts = [] + + if deprecated: + assert alternative, "If deprecated, alternative must be provided" # nosec + parts.append(FMSG_DEPRECATED_ROUTE_NOTICE.format(alternative)) + + if base: + parts.append(base) + + if changelog: + parts.append("\n".join(changelog)) + + return "\n\n".join(parts) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/files.py b/services/api-server/src/simcore_service_api_server/api/routes/files.py index 04e35bc00f9..6a4073eb2e3 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/files.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/files.py @@ -29,6 +29,11 @@ get_upload_links_from_s3, ) from simcore_sdk.node_ports_common.filemanager import upload_path as storage_upload_path +from simcore_service_api_server.api.routes._constants import ( + FMSG_CHANGELOG_ADDED_IN_VERSION, + FMSG_CHANGELOG_REMOVED_IN_VERSION_FORMAT, + create_route_description, +) from starlette.datastructures import URL from starlette.responses import RedirectResponse @@ -132,7 +137,23 @@ async def _create_domain_file( return file -@router.get("", response_model=list[OutputFile], responses=_FILE_STATUS_CODES) +@router.get( + "", + response_model=list[OutputFile], + responses=_FILE_STATUS_CODES, + description=create_route_description( + base="Lists all files stored in the system", + deprecated=True, + alternative="GET /v0/files/page", + changelog=[ + FMSG_CHANGELOG_ADDED_IN_VERSION.format("0.5", ""), + FMSG_CHANGELOG_REMOVED_IN_VERSION_FORMAT.format( + "0.7", + "This endpoint is deprecated and will be removed in a future version", + ), + ], + ), +) async def list_files( storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], user_id: Annotated[int, Depends(get_current_user_id)], diff --git a/services/api-server/src/simcore_service_api_server/api/routes/programs.py b/services/api-server/src/simcore_service_api_server/api/routes/programs.py index 15a6a229929..be2f27fe2ef 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/programs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/programs.py @@ -1,6 +1,5 @@ import logging from collections.abc import Callable -from operator import attrgetter from typing import Annotated from fastapi import APIRouter, Depends, Header, HTTPException, status @@ -17,51 +16,18 @@ complete_file_upload, get_upload_links_from_s3, ) -from simcore_service_api_server._service import create_solver_or_program_job -from simcore_service_api_server.api.dependencies.webserver_http import ( - get_webserver_session, -) -from simcore_service_api_server.services_http.webserver import AuthSession +from ..._service_job import JobService +from ..._service_programs import ProgramService from ...models.basic_types import VersionStr from ...models.schemas.jobs import Job, JobInputs from ...models.schemas.programs import Program, ProgramKeyId -from ...services_http.catalog import CatalogApi from ..dependencies.authentication import get_current_user_id, get_product_name -from ..dependencies.services import get_api_client _logger = logging.getLogger(__name__) router = APIRouter() -@router.get("", response_model=list[Program], include_in_schema=False) -async def list_programs( - user_id: Annotated[int, Depends(get_current_user_id)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], - url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], - product_name: Annotated[str, Depends(get_product_name)], -): - """Lists all available solvers (latest version) - - SEE get_solvers_page for paginated version of this function - """ - services = await catalog_client.list_services( - user_id=user_id, - product_name=product_name, - predicate=None, - type_filter="DYNAMIC", - ) - - programs = [service.to_program() for service in services] - - for program in programs: - program.url = url_for( - "get_program_release", program_key=program.id, version=program.version - ) - - return sorted(programs, key=attrgetter("id")) - - @router.get( "/{program_key:path}/releases/{version}", response_model=Program, @@ -70,13 +36,13 @@ async def get_program_release( program_key: ProgramKeyId, version: VersionStr, user_id: Annotated[int, Depends(get_current_user_id)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + program_service: Annotated[ProgramService, Depends()], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ) -> Program: """Gets a specific release of a solver""" try: - program = await catalog_client.get_program( + program = await program_service.get_program( user_id=user_id, name=program_key, version=version, @@ -109,8 +75,8 @@ async def create_program_job( program_key: ProgramKeyId, version: VersionStr, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], - webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + program_service: Annotated[ProgramService, Depends()], + job_service: Annotated[JobService, Depends()], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], x_simcore_parent_project_uuid: Annotated[ProjectID | None, Header()] = None, @@ -123,15 +89,14 @@ async def create_program_job( # ensures user has access to solver inputs = JobInputs(values={}) - program = await catalog_client.get_program( + program = await program_service.get_program( user_id=user_id, name=program_key, version=version, product_name=product_name, ) - job, project = await create_solver_or_program_job( - webserver_api=webserver_api, + job, project = await job_service.create_job( solver_or_program=program, inputs=inputs, parent_project_uuid=x_simcore_parent_project_uuid, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 89b220d2d5f..da286e491cd 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -7,6 +7,7 @@ from httpx import HTTPStatusError from pydantic import ValidationError +from ..._service_solvers import SolverService from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES from ...models.basic_types import VersionStr from ...models.pagination import OnePage, Page, PaginationParams @@ -19,7 +20,11 @@ from ..dependencies.services import get_api_client from ..dependencies.webserver_http import AuthSession, get_webserver_session from ._common import API_SERVER_DEV_FEATURES_ENABLED -from ._constants import FMSG_CHANGELOG_NEW_IN_VERSION +from ._constants import ( + FMSG_CHANGELOG_NEW_IN_VERSION, + FMSG_CHANGELOG_REMOVED_IN_VERSION_FORMAT, + create_route_description, +) _logger = logging.getLogger(__name__) @@ -43,17 +48,30 @@ # Would be nice to have /solvers/foo/releases/latest or solvers/foo/releases/3 , similar to docker tagging -@router.get("", response_model=list[Solver], responses=_SOLVER_STATUS_CODES) +@router.get( + "", + response_model=list[Solver], + responses=_SOLVER_STATUS_CODES, + description=create_route_description( + base="Lists all available solvers (latest version)", + deprecated=True, + alternative="GET /v0/solvers/page", + changelog=[ + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.5.0", ""), + FMSG_CHANGELOG_REMOVED_IN_VERSION_FORMAT.format( + "0.7", + "This endpoint is deprecated and will be removed in a future version", + ), + ], + ), +) async def list_solvers( user_id: Annotated[int, Depends(get_current_user_id)], catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ): - """Lists all available solvers (latest version) - - SEE get_solvers_page for paginated version of this function - """ + """Lists all available solvers (latest version)""" solvers: list[Solver] = await catalog_client.list_latest_releases( user_id=user_id, product_name=product_name ) @@ -135,7 +153,7 @@ async def get_solvers_releases_page( async def get_solver( solver_key: SolverKeyId, user_id: Annotated[int, Depends(get_current_user_id)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + solver_service: Annotated[SolverService, Depends(SolverService)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ) -> Solver: @@ -143,7 +161,7 @@ async def get_solver( # IMPORTANT: by adding /latest, we avoid changing the order of this entry in the router list # otherwise, {solver_key:path} will override and consume any of the paths that follow. try: - solver = await catalog_client.get_latest_release( + solver = await solver_service.get_latest_release( user_id=user_id, solver_key=solver_key, product_name=product_name, @@ -214,13 +232,13 @@ async def get_solver_release( solver_key: SolverKeyId, version: VersionStr, user_id: Annotated[int, Depends(get_current_user_id)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + solver_service: Annotated[SolverService, Depends(SolverService)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ) -> Solver: """Gets a specific release of a solver""" try: - solver: Solver = await catalog_client.get_solver( + solver: Solver = await solver_service.get_solver( user_id=user_id, name=solver_key, version=version, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 3fc1b1109ad..37c97d45e86 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -12,7 +12,8 @@ from models_library.projects_nodes_io import NodeID from pydantic.types import PositiveInt -from ..._service import create_solver_or_program_job +from ..._service_job import JobService +from ..._service_solvers import SolverService from ...exceptions.backend_errors import ProjectAlreadyStartedError from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES from ...models.basic_types import VersionStr @@ -26,7 +27,6 @@ JobStatus, ) from ...models.schemas.solvers import Solver, SolverKeyId -from ...services_http.catalog import CatalogApi from ...services_http.director_v2 import DirectorV2Api from ...services_http.jobs import replace_custom_metadata, start_project, stop_project from ...services_http.solver_job_models_converters import ( @@ -95,8 +95,8 @@ async def create_solver_job( version: VersionStr, inputs: JobInputs, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], - webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + solver_service: Annotated[SolverService, Depends()], + job_service: Annotated[JobService, Depends()], wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], @@ -110,14 +110,13 @@ async def create_solver_job( """ # ensures user has access to solver - solver = await catalog_client.get_solver( + solver = await solver_service.get_solver( user_id=user_id, name=solver_key, version=version, product_name=product_name, ) - job, project = await create_solver_or_program_job( - webserver_api=webserver_api, + job, project = await job_service.create_job( solver_or_program=solver, inputs=inputs, url_for=url_for, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index e156863c90b..b86595f3d92 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -19,6 +19,7 @@ from servicelib.fastapi.requests_decorators import cancel_on_disconnect from servicelib.logging_utils import log_context +from ..._service_solvers import SolverService from ...exceptions.custom_errors import InsufficientCreditsError, MissingWalletError from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES from ...models.basic_types import LogStreamingResponse, VersionStr @@ -39,7 +40,6 @@ WalletGetWithAvailableCreditsLegacy, ) from ...models.schemas.solvers import SolverKeyId -from ...services_http.catalog import CatalogApi from ...services_http.director_v2 import DirectorV2Api from ...services_http.jobs import ( get_custom_metadata, @@ -55,7 +55,11 @@ from ..dependencies.rabbitmq import get_log_check_timeout, get_log_distributor from ..dependencies.services import get_api_client from ..dependencies.webserver_http import AuthSession, get_webserver_session -from ._constants import FMSG_CHANGELOG_NEW_IN_VERSION +from ._constants import ( + FMSG_CHANGELOG_NEW_IN_VERSION, + FMSG_CHANGELOG_REMOVED_IN_VERSION_FORMAT, + create_route_description, +) from .solvers_jobs import ( JOBS_STATUS_CODES, METADATA_STATUS_CODES, @@ -118,23 +122,31 @@ "/{solver_key:path}/releases/{version}/jobs", response_model=list[Job], responses=JOBS_STATUS_CODES, + description=create_route_description( + base="List of jobs in a specific released solver", + deprecated=True, + alternative="GET /{solver_key}/releases/{version}/jobs/page", + changelog=[ + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.5"), + FMSG_CHANGELOG_REMOVED_IN_VERSION_FORMAT.format( + "0.7", + "This endpoint is deprecated and will be removed in a future version", + ), + ], + ), ) async def list_jobs( solver_key: SolverKeyId, version: VersionStr, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + solver_service: Annotated[SolverService, Depends(SolverService)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ): - """List of jobs in a specific released solver (limited to 20 jobs) - - - DEPRECATION: This implementation and returned values are deprecated and the will be replaced by that of get_jobs_page - - SEE `get_jobs_page` for paginated version of this function - """ + """List of jobs in a specific released solver (limited to 20 jobs)""" - solver = await catalog_client.get_solver( + solver = await solver_service.get_solver( user_id=user_id, name=solver_key, version=version, @@ -173,7 +185,7 @@ async def get_jobs_page( version: VersionStr, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], page_params: Annotated[PaginationParams, Depends()], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + solver_service: Annotated[SolverService, Depends(SolverService)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], @@ -181,7 +193,7 @@ async def get_jobs_page( # NOTE: Different entry to keep backwards compatibility with list_jobs. # Eventually use a header with agent version to switch to new interface - solver = await catalog_client.get_solver( + solver = await solver_service.get_solver( user_id=user_id, name=solver_key, version=version, @@ -217,7 +229,7 @@ async def get_job( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + solver_service: Annotated[SolverService, Depends(SolverService)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], ): """Gets job of a given solver""" @@ -225,7 +237,7 @@ async def get_job( "Getting Job '%s'", _compose_job_resource_name(solver_key, version, job_id) ) - solver = await catalog_client.get_solver( + solver = await solver_service.get_solver( user_id=user_id, name=solver_key, version=version, diff --git a/services/api-server/src/simcore_service_api_server/exceptions/backend_errors.py b/services/api-server/src/simcore_service_api_server/exceptions/backend_errors.py index 3653038ed28..bde18b2dbb5 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/backend_errors.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/backend_errors.py @@ -17,6 +17,11 @@ def named_fields(cls) -> set[str]: ) +class InvalidInputError(BaseBackEndError): + msg_template = "Invalid input" + status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + + class ListSolversOrStudiesError(BaseBackEndError): msg_template = "Cannot list solvers/studies" status_code = status.HTTP_404_NOT_FOUND @@ -37,11 +42,16 @@ class ProfileNotFoundError(BaseBackEndError): status_code = status.HTTP_404_NOT_FOUND -class SolverOrStudyNotFoundError(BaseBackEndError): - msg_template = "Could not get solver/study {name}:{version}" +class ProgramOrSolverOrStudyNotFoundError(BaseBackEndError): + msg_template = "Could not get program/solver/study {name}:{version}" status_code = status.HTTP_404_NOT_FOUND +class ServiceForbiddenAccessError(BaseBackEndError): + msg_template = "Forbidden access to program/solver/study {name}:{version}" + status_code = status.HTTP_403_FORBIDDEN + + class JobNotFoundError(BaseBackEndError): msg_template = "Could not get solver/study job {project_id}" status_code = status.HTTP_404_NOT_FOUND diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/programs.py b/services/api-server/src/simcore_service_api_server/models/schemas/programs.py index 25d1d1f5cf2..719af8a839e 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/programs.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/programs.py @@ -1,5 +1,6 @@ from typing import Annotated +from models_library.api_schemas_catalog.services import ServiceGetV2 from models_library.services import ServiceMetaDataPublished from models_library.services_regex import DYNAMIC_SERVICE_KEY_RE from pydantic import ConfigDict, StringConstraints @@ -56,6 +57,19 @@ def create_from_image(cls, image_meta: ServiceMetaDataPublished) -> "Program": **data, ) + @classmethod + def create_from_service(cls, service: ServiceGetV2) -> "Program": + data = service.model_dump( + include={"name", "key", "version", "description", "contact"}, + ) + return cls( + id=data.pop("key"), + version=data.pop("version"), + title=data.pop("name"), + url=None, + **data, + ) + @classmethod def compose_resource_name(cls, key: ProgramKeyId, version: VersionStr) -> str: return compose_resource_name("programs", key, "releases", version) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py index 6cae1156a7d..84e0b700723 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py @@ -1,5 +1,6 @@ from typing import Annotated, Any, Literal +from models_library.api_schemas_catalog.services import ServiceGetV2 from models_library.basic_regex import PUBLIC_VARIABLE_NAME_RE from models_library.services import ServiceMetaDataPublished from models_library.services_regex import COMPUTATIONAL_SERVICE_KEY_RE @@ -65,6 +66,20 @@ def create_from_image(cls, image_meta: ServiceMetaDataPublished) -> "Solver": **data, ) + @classmethod + def create_from_service(cls, service: ServiceGetV2) -> "Solver": + data = service.model_dump( + include={"name", "key", "version", "description", "contact"}, + ) + return cls( + id=data.pop("key"), + version=data.pop("version"), + title=data.pop("name"), + url=None, + maintainer=data.pop("contact"), + **data, + ) + @classmethod def compose_resource_name(cls, key: str, version: str) -> str: return compose_resource_name("solvers", key, "releases", version) diff --git a/services/api-server/src/simcore_service_api_server/services_http/catalog.py b/services/api-server/src/simcore_service_api_server/services_http/catalog.py index d9a8b26adb4..0b2b46dc061 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/catalog.py +++ b/services/api-server/src/simcore_service_api_server/services_http/catalog.py @@ -4,7 +4,6 @@ from collections.abc import Callable from dataclasses import dataclass from functools import partial -from operator import attrgetter from typing import Final, Literal from fastapi import FastAPI, status @@ -18,11 +17,11 @@ from ..exceptions.backend_errors import ( ListSolversOrStudiesError, - SolverOrStudyNotFoundError, + ProgramOrSolverOrStudyNotFoundError, ) from ..exceptions.service_errors_utils import service_exception_mapper from ..models.basic_types import VersionStr -from ..models.schemas.programs import Program, ProgramKeyId +from ..models.schemas.programs import Program from ..models.schemas.solvers import LATEST_VERSION, Solver, SolverKeyId, SolverPort from ..utils.client_base import BaseServiceClientApi, setup_client_instance @@ -145,70 +144,8 @@ async def list_services( services = [service for service in services if predicate(service)] return services - async def get_solver( - self, - *, - user_id: UserID, - name: SolverKeyId, - version: VersionStr, - product_name: ProductName, - ) -> Solver: - service = await self._get_service( - user_id=user_id, name=name, version=version, product_name=product_name - ) - assert ( # nosec - service.service_type == ServiceType.COMPUTATIONAL - ), "Expected by SolverName regex" - - solver: Solver = service.to_solver() - return solver - - async def get_program( - self, - *, - user_id: int, - name: ProgramKeyId, - version: VersionStr, - product_name: str, - ) -> Program: - service = await self._get_service( - user_id=user_id, name=name, version=version, product_name=product_name - ) - assert ( # nosec - service.service_type == ServiceType.DYNAMIC - ), "Expected by ProgramName regex" - - program = service.to_program() - return program - - @_exception_mapper( - http_status_map={status.HTTP_404_NOT_FOUND: SolverOrStudyNotFoundError} - ) - async def _get_service( - self, *, user_id: int, name: SolverKeyId, version: VersionStr, product_name: str - ) -> TruncatedCatalogServiceOut: - - assert version != LATEST_VERSION # nosec - - service_key = urllib.parse.quote_plus(name) - service_version = version - - response = await self.client.get( - f"/services/{service_key}/{service_version}", - params={"user_id": user_id}, - headers={"x-simcore-products-name": product_name}, - ) - response.raise_for_status() - - service: ( - TruncatedCatalogServiceOut - ) = await asyncio.get_event_loop().run_in_executor( - None, _parse_response, TruncatedCatalogServiceOutAdapter, response - ) - return service - @_exception_mapper( - http_status_map={status.HTTP_404_NOT_FOUND: SolverOrStudyNotFoundError} + http_status_map={status.HTTP_404_NOT_FOUND: ProgramOrSolverOrStudyNotFoundError} ) async def get_service_ports( self, @@ -272,22 +209,6 @@ def _this_solver(solver: Solver) -> bool: solvers = [service.to_solver() for service in services] return [solver for solver in solvers if _this_solver(solver)] - async def get_latest_release( - self, - *, - user_id: int, - solver_key: SolverKeyId, - product_name: str, - ) -> Solver: - releases = await self.list_service_releases( - user_id=user_id, - solver_key=solver_key, - product_name=product_name, - ) - - # raises IndexError if None - return sorted(releases, key=attrgetter("pep404_version"))[-1] - # MODULES APP SETUP ------------------------------------------------------------- diff --git a/services/api-server/src/simcore_service_api_server/services_http/rabbitmq.py b/services/api-server/src/simcore_service_api_server/services_http/rabbitmq.py index d2daaf82434..3466403d590 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/rabbitmq.py +++ b/services/api-server/src/simcore_service_api_server/services_http/rabbitmq.py @@ -38,8 +38,9 @@ async def _on_startup() -> None: await app.state.health_checker.setup( app.state.settings.API_SERVER_HEALTH_CHECK_TASK_PERIOD_SECONDS ) - wb_api_server.setup(app, get_rabbitmq_rpc_client(app)) + # setup rpc clients resource_usage_tracker.setup(app, get_rabbitmq_rpc_client(app)) + wb_api_server.setup(app, get_rabbitmq_rpc_client(app)) async def _on_shutdown() -> None: if app.state.health_checker: diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py index 0a9fde924de..8ecf61bf88f 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py @@ -1,5 +1,7 @@ -from dataclasses import dataclass +from functools import partial +from typing import Annotated +from fastapi import Depends from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2 from models_library.products import ProductName from models_library.rest_pagination import ( @@ -11,16 +13,33 @@ from models_library.services_history import ServiceRelease from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from servicelib.fastapi.app_state import SingletonInAppStateMixin +from pydantic import ValidationError from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc +from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( + CatalogForbiddenError, + CatalogItemNotFoundError, +) +from simcore_service_api_server.exceptions.backend_errors import ( + InvalidInputError, + ProgramOrSolverOrStudyNotFoundError, + ServiceForbiddenAccessError, +) + +from ..api.dependencies.rabbitmq import get_rabbitmq_rpc_client +from ..exceptions.service_errors_utils import service_exception_mapper + +_exception_mapper = partial(service_exception_mapper, service_name="CatalogService") -@dataclass -class CatalogService(SingletonInAppStateMixin): - app_state_name = "CatalogService" +class CatalogService: _client: RabbitMQRPCClient + def __init__( + self, client: Annotated[RabbitMQRPCClient, Depends(get_rabbitmq_rpc_client)] + ): + self._client = client + async def list_latest_releases( self, *, @@ -71,19 +90,26 @@ async def list_release_history( ) return page.data, meta + @_exception_mapper( + rpc_exception_map={ + CatalogItemNotFoundError: ProgramOrSolverOrStudyNotFoundError, + CatalogForbiddenError: ServiceForbiddenAccessError, + ValidationError: InvalidInputError, + } + ) async def get( self, *, product_name: ProductName, user_id: UserID, - service_key: ServiceKey, - service_version: ServiceVersion, + name: ServiceKey, + version: ServiceVersion, ) -> ServiceGetV2: return await catalog_rpc.get_service( self._client, product_name=product_name, user_id=user_id, - service_key=service_key, - service_version=service_version, + service_key=name, + service_version=version, ) diff --git a/services/api-server/tests/mocks/create_program_job_success.json b/services/api-server/tests/mocks/create_program_job_success.json new file mode 100644 index 00000000000..5e1d2b3a5c0 --- /dev/null +++ b/services/api-server/tests/mocks/create_program_job_success.json @@ -0,0 +1,196 @@ +[ + { + "name": "POST /projects", + "description": "", + "method": "POST", + "host": "webserver", + "path": { + "path": "/v0/projects", + "path_parameters": [] + }, + "query": "hidden=false", + "request_payload": { + "uuid": "a6677890-356b-4113-b26e-b77a748427f4", + "name": "programs/simcore%2Fservices%2Fdynamic%2Felectrode-selector/releases/2.1.3/jobs/a6677890-356b-4113-b26e-b77a748427f4", + "description": "Study associated to solver job:\n{\n \"id\": \"a6677890-356b-4113-b26e-b77a748427f4\",\n \"name\": \"programs/simcore%2Fservices%2Fdynamic%2Felectrode-selector/releases/2.1.3/jobs/a6677890-356b-4113-b26e-b77a748427f4\",\n \"inputs_checksum\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\n \"created_at\": \"2025-04-15T13:14:44.594564Z\"\n}", + "thumbnail": "https://via.placeholder.com/170x120.png", + "workbench": { + "f298a7d7-900d-533d-9385-89b39ea023fe": { + "key": "simcore/services/dynamic/electrode-selector", + "version": "2.1.3", + "label": "sleeper", + "progress": null, + "thumbnail": null, + "runHash": null, + "inputs": {}, + "inputsRequired": [], + "inputsUnits": {}, + "inputAccess": null, + "inputNodes": [], + "outputs": {}, + "outputNode": null, + "outputNodes": null, + "parent": null, + "position": null, + "state": { + "modified": true, + "dependencies": [], + "currentStatus": "NOT_STARTED", + "progress": 0.0 + }, + "bootOptions": null + } + }, + "accessRights": {}, + "tags": [], + "classifiers": [], + "ui": { + "icon": null, + "workbench": { + "f298a7d7-900d-533d-9385-89b39ea023fe": { + "position": { + "x": 633, + "y": 229 + }, + "marker": null + } + }, + "slideshow": {}, + "currentNodeId": "f298a7d7-900d-533d-9385-89b39ea023fe", + "annotations": {} + }, + "workspaceId": null, + "folderId": null + }, + "response_body": { + "data": { + "task_id": "POST%20%2Fv0%2Fprojects%3Fhidden%3Dfalse.24ca757c-9edb-4017-b0e2-e6f7dcc447c8", + "task_name": "POST /v0/projects?hidden=false", + "status_href": "http://webserver:8080/v0/tasks-legacy/POST%2520%252Fv0%252Fprojects%253Fhidden%253Dfalse.24ca757c-9edb-4017-b0e2-e6f7dcc447c8", + "result_href": "http://webserver:8080/v0/tasks-legacy/POST%2520%252Fv0%252Fprojects%253Fhidden%253Dfalse.24ca757c-9edb-4017-b0e2-e6f7dcc447c8/result", + "abort_href": "http://webserver:8080/v0/tasks-legacy/POST%2520%252Fv0%252Fprojects%253Fhidden%253Dfalse.24ca757c-9edb-4017-b0e2-e6f7dcc447c8" + } + }, + "status_code": 202 + }, + { + "name": "GET http://webserver:8080/v0/tasks-legacy/POST%2520%252Fv0%252Fprojects%253Fhidden%253Dfalse.24ca757c-9edb-4017-b0e2-e6f7dcc447c8", + "description": "", + "method": "GET", + "host": "webserver", + "path": { + "path": "/v0/tasks-legacy/{task_id}", + "path_parameters": [ + { + "in": "path", + "name": "task_id", + "required": true, + "schema": { + "title": "Task Id", + "type": "str" + }, + "response_value": "tasks-legacy" + } + ] + }, + "response_body": { + "data": { + "task_progress": { + "task_id": "POST%20%2Fv0%2Fprojects%3Fhidden%3Dfalse.24ca757c-9edb-4017-b0e2-e6f7dcc447c8", + "message": "finished", + "percent": 1.0 + }, + "done": true, + "started": "2025-04-15T13:14:44.617066" + } + } + }, + { + "name": "GET http://webserver:8080/v0/tasks-legacy/POST%2520%252Fv0%252Fprojects%253Fhidden%253Dfalse.24ca757c-9edb-4017-b0e2-e6f7dcc447c8/result", + "description": "", + "method": "GET", + "host": "webserver", + "path": { + "path": "/v0/tasks-legacy/{task_id}/result", + "path_parameters": [ + { + "in": "path", + "name": "task_id", + "required": true, + "schema": { + "title": "Task Id", + "type": "str" + }, + "response_value": "tasks-legacy" + } + ] + }, + "response_body": { + "data": { + "uuid": "a6677890-356b-4113-b26e-b77a748427f4", + "name": "programs/simcore%2Fservices%2Fdynamic%2Felectrode-selector/releases/2.1.3/jobs/a6677890-356b-4113-b26e-b77a748427f4", + "description": "Study associated to solver job:\n{\n \"id\": \"a6677890-356b-4113-b26e-b77a748427f4\",\n \"name\": \"programs/simcore%2Fservices%2Fdynamic%2Felectrode-selector/releases/2.1.3/jobs/a6677890-356b-4113-b26e-b77a748427f4\",\n \"inputs_checksum\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\n \"created_at\": \"2025-04-15T13:14:44.594564Z\"\n}", + "thumbnail": "https://via.placeholder.com/170x120.png", + "workbench": { + "f298a7d7-900d-533d-9385-89b39ea023fe": { + "key": "simcore/services/dynamic/electrode-selector", + "version": "2.1.3", + "label": "sleeper", + "inputs": {}, + "inputsRequired": [], + "inputsUnits": {}, + "inputNodes": [], + "outputs": {}, + "state": { + "modified": true, + "dependencies": [], + "currentStatus": "NOT_STARTED", + "progress": 0.0 + } + } + }, + "prjOwner": "bisgaard@itis.swiss", + "accessRights": { + "4": { + "read": true, + "write": true, + "delete": true + } + }, + "creationDate": "2025-04-15T13:14:44.636Z", + "lastChangeDate": "2025-04-15T13:14:46.704Z", + "state": { + "locked": { + "value": false, + "status": "CLOSED" + }, + "state": { + "value": "UNKNOWN" + } + }, + "trashedAt": null, + "trashedBy": null, + "tags": [], + "classifiers": [], + "quality": {}, + "ui": { + "workbench": { + "f298a7d7-900d-533d-9385-89b39ea023fe": { + "position": { + "x": 633, + "y": 229 + } + } + }, + "slideshow": {}, + "currentNodeId": "f298a7d7-900d-533d-9385-89b39ea023fe", + "annotations": {} + }, + "dev": {}, + "workspaceId": null, + "folderId": null + } + }, + "status_code": 201 + } +] diff --git a/services/api-server/tests/unit/_with_db/test_product.py b/services/api-server/tests/unit/_with_db/test_product.py index acf20949618..5c83ec557fa 100644 --- a/services/api-server/tests/unit/_with_db/test_product.py +++ b/services/api-server/tests/unit/_with_db/test_product.py @@ -15,9 +15,9 @@ from fastapi.encoders import jsonable_encoder from models_library.api_schemas_api_server.api_keys import ApiKeyInDB from models_library.generics import Envelope -from models_library.users import UserID from models_library.wallets import WalletStatus from pydantic import PositiveInt +from pytest_mock import MockType from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.models.schemas.model_adapter import ( WalletGetWithAvailableCreditsLegacy, @@ -80,7 +80,7 @@ def _check_key_product_compatibility(request: httpx.Request, **kwargs): async def test_product_catalog( client: httpx.AsyncClient, - mocked_catalog_rest_api_base: respx.MockRouter, + mocked_rpc_catalog_service_api: dict[str, MockType], create_fake_api_keys: Callable[[PositiveInt], AsyncGenerator[ApiKeyInDB, None]], ) -> None: assert client @@ -88,25 +88,10 @@ async def test_product_catalog( keys: list[ApiKeyInDB] = [key async for key in create_fake_api_keys(2)] assert len({key.product_name for key in keys}) == 2 - def _get_service_side_effect(request: httpx.Request, **kwargs): - assert ( - received_product := request.headers.get("x-simcore-products-name") - ) is not None - assert (user_id := request.url.params.get("user_id")) is not None - assert ( - key := {UserID(key.id_): key for key in keys}.get(UserID(user_id)) - ) is not None - assert key.product_name == received_product - return httpx.Response(status_code=status.HTTP_200_OK) - - respx_mock = mocked_catalog_rest_api_base.get( - r"/v0/services/simcore%2Fservices%2Fcomp%2Fisolve/2.0.24" - ).mock(side_effect=_get_service_side_effect) - for key in keys: await client.get( f"{API_VTAG}/solvers/simcore/services/comp/isolve/releases/2.0.24", auth=httpx.BasicAuth(key.api_key, key.api_secret), ) - assert respx_mock.call_count == len(keys) + assert mocked_rpc_catalog_service_api["get_service"].called diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py index c4e017fa026..fa8160ce87a 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py @@ -204,7 +204,7 @@ async def test_run_solver_job( client: httpx.AsyncClient, directorv2_service_openapi_specs: dict[str, Any], catalog_service_openapi_specs: dict[str, Any], - mocked_catalog_rest_api: MockRouter, + mocked_rpc_catalog_service_api: dict[str, MockType], mocked_directorv2_service_api: MockRouter, mocked_webserver_rest_api: MockRouter, mocked_webserver_rpc_api: dict[str, MockType], @@ -319,22 +319,6 @@ async def test_run_solver_job( if "boot-options" in e ) - mocked_catalog_rest_api.get( - # path__regex=r"/services/(?P[\w-]+)/(?P[0-9\.]+)", - path=f"/v0/services/{solver_key}/{solver_version}", - name="get_service_v0_services__service_key___service_version__get", - ).respond( - status.HTTP_200_OK, - json=example - | { - "name": solver_key.split("/")[-1].capitalize(), - "description": solver_key.replace("/", " "), - "key": solver_key, - "version": solver_version, - "type": "computational", - }, - ) - # --------------------------------------------------------------------------------------------------------- resp = await client.get(f"/{API_VTAG}/meta") diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py index c5fad3f6d2a..23966e15bd1 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py @@ -27,7 +27,6 @@ class MockedBackendApiDict(TypedDict): - catalog: MockRouter | None webserver: MockRouter | None @@ -56,7 +55,7 @@ def _response(request: httpx.Request, project_id: str): name="delete_project", ).mock(side_effect=_response) - return MockedBackendApiDict(webserver=mocked_webserver_rest_api, catalog=None) + return MockedBackendApiDict(webserver=mocked_webserver_rest_api) @pytest.mark.acceptance_test( @@ -88,7 +87,6 @@ async def test_delete_non_existing_solver_job( def mocked_backend_services_apis_for_create_and_delete_solver_job( mocked_webserver_rest_api: MockRouter, mocked_webserver_rpc_api: dict[str, MockType], - mocked_catalog_rest_api: MockRouter, project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "on_create_job.json" @@ -98,12 +96,12 @@ def mocked_backend_services_apis_for_create_and_delete_solver_job( Path(project_tests_dir / "mocks" / mock_name).read_text() ) - capture = captures[0] - assert capture.host == "catalog" - assert capture.method == "GET" - mocked_catalog_rest_api.request( - method=capture.method, path=capture.path, name="get_service" # GET service - ).respond(status_code=capture.status_code, json=capture.response_body) + # capture = captures[0] + # assert capture.host == "catalog" + # assert capture.method == "GET" + # mocked_catalog_rest_api.request( + # method=capture.method, path=capture.path, name="get_service" # GET service + # ).respond(status_code=capture.status_code, json=capture.response_body) capture = captures[-1] assert capture.host == "webserver" @@ -115,7 +113,6 @@ def mocked_backend_services_apis_for_create_and_delete_solver_job( ).respond(status_code=capture.status_code, json=capture.response_body) return MockedBackendApiDict( - catalog=mocked_catalog_rest_api, webserver=mocked_webserver_rest_api, ) @@ -128,6 +125,7 @@ async def test_create_and_delete_solver_job( client: httpx.AsyncClient, solver_key: str, solver_version: str, + mocked_rpc_catalog_service_api: dict[str, MockType], mocked_backend_services_apis_for_create_and_delete_solver_job: MockedBackendApiDict, ): # create Job @@ -157,11 +155,9 @@ async def test_create_and_delete_solver_job( assert mock_webserver_router assert mock_webserver_router["delete_project"].called - mock_catalog_router = mocked_backend_services_apis_for_create_and_delete_solver_job[ - "catalog" - ] - assert mock_catalog_router - assert mock_catalog_router["get_service"].called + get_service = mocked_rpc_catalog_service_api["get_service"] + assert get_service + assert get_service.called # NOTE: ideas for further tests # Run job and try to delete while running @@ -179,6 +175,7 @@ async def test_create_job( solver_key: str, solver_version: str, mocked_backend_services_apis_for_create_and_delete_solver_job: MockedBackendApiDict, + mocked_rpc_catalog_service_api: dict[str, MockType], hidden: bool, parent_project_id: UUID | None, parent_node_id: UUID | None, diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py index 1693e579fa8..41949228763 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py @@ -25,7 +25,6 @@ class MockedBackendApiDict(TypedDict): - catalog: MockRouter | None webserver: MockRouter | None @@ -40,7 +39,7 @@ def _as_path_regex(initial_path: str): def mocked_backend( mocked_webserver_rest_api: MockRouter, mocked_webserver_rpc_api: dict[str, MockType], - mocked_catalog_rest_api: MockRouter, + mocked_rpc_catalog_service_api: dict[str, MockType], project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "for_test_get_and_update_job_metadata.json" @@ -54,14 +53,6 @@ def mocked_backend( capture = captures["get_service"] assert capture.host == "catalog" - mocked_catalog_rest_api.request( - method=capture.method, - path=capture.path, - name=capture.name, - ).respond( - status_code=capture.status_code, - json=capture.response_body, - ) for name in ("get_project_metadata", "update_project_metadata", "delete_project"): capture = captures[name] @@ -87,9 +78,7 @@ def mocked_backend( json=capture.response_body, ) - return MockedBackendApiDict( - webserver=mocked_webserver_rest_api, catalog=mocked_catalog_rest_api - ) + return MockedBackendApiDict(webserver=mocked_webserver_rest_api) @pytest.mark.acceptance_test( diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py index 09fc53194f6..7f3be4e216f 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py @@ -8,6 +8,7 @@ import httpx import pytest from pydantic import TypeAdapter +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from simcore_service_api_server._meta import API_VTAG @@ -17,14 +18,14 @@ class MockBackendRouters(NamedTuple): - catalog: MockRouter webserver: MockRouter + catalog: dict[str, MockType] @pytest.fixture def mocked_backend( mocked_webserver_rest_api_base: MockRouter, - mocked_catalog_rest_api_base: MockRouter, + mocked_rpc_catalog_service_api: dict[str, MockType], project_tests_dir: Path, ) -> MockBackendRouters: mock_name = "on_list_jobs.json" @@ -32,18 +33,6 @@ def mocked_backend( Path(project_tests_dir / "mocks" / mock_name).read_text() ) - capture = captures[0] - assert capture.host == "catalog" - assert capture.name == "get_service" - mocked_catalog_rest_api_base.request( - method=capture.method, - path=capture.path, - name=capture.name, - ).respond( - status_code=capture.status_code, - json=capture.response_body, - ) - capture = captures[1] assert capture.host == "webserver" assert capture.name == "list_projects" @@ -57,8 +46,8 @@ def mocked_backend( ) return MockBackendRouters( - catalog=mocked_catalog_rest_api_base, webserver=mocked_webserver_rest_api_base, + catalog=mocked_rpc_catalog_service_api, ) diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 8fb9fb2a445..bed207c212f 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -5,7 +5,7 @@ import json import subprocess -from collections.abc import AsyncIterator, Callable, Iterator +from collections.abc import AsyncIterator, Callable, Iterable, Iterator from copy import deepcopy from pathlib import Path from typing import Any @@ -38,6 +38,7 @@ from packaging.version import Version from pydantic import EmailStr, HttpUrl, TypeAdapter from pytest_mock import MockerFixture, MockType +from pytest_simcore.helpers.catalog_rpc_server import CatalogRpcSideEffects from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from pytest_simcore.helpers.webserver_rpc_server import WebserverRpcSideEffects @@ -45,12 +46,60 @@ from requests.auth import HTTPBasicAuth from respx import MockRouter from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc +from simcore_service_api_server.api.dependencies.rabbitmq import get_rabbitmq_rpc_client from simcore_service_api_server.core.application import init_app from simcore_service_api_server.core.settings import ApplicationSettings from simcore_service_api_server.db.repositories.api_keys import UserAndProductTuple from simcore_service_api_server.services_http.solver_job_outputs import ResultsTypes +@pytest.fixture +def mocked_rpc_catalog_service_api( + app: FastAPI, mocker: MockerFixture +) -> Iterable[dict[str, MockType]]: + """ + Mocks the RPC catalog service API for testing purposes. + """ + + class MockRabbitMQRPCClient: + pass + + def get_mock_rabbitmq_rpc_client(): + return MockRabbitMQRPCClient() + + app.dependency_overrides[get_rabbitmq_rpc_client] = get_mock_rabbitmq_rpc_client + side_effects = CatalogRpcSideEffects() + + yield { + "list_services_paginated": mocker.patch.object( + catalog_rpc, + "list_services_paginated", + autospec=True, + side_effect=side_effects.list_services_paginated, + ), + "get_service": mocker.patch.object( + catalog_rpc, + "get_service", + autospec=True, + side_effect=side_effects.get_service, + ), + "update_service": mocker.patch.object( + catalog_rpc, + "update_service", + autospec=True, + side_effect=side_effects.update_service, + ), + "list_my_service_history_paginated": mocker.patch.object( + catalog_rpc, + "list_my_service_history_paginated", + autospec=True, + side_effect=side_effects.list_my_service_history_paginated, + ), + } + app.dependency_overrides.pop(get_rabbitmq_rpc_client) + + @pytest.fixture def app_environment( monkeypatch: pytest.MonkeyPatch, @@ -499,7 +548,7 @@ def _patch(href): return data.status_href, data.result_href return mocker.patch( - "simcore_service_api_server.services.webserver._get_lrt_urls", + "simcore_service_api_server.services_http.webserver._get_lrt_urls", side_effect=_get_lrt_urls, ) diff --git a/services/api-server/tests/unit/test_api_programs.py b/services/api-server/tests/unit/test_api_programs.py new file mode 100644 index 00000000000..a0ad30eb1d1 --- /dev/null +++ b/services/api-server/tests/unit/test_api_programs.py @@ -0,0 +1,115 @@ +# pylint: disable=unused-argument +# pylint: disable=unused-variable +import json +from functools import partial +from pathlib import Path +from typing import Any + +import httpx +import pytest +from fastapi import status +from httpx import AsyncClient +from models_library.api_schemas_storage.storage_schemas import FileUploadSchema +from models_library.users import UserID +from pytest_mock import MockerFixture, MockType +from pytest_simcore.helpers.httpx_calls_capture_models import ( + CreateRespxMockCallback, + HttpApiCallCaptureModel, +) +from simcore_service_api_server._meta import API_VTAG +from simcore_service_api_server.models.schemas.jobs import Job +from simcore_service_api_server.models.schemas.programs import Program + + +async def test_get_program_release( + auth: httpx.BasicAuth, + client: AsyncClient, + mocked_rpc_catalog_service_api: dict[str, MockType], + mocker: MockerFixture, + user_id: UserID, +): + # Arrange + program_key = "simcore/services/dynamic/my_program" + version = "1.0.0" + + response = await client.get( + f"/{API_VTAG}/programs/{program_key}/releases/{version}", auth=auth + ) + + # Assert + assert response.status_code == status.HTTP_200_OK + program = Program.model_validate(response.json()) + assert program.id == program_key + assert program.version == version + + +@pytest.mark.parametrize("capture_name", ["create_program_job_success.json"]) +async def test_create_program_job( + auth: httpx.BasicAuth, + client: AsyncClient, + mocked_webserver_rest_api_base, + mocked_rpc_catalog_service_api: dict[str, MockType], + create_respx_mock_from_capture: CreateRespxMockCallback, + mocker: MockerFixture, + user_id: UserID, + capture_name: str, + project_tests_dir: Path, +): + + mocker.patch( + "simcore_service_api_server.api.routes.programs.get_upload_links_from_s3", + return_value=( + None, + FileUploadSchema.model_validate( + next(iter(FileUploadSchema.model_json_schema()["examples"])) + ), + ), + ) + mocker.patch("simcore_service_api_server.api.routes.programs.complete_file_upload") + + def _side_effect( + server_state: dict, + request: httpx.Request, + kwargs: dict[str, Any], + capture: HttpApiCallCaptureModel, + ) -> dict[str, Any]: + + response_body = capture.response_body + + # first call defines the project uuid + if server_state.get("project_uuid") is None: + _project_uuid = json.loads(request.content.decode("utf-8")).get("uuid") + assert _project_uuid + server_state["project_uuid"] = _project_uuid + + if request.url.path.endswith("/result"): + capture_uuid = response_body["data"]["uuid"] + response_body["data"]["uuid"] = server_state["project_uuid"] + response_body["data"]["name"] = response_body["data"]["name"].replace( + capture_uuid, server_state["project_uuid"] + ) + assert isinstance(response_body, dict) + return response_body + + # simulate server state + _server_state = dict() + + create_respx_mock_from_capture( + respx_mocks=[mocked_webserver_rest_api_base], + capture_path=project_tests_dir / "mocks" / capture_name, + side_effects_callbacks=3 * [partial(_side_effect, _server_state)], + ) + + # Arrange + program_key = "simcore/services/dynamic/electrode-selector" + version = "2.1.3" + + response = await client.post( + f"/{API_VTAG}/programs/{program_key}/releases/{version}/jobs", + # headers=headers, + auth=auth, + ) + + # Assert + assert response.status_code == status.HTTP_201_CREATED + job = Job.model_validate(response.json()) diff --git a/services/api-server/tests/unit/test_api_solvers.py b/services/api-server/tests/unit/test_api_solvers.py index d35b648629e..e7ac79d0cc1 100644 --- a/services/api-server/tests/unit/test_api_solvers.py +++ b/services/api-server/tests/unit/test_api_solvers.py @@ -50,3 +50,26 @@ async def test_get_solver_pricing_plan( assert expected_status_code == response.status_code if response.status_code == status.HTTP_200_OK: _ = ServicePricingPlanGet.model_validate(response.json()) + + +@pytest.mark.parametrize( + "solver_key,expected_status_code", + [ + ("simcore/services/comp/valid_solver", status.HTTP_200_OK), + ], +) +async def test_get_latest_solver_release( + client: AsyncClient, + mocked_rpc_catalog_service_api, + auth: httpx.BasicAuth, + solver_key: str, + expected_status_code: int, +): + response = await client.get( + f"{API_VTAG}/solvers/{solver_key}/latest", + auth=auth, + ) + assert response.status_code == expected_status_code + if response.status_code == status.HTTP_200_OK: + assert "id" in response.json() + assert response.json()["id"] == solver_key diff --git a/services/api-server/tests/unit/test_services_catalog.py b/services/api-server/tests/unit/test_services_catalog.py index 99c5dc58b4c..767ec869889 100644 --- a/services/api-server/tests/unit/test_services_catalog.py +++ b/services/api-server/tests/unit/test_services_catalog.py @@ -12,9 +12,8 @@ from models_library.users import UserID from pydantic import HttpUrl from pytest_mock import MockerFixture, MockType -from pytest_simcore.helpers.catalog_rpc_server import CatalogRpcSideEffects from simcore_service_api_server.models.schemas.solvers import Solver -from simcore_service_api_server.services_rpc.catalog import CatalogService, catalog_rpc +from simcore_service_api_server.services_rpc.catalog import CatalogService @pytest.fixture @@ -22,39 +21,6 @@ def product_name() -> ProductName: return "osparc" -@pytest.fixture -def mocked_rpc_catalog_service_api(mocker: MockerFixture) -> dict[str, MockType]: - - side_effects = CatalogRpcSideEffects() - - return { - "list_services_paginated": mocker.patch.object( - catalog_rpc, - "list_services_paginated", - autospec=True, - side_effect=side_effects.list_services_paginated, - ), - "get_service": mocker.patch.object( - catalog_rpc, - "get_service", - autospec=True, - side_effect=side_effects.get_service, - ), - "update_service": mocker.patch.object( - catalog_rpc, - "update_service", - autospec=True, - side_effect=side_effects.update_service, - ), - "list_my_service_history_paginated": mocker.patch.object( - catalog_rpc, - "list_my_service_history_paginated", - autospec=True, - side_effect=side_effects.list_my_service_history_paginated, - ), - } - - def to_solver( service: LatestServiceGet | ServiceGetV2, href_self: HttpUrl | None = None ) -> Solver: @@ -70,13 +36,13 @@ def to_solver( async def test_catalog_service_read_solvers( + app: FastAPI, product_name: ProductName, user_id: UserID, mocker: MockerFixture, mocked_rpc_catalog_service_api: dict[str, MockType], ): - catalog_service = CatalogService(_client=mocker.MagicMock()) - catalog_service.set_to_app_state(app=FastAPI()) + catalog_service = CatalogService(client=mocker.MagicMock()) # Step 1: List latest releases in a page latest_releases, meta = await catalog_service.list_latest_releases( @@ -103,8 +69,8 @@ async def test_catalog_service_read_solvers( service: ServiceGetV2 = await catalog_service.get( product_name=product_name, user_id=user_id, - service_key=selected_solver.id, - service_version=oldest_release.version, + name=selected_solver.id, + version=oldest_release.version, ) solver = to_solver(service) assert solver.id == selected_solver.id