From f48164274d480b260760c818003ee89c552e4ddc Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 15 Jul 2024 13:51:20 +0200 Subject: [PATCH 01/17] draft batch --- .../services/director.py | 7 ------ .../services/manifest.py | 25 +++++++++++++++++++ .../tests/unit/test_services_manifest.py | 8 ++++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/services/director.py b/services/catalog/src/simcore_service_catalog/services/director.py index 1f40b8ea3ec..bde5ce8a5f3 100644 --- a/services/catalog/src/simcore_service_catalog/services/director.py +++ b/services/catalog/src/simcore_service_catalog/services/director.py @@ -11,7 +11,6 @@ from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from models_library.utils.json_serialization import json_dumps -from pydantic import parse_obj_as from servicelib.logging_utils import log_context from starlette import status from tenacity._asyncio import AsyncRetrying @@ -140,12 +139,6 @@ async def is_responsive(self) -> bool: except (httpx.HTTPStatusError, httpx.RequestError, httpx.TimeoutException): return False - async def list_all_services(self) -> list[ServiceMetaDataPublished]: - # WARNING: this function probably raise ValidationError since director does NOT offer guarantees. - # SEE list_registered_services - data = await self.get("/services") - return parse_obj_as(list[ServiceMetaDataPublished], data) - async def get_service( self, service_key: ServiceKey, service_version: ServiceVersion ) -> ServiceMetaDataPublished: diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/services/manifest.py index 33ab9bfe657..75c5a0a74ec 100644 --- a/services/catalog/src/simcore_service_catalog/services/manifest.py +++ b/services/catalog/src/simcore_service_catalog/services/manifest.py @@ -27,11 +27,14 @@ import logging from typing import Any, TypeAlias, cast +from aiocache import cached from models_library.function_services_catalog.api import iter_service_docker_data from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from pydantic import ValidationError +from servicelib.utils import limited_gather +from .._constants import DIRECTOR_CACHING_TTL from .director import DirectorApi from .function_services import get_function_service, is_function_service @@ -80,6 +83,11 @@ async def get_services_map( return services +@cached( + ttl=DIRECTOR_CACHING_TTL, + namespace=__name__, + key_builder=lambda f, *ag, **kw: f"{f.__name__}/{kw['service_key']}/{kw['service_version']}", +) async def get_service( service_key: ServiceKey, service_version: ServiceVersion, @@ -95,3 +103,20 @@ async def get_service( service_key=service_key, service_version=service_version ) return service + + +async def get_batch_services( + selection: list[tuple[ServiceKey, ServiceVersion]], director_client: DirectorApi +) -> list[ServiceMetaDataPublished | BaseException]: + + return await limited_gather( + *( + get_service( + service_key=k, service_version=v, director_client=director_client + ) + for k, v in selection + ), + reraise=False, + log=_logger, + tasks_group_prefix="manifest.get_batch_services", + ) diff --git a/services/catalog/tests/unit/test_services_manifest.py b/services/catalog/tests/unit/test_services_manifest.py index 4a6fcbdd025..1f88e4af318 100644 --- a/services/catalog/tests/unit/test_services_manifest.py +++ b/services/catalog/tests/unit/test_services_manifest.py @@ -7,6 +7,7 @@ import pytest +import toolz from fastapi import FastAPI from models_library.function_services_catalog.api import is_function_service from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict @@ -65,3 +66,10 @@ async def test_services_manifest_api( assert service == expected_service if not is_function_service(service.key): assert mocked_director_service_api["get_service"].called + + # BATCH + for expected_services in toolz.partition(2, all_services_map.values()): + got_services = await manifest.get_batch_services( + [(s.key, s.value) for s in expected_services], director_api + ) + assert got_services == expected_services From 9b1de2db339036443740852c4a21bc6106af9190 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:44:40 +0200 Subject: [PATCH 02/17] rename services --- .../src/simcore_service_catalog/api/rpc/_services.py | 8 ++++---- .../services/{catalog.py => services.py} | 0 ...test_services_catalog.py => test_services_services.py} | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename services/catalog/src/simcore_service_catalog/services/{catalog.py => services.py} (100%) rename services/catalog/tests/unit/with_dbs/{test_services_catalog.py => test_services_services.py} (92%) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 61623d05ee6..4458b4d3095 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -19,7 +19,7 @@ ) from ...db.repositories.services import ServicesRepository -from ...services import catalog +from ...services import services _logger = logging.getLogger(__name__) @@ -38,7 +38,7 @@ async def list_services_paginated( ) -> PageRpcServicesGetV2: assert app.state.engine # nosec - total_count, items = await catalog.list_services_paginated( + total_count, items = await services.list_services_paginated( repo=ServicesRepository(app.state.engine), product_name=product_name, user_id=user_id, @@ -69,7 +69,7 @@ async def get_service( ) -> ServiceGetV2: assert app.state.engine # nosec - service = await catalog.get_service( + service = await services.get_service( repo=ServicesRepository(app.state.engine), product_name=product_name, user_id=user_id, @@ -98,7 +98,7 @@ async def update_service( assert app.state.engine # nosec - service = await catalog.update_service( + service = await services.update_service( repo=ServicesRepository(app.state.engine), product_name=product_name, user_id=user_id, diff --git a/services/catalog/src/simcore_service_catalog/services/catalog.py b/services/catalog/src/simcore_service_catalog/services/services.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/services/catalog.py rename to services/catalog/src/simcore_service_catalog/services/services.py diff --git a/services/catalog/tests/unit/with_dbs/test_services_catalog.py b/services/catalog/tests/unit/with_dbs/test_services_services.py similarity index 92% rename from services/catalog/tests/unit/with_dbs/test_services_catalog.py rename to services/catalog/tests/unit/with_dbs/test_services_services.py index f828b300d56..7cc10705ddb 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_catalog.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services.py @@ -8,7 +8,7 @@ from models_library.products import ProductName from models_library.users import UserID from simcore_service_catalog.db.repositories.services import ServicesRepository -from simcore_service_catalog.services import catalog +from simcore_service_catalog.services import services from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = [ @@ -52,7 +52,7 @@ async def test_list_services_paginated( assert limit < num_services offset = 1 - total_count, page_items = await catalog.list_services_paginated( + total_count, page_items = await services.list_services_paginated( services_repo, product_name=target_product, user_id=user_id, @@ -68,7 +68,7 @@ async def test_list_services_paginated( assert item.owner is not None assert item.history[0].version == item.version - got = await catalog.get_service( + got = await services.get_service( services_repo, product_name=target_product, user_id=user_id, From 4921232677ba73cde705c49b7c0665f15845bed3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:27:58 +0200 Subject: [PATCH 03/17] drafting tests --- .../api/rpc/_services.py | 4 ++ .../services/services.py | 53 +++++++++++---- .../unit/with_dbs/test_services_services.py | 67 ++++++++++++++----- 3 files changed, 94 insertions(+), 30 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 4458b4d3095..8d029be9138 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -20,6 +20,7 @@ from ...db.repositories.services import ServicesRepository from ...services import services +from ..dependencies.director import get_director_api _logger = logging.getLogger(__name__) @@ -40,6 +41,7 @@ async def list_services_paginated( total_count, items = await services.list_services_paginated( repo=ServicesRepository(app.state.engine), + director_api=get_director_api(app), product_name=product_name, user_id=user_id, limit=limit, @@ -71,6 +73,7 @@ async def get_service( service = await services.get_service( repo=ServicesRepository(app.state.engine), + director_api=get_director_api(app), product_name=product_name, user_id=user_id, service_key=service_key, @@ -100,6 +103,7 @@ async def update_service( service = await services.update_service( repo=ServicesRepository(app.state.engine), + director_api=get_director_api(app), product_name=product_name, user_id=user_id, service_key=service_key, diff --git a/services/catalog/src/simcore_service_catalog/services/services.py b/services/catalog/src/simcore_service_catalog/services/services.py index 5a8b044d47d..582a947cb48 100644 --- a/services/catalog/src/simcore_service_catalog/services/services.py +++ b/services/catalog/src/simcore_service_catalog/services/services.py @@ -7,8 +7,8 @@ ) from models_library.products import ProductName from models_library.rest_pagination import PageLimitInt -from models_library.services_authoring import Author from models_library.services_enums import ServiceType +from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID from pydantic import NonNegativeInt @@ -21,6 +21,8 @@ ServiceMetaDataAtDB, ServiceWithHistoryFromDB, ) +from simcore_service_catalog.services import manifest +from simcore_service_catalog.services.director import DirectorApi from ..db.repositories.services import ServicesRepository from .function_services import is_function_service @@ -39,29 +41,33 @@ def _deduce_service_type_from(key: str) -> ServiceType: def _db_to_api_model( service_db: ServiceWithHistoryFromDB, access_rights_db: list[ServiceAccessRightsAtDB], + service_manifest: ServiceMetaDataPublished, ) -> ServiceGetV2: + assert ( + _deduce_service_type_from(service_db.key) == service_manifest.service_type + ) # nosec return ServiceGetV2( key=service_db.key, version=service_db.version, name=service_db.name, thumbnail=service_db.thumbnail or None, description=service_db.description, - version_display=f"V{service_db.version}", # rg.version_display, - type=_deduce_service_type_from(service_db.key), # rg.service_type, - contact=Author.Config.schema_extra["examples"][0]["email"], # rg.contact, - authors=Author.Config.schema_extra["examples"], + version_display=service_manifest.version_display, + type=service_manifest.service_type, + contact=service_manifest.contact, + authors=service_manifest.authors, owner=service_db.owner_email or None, - inputs={}, # rg.inputs, - outputs={}, # rg.outputs, - boot_options=None, # rg.boot_options, - min_visible_inputs=None, # rg.min_visible_inputs, + inputs=service_manifest.inputs or {}, + outputs=service_manifest.outputs or {}, + boot_options=service_manifest.boot_options, + min_visible_inputs=service_manifest.min_visible_inputs, access_rights={ a.gid: ServiceGroupAccessRightsV2.construct( execute=a.execute_access, write=a.write_access, ) for a in access_rights_db - }, # db.access_rights, + }, classifiers=service_db.classifiers, quality=service_db.quality, history=[h.to_api_model() for h in service_db.history], @@ -70,6 +76,7 @@ def _db_to_api_model( async def list_services_paginated( repo: ServicesRepository, + director_api: DirectorApi, product_name: ProductName, user_id: UserID, limit: PageLimitInt | None, @@ -94,11 +101,23 @@ async def list_services_paginated( product_name=product_name, ) + # get manifest of those with access rights + got = await manifest.get_batch_services( + [(s.key, s.version) for s in services if access_rights.get((s.key, s.version))], + director_api, + ) + service_manifest = { + (s.key, s.version): s for s in got if isinstance(s, ServiceMetaDataPublished) + } + # NOTE: aggregates published (i.e. not editable) is still missing in this version items = [ - _db_to_api_model(s, ar) + _db_to_api_model(s, ar, sm) for s in services - if (ar := access_rights.get((s.key, s.version))) + if ( + (ar := access_rights.get((s.key, s.version))) + and (sm := service_manifest.get((s.key, s.version))) + ) ] return total_count, items @@ -106,7 +125,7 @@ async def list_services_paginated( async def get_service( repo: ServicesRepository, - # image_registry, + director_api: DirectorApi, product_name: ProductName, user_id: UserID, service_key: ServiceKey, @@ -142,11 +161,16 @@ async def get_service( product_name=product_name, ) - return _db_to_api_model(service, access_rights) + service_manifest = await manifest.get_service( + service_key, service_version, director_api + ) + + return _db_to_api_model(service, access_rights, service_manifest) async def update_service( repo: ServicesRepository, + director_api: DirectorApi, *, product_name: ProductName, user_id: UserID, @@ -229,6 +253,7 @@ async def update_service( return await get_service( repo=repo, + director_api=director_api, product_name=product_name, user_id=user_id, service_key=service_key, diff --git a/services/catalog/tests/unit/with_dbs/test_services_services.py b/services/catalog/tests/unit/with_dbs/test_services_services.py index 7cc10705ddb..3117a70fe1b 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services.py @@ -3,10 +3,14 @@ # pylint: disable=unused-variable from collections.abc import Callable +from typing import Any import pytest +from fastapi import FastAPI from models_library.products import ProductName from models_library.users import UserID +from respx.router import MockRouter +from simcore_service_catalog.api.dependencies.director import get_director_api from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.services import services from sqlalchemy.ext.asyncio import AsyncEngine @@ -24,36 +28,65 @@ def services_repo(sqlalchemy_async_engine: AsyncEngine): return ServicesRepository(sqlalchemy_async_engine) -async def test_list_services_paginated( +num_services = 5 +num_versions_per_service = 20 + + +@pytest.fixture +def fake_services_data( target_product: ProductName, create_fake_service_data: Callable, +): + return [ + create_fake_service_data( + f"simcore/services/dynamic/some-service-{n}", + f"{v}.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + for n in range(num_services) + for v in range(num_versions_per_service) + ] + + +@pytest.fixture +def expected_director_list_services( + expected_director_list_services: list[dict[str, Any]], fake_services_data: list +) -> list[dict[str, Any]]: + expected = [] + + return expected + + +async def test_list_services_paginated( + background_tasks_setup_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_service_api: MockRouter, + target_product: ProductName, + fake_services_data: list, services_db_tables_injector: Callable, services_repo: ServicesRepository, user_id: UserID, + app: FastAPI, ): # inject services - num_services = 5 - num_versions_per_service = 20 - await services_db_tables_injector( - [ - create_fake_service_data( - f"simcore/services/dynamic/some-service-{n}", - f"{v}.0.0", - team_access=None, - everyone_access=None, - product=target_product, - ) - for n in range(num_services) - for v in range(num_versions_per_service) - ] - ) + await services_db_tables_injector(fake_services_data) limit = 2 assert limit < num_services offset = 1 + # ---- + director_api = get_director_api(app) + + # --- + + assert not mocked_director_service_api["get_service"].called + total_count, page_items = await services.list_services_paginated( services_repo, + director_api, product_name=target_product, user_id=user_id, limit=limit, @@ -62,6 +95,7 @@ async def test_list_services_paginated( assert total_count == num_services assert len(page_items) <= limit + assert mocked_director_service_api["get_service"].called for item in page_items: assert item.access_rights @@ -70,6 +104,7 @@ async def test_list_services_paginated( got = await services.get_service( services_repo, + director_api, product_name=target_product, user_id=user_id, service_key=item.key, From 3ce4b53c8dba208b990567d1fdf2e3ef8be7241f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:29:19 +0200 Subject: [PATCH 04/17] rename --- .../src/simcore_service_catalog/api/rpc/_services.py | 8 ++++---- .../services/{services.py => services_api.py} | 0 ...services_services.py => test_services_services_api.py} | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename services/catalog/src/simcore_service_catalog/services/{services.py => services_api.py} (100%) rename services/catalog/tests/unit/with_dbs/{test_services_services.py => test_services_services_api.py} (94%) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 8d029be9138..20df47ad97a 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -19,7 +19,7 @@ ) from ...db.repositories.services import ServicesRepository -from ...services import services +from ...services import services_api from ..dependencies.director import get_director_api _logger = logging.getLogger(__name__) @@ -39,7 +39,7 @@ async def list_services_paginated( ) -> PageRpcServicesGetV2: assert app.state.engine # nosec - total_count, items = await services.list_services_paginated( + total_count, items = await services_api.list_services_paginated( repo=ServicesRepository(app.state.engine), director_api=get_director_api(app), product_name=product_name, @@ -71,7 +71,7 @@ async def get_service( ) -> ServiceGetV2: assert app.state.engine # nosec - service = await services.get_service( + service = await services_api.get_service( repo=ServicesRepository(app.state.engine), director_api=get_director_api(app), product_name=product_name, @@ -101,7 +101,7 @@ async def update_service( assert app.state.engine # nosec - service = await services.update_service( + service = await services_api.update_service( repo=ServicesRepository(app.state.engine), director_api=get_director_api(app), product_name=product_name, diff --git a/services/catalog/src/simcore_service_catalog/services/services.py b/services/catalog/src/simcore_service_catalog/services/services_api.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/services/services.py rename to services/catalog/src/simcore_service_catalog/services/services_api.py diff --git a/services/catalog/tests/unit/with_dbs/test_services_services.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py similarity index 94% rename from services/catalog/tests/unit/with_dbs/test_services_services.py rename to services/catalog/tests/unit/with_dbs/test_services_services_api.py index 3117a70fe1b..ebbaeac9026 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -12,7 +12,7 @@ from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api from simcore_service_catalog.db.repositories.services import ServicesRepository -from simcore_service_catalog.services import services +from simcore_service_catalog.services import services_api from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = [ @@ -84,7 +84,7 @@ async def test_list_services_paginated( assert not mocked_director_service_api["get_service"].called - total_count, page_items = await services.list_services_paginated( + total_count, page_items = await services_api.list_services_paginated( services_repo, director_api, product_name=target_product, @@ -102,7 +102,7 @@ async def test_list_services_paginated( assert item.owner is not None assert item.history[0].version == item.version - got = await services.get_service( + got = await services_api.get_service( services_repo, director_api, product_name=target_product, From 56a5fbf1ae82bf00bd495e6c5bb6566ca0d35be2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 15 Jul 2024 19:13:54 +0200 Subject: [PATCH 05/17] fixtures and manifest.get_service --- .../api/dependencies/services.py | 4 +- .../services/manifest.py | 19 ++--- .../services/services_api.py | 2 +- .../tests/unit/test_services_manifest.py | 4 +- .../with_dbs/test_services_services_api.py | 69 ++++++++++++++----- 5 files changed, 67 insertions(+), 31 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py index 7855bc97ad5..7d55dab5f59 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py @@ -93,8 +93,8 @@ async def get_service_from_manifest( """ try: return await manifest.get_service( - service_key=service_key, - service_version=service_version, + key=service_key, + version=service_version, director_client=director_client, ) diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/services/manifest.py index 75c5a0a74ec..3c0baa62e50 100644 --- a/services/catalog/src/simcore_service_catalog/services/manifest.py +++ b/services/catalog/src/simcore_service_catalog/services/manifest.py @@ -86,21 +86,24 @@ async def get_services_map( @cached( ttl=DIRECTOR_CACHING_TTL, namespace=__name__, - key_builder=lambda f, *ag, **kw: f"{f.__name__}/{kw['service_key']}/{kw['service_version']}", + key_builder=lambda f, *ag, **kw: f"{f.__name__}/{kw['key']}/{kw['version']}", ) async def get_service( - service_key: ServiceKey, - service_version: ServiceVersion, director_client: DirectorApi, + *, + key: ServiceKey, + version: ServiceVersion, ) -> ServiceMetaDataPublished: """ Retrieves service metadata from the docker registry via the director and accounting + + raises if does not exist or if validation fails """ - if is_function_service(service_key): - service = get_function_service(key=service_key, version=service_version) + if is_function_service(key): + service = get_function_service(key=key, version=version) else: service = await director_client.get_service( - service_key=service_key, service_version=service_version + service_key=key, service_version=version ) return service @@ -111,9 +114,7 @@ async def get_batch_services( return await limited_gather( *( - get_service( - service_key=k, service_version=v, director_client=director_client - ) + get_service(key=k, version=v, director_client=director_client) for k, v in selection ), reraise=False, diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index 582a947cb48..2d59d6c39f3 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -162,7 +162,7 @@ async def get_service( ) service_manifest = await manifest.get_service( - service_key, service_version, director_api + key=service_key, service_version=service_version, director_client=director_api ) return _db_to_api_model(service, access_rights, service_manifest) diff --git a/services/catalog/tests/unit/test_services_manifest.py b/services/catalog/tests/unit/test_services_manifest.py index 1f88e4af318..0164141b306 100644 --- a/services/catalog/tests/unit/test_services_manifest.py +++ b/services/catalog/tests/unit/test_services_manifest.py @@ -60,7 +60,9 @@ async def test_services_manifest_api( # GET for expected_service in all_services_map.values(): service = await manifest.get_service( - expected_service.key, expected_service.version, director_api + key=expected_service.key, + version=expected_service.version, + director_client=director_api, ) assert service == expected_service diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py index ebbaeac9026..94df9e917f9 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -2,13 +2,17 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable +import itertools from collections.abc import Callable from typing import Any import pytest from fastapi import FastAPI +from fastapi.encoders import jsonable_encoder from models_library.products import ProductName +from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.users import UserID +from pydantic import Extra from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api from simcore_service_catalog.db.repositories.services import ServicesRepository @@ -28,18 +32,26 @@ def services_repo(sqlalchemy_async_engine: AsyncEngine): return ServicesRepository(sqlalchemy_async_engine) -num_services = 5 -num_versions_per_service = 20 +@pytest.fixture +def num_services() -> int: + return 5 + + +@pytest.fixture +def num_versions_per_service() -> int: + return 20 @pytest.fixture -def fake_services_data( +def fake_data_for_services( target_product: ProductName, create_fake_service_data: Callable, -): + num_services: int, + num_versions_per_service: int, +) -> list: return [ create_fake_service_data( - f"simcore/services/dynamic/some-service-{n}", + f"simcore/services/comp/some-service-{n}", f"{v}.0.0", team_access=None, everyone_access=None, @@ -52,11 +64,37 @@ def fake_services_data( @pytest.fixture def expected_director_list_services( - expected_director_list_services: list[dict[str, Any]], fake_services_data: list + expected_director_list_services: list[dict[str, Any]], fake_data_for_services: list ) -> list[dict[str, Any]]: - expected = [] + # OVERRIDES: Changes the values returned by the director API by - return expected + class _Loader(ServiceMetaDataPublished): + class Config: + extra = Extra.ignore + allow_population_by_field_name = True + + return [ + jsonable_encoder( + _Loader.parse_obj( + { + **next(itertools.cycle(expected_director_list_services)), + **service_and_access_rights_data[0], # service, **access_rights + } + ), + exclude_unset=True, + ) + for service_and_access_rights_data in fake_data_for_services + ] + + +@pytest.fixture +async def background_tasks_setup_disabled( + background_tasks_setup_disabled: None, + services_db_tables_injector: Callable, + fake_data_for_services: list, +) -> None: + # inject db services (typically done by the sync background task) + await services_db_tables_injector(fake_data_for_services) async def test_list_services_paginated( @@ -64,23 +102,16 @@ async def test_list_services_paginated( rabbitmq_and_rpc_setup_disabled: None, mocked_director_service_api: MockRouter, target_product: ProductName, - fake_services_data: list, - services_db_tables_injector: Callable, services_repo: ServicesRepository, user_id: UserID, app: FastAPI, + num_services: int, ): - # inject services - await services_db_tables_injector(fake_services_data) + director_api = get_director_api(app) + offset = 1 limit = 2 assert limit < num_services - offset = 1 - - # ---- - director_api = get_director_api(app) - - # --- assert not mocked_director_service_api["get_service"].called @@ -94,8 +125,10 @@ async def test_list_services_paginated( ) assert total_count == num_services + assert page_items assert len(page_items) <= limit assert mocked_director_service_api["get_service"].called + assert mocked_director_service_api["get_service"].call_count == limit for item in page_items: assert item.access_rights From b84209318ccce488753bc9b9fcfa6165ab8c9cd5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 15 Jul 2024 19:16:53 +0200 Subject: [PATCH 06/17] fixes cache keys --- .../catalog/src/simcore_service_catalog/services/manifest.py | 3 ++- .../src/simcore_service_catalog/services/services_api.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/services/manifest.py index 3c0baa62e50..42144f00722 100644 --- a/services/catalog/src/simcore_service_catalog/services/manifest.py +++ b/services/catalog/src/simcore_service_catalog/services/manifest.py @@ -109,7 +109,8 @@ async def get_service( async def get_batch_services( - selection: list[tuple[ServiceKey, ServiceVersion]], director_client: DirectorApi + selection: list[tuple[ServiceKey, ServiceVersion]], + director_client: DirectorApi, ) -> list[ServiceMetaDataPublished | BaseException]: return await limited_gather( diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index 2d59d6c39f3..28fd0ebe347 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -162,7 +162,9 @@ async def get_service( ) service_manifest = await manifest.get_service( - key=service_key, service_version=service_version, director_client=director_api + key=service_key, + version=service_version, + director_client=director_api, ) return _db_to_api_model(service, access_rights, service_manifest) From 485ae6fdced5d0a38886c148a91416774ac1c64b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 15 Jul 2024 19:20:03 +0200 Subject: [PATCH 07/17] sqlalchemy 2 compatibility --- .../simcore_service_catalog/db/repositories/services.py | 9 +++------ .../tests/unit/with_dbs/test_services_services_api.py | 3 +++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/db/repositories/services.py index b9ad6c745df..9b787e3741b 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/services.py @@ -432,12 +432,9 @@ async def list_services_access_rights( ) async with self.db_engine.connect() as conn: async for row in await conn.stream(query): - service_to_access_rights[ - ( - row[services_access_rights.c.key], - row[services_access_rights.c.version], - ) - ].append(ServiceAccessRightsAtDB.from_orm(row)) + service_to_access_rights[(row.key, row.version)].append( + ServiceAccessRightsAtDB.from_orm(row) + ) return service_to_access_rights async def upsert_service_access_rights( diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py index 94df9e917f9..22cb773ed52 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -145,3 +145,6 @@ async def test_list_services_paginated( ) assert got == item + + # since it is cached, it should only call it `limit` times + assert mocked_director_service_api["get_service"].call_count == limit From 329a619f0f12c84d0e2d6aa58caee9e4725dff0c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 15 Jul 2024 19:42:07 +0200 Subject: [PATCH 08/17] fixes director --- .../catalog/tests/unit/test_services_director.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/services/catalog/tests/unit/test_services_director.py b/services/catalog/tests/unit/test_services_director.py index b809e7a8315..7458ad6f7ad 100644 --- a/services/catalog/tests/unit/test_services_director.py +++ b/services/catalog/tests/unit/test_services_director.py @@ -11,6 +11,7 @@ import pytest from fastapi import FastAPI +from models_library.services_metadata_published import ServiceMetaDataPublished from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter @@ -35,6 +36,7 @@ def app_environment( async def test_director_client_high_level_api( background_tasks_setup_disabled: None, rabbitmq_and_rpc_setup_disabled: None, + expected_director_list_services: list[dict[str, Any]], mocked_director_service_api: MockRouter, app: FastAPI, ): @@ -47,20 +49,13 @@ async def test_director_client_high_level_api( # PING assert await director_api.is_responsive() - # LIST - all_services = await director_api.list_all_services() - assert mocked_director_service_api["list_services"].called - - services_image_digest = {service.image_digest for service in all_services} - assert None not in services_image_digest - assert len(services_image_digest) == len(all_services) - # GET - expected_service = all_services[0] + expected_service = ServiceMetaDataPublished(**expected_director_list_services[0]) assert ( await director_api.get_service(expected_service.key, expected_service.version) == expected_service ) + # TODO: error handling! async def test_director_client_low_level_api( From 14b726315c0045157e1d6b5b0706634a6c0b7c04 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 15 Jul 2024 19:48:04 +0200 Subject: [PATCH 09/17] fixe batch tests --- .../catalog/tests/unit/test_services_manifest.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/services/catalog/tests/unit/test_services_manifest.py b/services/catalog/tests/unit/test_services_manifest.py index 0164141b306..a43d82d5220 100644 --- a/services/catalog/tests/unit/test_services_manifest.py +++ b/services/catalog/tests/unit/test_services_manifest.py @@ -71,7 +71,11 @@ async def test_services_manifest_api( # BATCH for expected_services in toolz.partition(2, all_services_map.values()): - got_services = await manifest.get_batch_services( - [(s.key, s.value) for s in expected_services], director_api - ) - assert got_services == expected_services + selection = [(s.key, s.version) for s in expected_services] + got_services = await manifest.get_batch_services(selection, director_api) + + assert [(s.key, s.version) for s in got_services] == selection + + # NOTE: simplier to visualize + for got, expected in zip(got_services, expected_services, strict=True): + assert got == expected From d620a604dbb8197fa2cdaf3648d98910b796f56d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:00:43 +0200 Subject: [PATCH 10/17] enabing api-rpc --- .../catalog/tests/unit/with_dbs/conftest.py | 39 +++++++++- .../tests/unit/with_dbs/test_api_rpc.py | 74 +++++++++++++------ .../with_dbs/test_services_services_api.py | 40 +++------- 3 files changed, 103 insertions(+), 50 deletions(-) diff --git a/services/catalog/tests/unit/with_dbs/conftest.py b/services/catalog/tests/unit/with_dbs/conftest.py index f720e89bcdc..0d4aea3d4e2 100644 --- a/services/catalog/tests/unit/with_dbs/conftest.py +++ b/services/catalog/tests/unit/with_dbs/conftest.py @@ -14,10 +14,11 @@ import pytest import sqlalchemy as sa from faker import Faker +from fastapi.encoders import jsonable_encoder from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import Extra, parse_obj_as from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.postgres_tools import ( PostgresTestConfig, @@ -447,3 +448,39 @@ def _fake_factory( return tuple(fakes) return _fake_factory + + +@pytest.fixture +def create_director_list_services_from() -> Callable[ + [list[dict[str, Any]], list], list[dict[str, Any]] +]: + """Convenience function to merge outputs of + - `create_fake_service_data` callable with those of + - `expected_director_list_services` fixture + + to produce a new expected_director_list_services + """ + + class _Loader(ServiceMetaDataPublished): + class Config: + extra = Extra.ignore + allow_population_by_field_name = True + + def _( + expected_director_list_services: list[dict[str, Any]], + fake_services_data: list, + ): + return [ + jsonable_encoder( + _Loader.parse_obj( + { + **next(itertools.cycle(expected_director_list_services)), + **data[0], # service, **access_rights = data + } + ), + exclude_unset=True, + ) + for data in fake_services_data + ] + + return _ diff --git a/services/catalog/tests/unit/with_dbs/test_api_rpc.py b/services/catalog/tests/unit/with_dbs/test_api_rpc.py index 09e2678b872..817b0efc573 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rpc.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rpc.py @@ -5,6 +5,7 @@ from collections.abc import Callable +from typing import Any import pytest from fastapi import FastAPI @@ -14,6 +15,7 @@ from pydantic import ValidationError from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict +from respx.router import MockRouter from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.catalog.errors import CatalogItemNotFoundError from servicelib.rabbitmq.rpc_interfaces.catalog.services import ( @@ -43,35 +45,65 @@ def app_environment( @pytest.fixture -async def fake_services_inserted_in_db( +def num_services() -> int: + return 5 + + +@pytest.fixture +def num_versions_per_service() -> int: + return 20 + + +@pytest.fixture +def fake_data_for_services( target_product: ProductName, create_fake_service_data: Callable, + num_services: int, + num_versions_per_service: int, +) -> list: + return [ + create_fake_service_data( + f"simcore/services/comp/test-api-rpc-service-{n}", + f"{v}.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + for n in range(num_services) + for v in range(num_versions_per_service) + ] + + +@pytest.fixture +def expected_director_list_services( + expected_director_list_services: list[dict[str, Any]], + fake_data_for_services: list, + create_director_list_services_from: Callable, +) -> list[dict[str, Any]]: + # OVERRIDES: Changes the values returned by the mocked_director_service_api + + return create_director_list_services_from( + expected_director_list_services, fake_data_for_services + ) + + +@pytest.fixture +async def background_sync_task_mocked( + background_tasks_setup_disabled: None, services_db_tables_injector: Callable, + fake_data_for_services: list, ) -> None: - num_services = 5 - num_versions_per_service = 20 - await services_db_tables_injector( - [ - create_fake_service_data( - f"simcore/services/dynamic/some-service-{n}", - f"{v}.0.0", - team_access=None, - everyone_access=None, - product=target_product, - ) - for n in range(num_services) - for v in range(num_versions_per_service) - ] - ) + # inject db services (typically done by the sync background task) + await services_db_tables_injector(fake_data_for_services) async def test_rpc_catalog_client( - director_setup_disabled: None, - fake_services_inserted_in_db: None, - app: FastAPI, + background_sync_task_mocked: None, + mocked_director_service_api: MockRouter, rpc_client: RabbitMQRPCClient, product_name: ProductName, user_id: UserID, + app: FastAPI, ): assert app @@ -136,8 +168,8 @@ async def test_rpc_catalog_client( async def test_rpc_service_not_found_error( - director_setup_disabled: None, - fake_services_inserted_in_db: None, + background_sync_task_mocked: None, + mocked_director_service_api: MockRouter, app: FastAPI, rpc_client: RabbitMQRPCClient, product_name: ProductName, diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py index 22cb773ed52..bfc452e5b03 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -2,17 +2,13 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -import itertools from collections.abc import Callable from typing import Any import pytest from fastapi import FastAPI -from fastapi.encoders import jsonable_encoder from models_library.products import ProductName -from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.users import UserID -from pydantic import Extra from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api from simcore_service_catalog.db.repositories.services import ServicesRepository @@ -43,7 +39,7 @@ def num_versions_per_service() -> int: @pytest.fixture -def fake_data_for_services( +def fake_services_data( target_product: ProductName, create_fake_service_data: Callable, num_services: int, @@ -64,41 +60,29 @@ def fake_data_for_services( @pytest.fixture def expected_director_list_services( - expected_director_list_services: list[dict[str, Any]], fake_data_for_services: list + expected_director_list_services: list[dict[str, Any]], + fake_services_data: list, + create_director_list_services_from: Callable, ) -> list[dict[str, Any]]: - # OVERRIDES: Changes the values returned by the director API by + # OVERRIDES: Changes the values returned by the mocked_director_service_api - class _Loader(ServiceMetaDataPublished): - class Config: - extra = Extra.ignore - allow_population_by_field_name = True - - return [ - jsonable_encoder( - _Loader.parse_obj( - { - **next(itertools.cycle(expected_director_list_services)), - **service_and_access_rights_data[0], # service, **access_rights - } - ), - exclude_unset=True, - ) - for service_and_access_rights_data in fake_data_for_services - ] + return create_director_list_services_from( + expected_director_list_services, fake_services_data + ) @pytest.fixture -async def background_tasks_setup_disabled( +async def background_sync_task_mocked( background_tasks_setup_disabled: None, services_db_tables_injector: Callable, - fake_data_for_services: list, + fake_services_data: list, ) -> None: # inject db services (typically done by the sync background task) - await services_db_tables_injector(fake_data_for_services) + await services_db_tables_injector(fake_services_data) async def test_list_services_paginated( - background_tasks_setup_disabled: None, + background_sync_task_mocked: None, rabbitmq_and_rpc_setup_disabled: None, mocked_director_service_api: MockRouter, target_product: ProductName, From f4e3ddf2f51ceaf71db77e30ecedb5c40de6d3b3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:34:18 +0200 Subject: [PATCH 11/17] mypy --- .../catalog/src/simcore_service_catalog/services/manifest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/services/manifest.py index 42144f00722..59149111780 100644 --- a/services/catalog/src/simcore_service_catalog/services/manifest.py +++ b/services/catalog/src/simcore_service_catalog/services/manifest.py @@ -113,7 +113,7 @@ async def get_batch_services( director_client: DirectorApi, ) -> list[ServiceMetaDataPublished | BaseException]: - return await limited_gather( + batch: list[ServiceMetaDataPublished | BaseException] = await limited_gather( *( get_service(key=k, version=v, director_client=director_client) for k, v in selection @@ -122,3 +122,4 @@ async def get_batch_services( log=_logger, tasks_group_prefix="manifest.get_batch_services", ) + return batch From a7628b510bc17ed9ffcf9f2aa326edc29f8fc2ac Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:08:20 +0200 Subject: [PATCH 12/17] resets cache --- .../with_dbs/test_api_rest_services__get.py | 4 ++++ .../with_dbs/test_services_services_api.py | 21 ++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py index aa6e569fff8..a13f0f87add 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py @@ -15,6 +15,7 @@ from models_library.api_schemas_catalog.services import ServiceGet from models_library.products import ProductName from models_library.users import UserID +from simcore_service_catalog.services import manifest from yarl import URL pytest_simcore_core_services_selection = [ @@ -73,6 +74,9 @@ def test_get_service_with_details( target_product: ProductName, client: TestClient, ): + assert hasattr(manifest.get_service, "cache") + assert manifest.get_service.cache.clear() + service_key = expected_service["key"] service_version = expected_service["version"] diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py index bfc452e5b03..63f0fd06e56 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -12,7 +12,8 @@ from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api from simcore_service_catalog.db.repositories.services import ServicesRepository -from simcore_service_catalog.services import services_api +from simcore_service_catalog.services import manifest, services_api +from simcore_service_catalog.services.director import DirectorApi from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = [ @@ -81,6 +82,17 @@ async def background_sync_task_mocked( await services_db_tables_injector(fake_services_data) +@pytest.fixture +async def director_client(app: FastAPI) -> DirectorApi: + director_api = get_director_api(app) + + # ensures manifest API cache is reset + assert hasattr(manifest.get_service, "cache") + assert manifest.get_service.cache.clear() + + return director_api + + async def test_list_services_paginated( background_sync_task_mocked: None, rabbitmq_and_rpc_setup_disabled: None, @@ -88,10 +100,9 @@ async def test_list_services_paginated( target_product: ProductName, services_repo: ServicesRepository, user_id: UserID, - app: FastAPI, + director_client: DirectorApi, num_services: int, ): - director_api = get_director_api(app) offset = 1 limit = 2 @@ -101,7 +112,7 @@ async def test_list_services_paginated( total_count, page_items = await services_api.list_services_paginated( services_repo, - director_api, + director_client, product_name=target_product, user_id=user_id, limit=limit, @@ -121,7 +132,7 @@ async def test_list_services_paginated( got = await services_api.get_service( services_repo, - director_api, + director_client, product_name=target_product, user_id=user_id, service_key=item.key, From a3fa38df2a3d10bfd0062d1c9450f0f1f4cd9761 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:12:30 +0200 Subject: [PATCH 13/17] makes less noisy --- .../src/pytest_simcore/docker_compose.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index a242e819b19..797584e26ec 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -167,20 +167,13 @@ def simcore_docker_compose( docker_compose_path.exists() for docker_compose_path in docker_compose_paths ) - compose_specs = run_docker_compose_config( + return run_docker_compose_config( project_dir=osparc_simcore_root_dir / "services", scripts_dir=osparc_simcore_scripts_dir, docker_compose_paths=docker_compose_paths, env_file_path=env_file_for_testing, destination_path=temp_folder / "simcore_docker_compose.yml", ) - # NOTE: do not add indent. Copy&Paste log into editor instead - print( - HEADER_STR.format("simcore docker-compose"), - json.dumps(compose_specs), - HEADER_STR.format("-"), - ) - return compose_specs @pytest.fixture(scope="module") @@ -203,20 +196,13 @@ def ops_docker_compose( ) assert docker_compose_path.exists() - compose_specs = run_docker_compose_config( + return run_docker_compose_config( project_dir=osparc_simcore_root_dir / "services", scripts_dir=osparc_simcore_scripts_dir, docker_compose_paths=docker_compose_path, env_file_path=env_file_for_testing, destination_path=temp_folder / "ops_docker_compose.yml", ) - # NOTE: do not add indent. Copy&Paste log into editor instead - print( - HEADER_STR.format("ops docker-compose"), - json.dumps(compose_specs), - HEADER_STR.format("-"), - ) - return compose_specs @pytest.fixture(scope="module") @@ -245,6 +231,11 @@ def core_docker_compose_file( core_services_selection, simcore_docker_compose, docker_compose_path ) + print( + HEADER_STR.format("simcore docker-compose"), + json.dumps(docker_compose_path.read_text()), + HEADER_STR.format("-"), + ) return docker_compose_path @@ -281,6 +272,11 @@ def ops_docker_compose_file( ops_services_selection, ops_docker_compose, docker_compose_path ) + print( + HEADER_STR.format("ops docker-compose"), + json.dumps(docker_compose_path.read_text()), + HEADER_STR.format("-"), + ) return docker_compose_path From c48b0f182b3a92a6deb1afecbce87cfbe1e3b85d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:15:08 +0200 Subject: [PATCH 14/17] print --- packages/pytest-simcore/src/pytest_simcore/docker_compose.py | 4 ++-- .../pytest-simcore/src/pytest_simcore/helpers/constants.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index 797584e26ec..6cbbe4fb368 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -232,7 +232,7 @@ def core_docker_compose_file( ) print( - HEADER_STR.format("simcore docker-compose"), + HEADER_STR.format(docker_compose_path), json.dumps(docker_compose_path.read_text()), HEADER_STR.format("-"), ) @@ -273,7 +273,7 @@ def ops_docker_compose_file( ) print( - HEADER_STR.format("ops docker-compose"), + HEADER_STR.format(docker_compose_path), json.dumps(docker_compose_path.read_text()), HEADER_STR.format("-"), ) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/constants.py b/packages/pytest-simcore/src/pytest_simcore/helpers/constants.py index cd6edbf427f..5d517b9a071 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/constants.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/constants.py @@ -7,4 +7,4 @@ # string templates -HEADER_STR: str = "{:-^50}\n" +HEADER_STR: str = "{:-^100}\n" From dc4b3d23963bfd750b531ec7b878c6fd77e32de5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:18:13 +0200 Subject: [PATCH 15/17] fixes fixture --- packages/pytest-simcore/src/pytest_simcore/docker_compose.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index 6cbbe4fb368..aa1e2ebecee 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -232,7 +232,7 @@ def core_docker_compose_file( ) print( - HEADER_STR.format(docker_compose_path), + HEADER_STR.format(f"{docker_compose_path}"), json.dumps(docker_compose_path.read_text()), HEADER_STR.format("-"), ) @@ -273,7 +273,7 @@ def ops_docker_compose_file( ) print( - HEADER_STR.format(docker_compose_path), + HEADER_STR.format(f"{docker_compose_path}"), json.dumps(docker_compose_path.read_text()), HEADER_STR.format("-"), ) From 7fb9b4dbca5de7010bb1116ae60939aed2989a53 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:19:04 +0200 Subject: [PATCH 16/17] tmp --- .../catalog/tests/unit/with_dbs/test_api_rest_services__get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py index a13f0f87add..7f9518f325c 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py @@ -97,4 +97,4 @@ def test_get_service_with_details( assert got.key == service_key assert got.version == service_version - assert mocked_director_service_api["get_service"].called + # assert mocked_director_service_api["get_service"].called From 0c36058c2f5ec336eda6b80c268ec066dc0de70d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:38:27 +0200 Subject: [PATCH 17/17] disable caching --- .../tests/unit/with_dbs/test_api_rest_services__get.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py index 7f9518f325c..a3c85d3f31b 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py @@ -15,7 +15,6 @@ from models_library.api_schemas_catalog.services import ServiceGet from models_library.products import ProductName from models_library.users import UserID -from simcore_service_catalog.services import manifest from yarl import URL pytest_simcore_core_services_selection = [ @@ -66,6 +65,7 @@ async def expected_service( def test_get_service_with_details( + service_caching_disabled: None, background_tasks_setup_disabled: None, rabbitmq_and_rpc_setup_disabled: None, mocked_director_service_api: respx.MockRouter, @@ -74,9 +74,6 @@ def test_get_service_with_details( target_product: ProductName, client: TestClient, ): - assert hasattr(manifest.get_service, "cache") - assert manifest.get_service.cache.clear() - service_key = expected_service["key"] service_version = expected_service["version"] @@ -97,4 +94,4 @@ def test_get_service_with_details( assert got.key == service_key assert got.version == service_version - # assert mocked_director_service_api["get_service"].called + assert mocked_director_service_api["get_service"].called