🎨 web-api services api response includes manifest info (part 5) (#6061)

Author: pcrespov

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -167,20 +167,13 @@ def simcore_docker_compose(
         docker_compose_path.exists() for docker_compose_path in docker_compose_paths
     )
 
-    compose_specs = run_docker_compose_config(
+    return run_docker_compose_config(
         project_dir=osparc_simcore_root_dir / "services",
         scripts_dir=osparc_simcore_scripts_dir,
         docker_compose_paths=docker_compose_paths,
         env_file_path=env_file_for_testing,
         destination_path=temp_folder / "simcore_docker_compose.yml",
     )
-    # NOTE: do not add indent. Copy&Paste log into editor instead
-    print(
-        HEADER_STR.format("simcore docker-compose"),
-        json.dumps(compose_specs),
-        HEADER_STR.format("-"),
-    )
-    return compose_specs
 
 
 @pytest.fixture(scope="module")
@@ -203,20 +196,13 @@ def ops_docker_compose(
     )
     assert docker_compose_path.exists()
 
-    compose_specs = run_docker_compose_config(
+    return run_docker_compose_config(
         project_dir=osparc_simcore_root_dir / "services",
         scripts_dir=osparc_simcore_scripts_dir,
         docker_compose_paths=docker_compose_path,
         env_file_path=env_file_for_testing,
         destination_path=temp_folder / "ops_docker_compose.yml",
     )
-    # NOTE: do not add indent. Copy&Paste log into editor instead
-    print(
-        HEADER_STR.format("ops docker-compose"),
-        json.dumps(compose_specs),
-        HEADER_STR.format("-"),
-    )
-    return compose_specs
 
 
 @pytest.fixture(scope="module")
@@ -245,6 +231,11 @@ def core_docker_compose_file(
         core_services_selection, simcore_docker_compose, docker_compose_path
     )
 
+    print(
+        HEADER_STR.format(f"{docker_compose_path}"),
+        json.dumps(docker_compose_path.read_text()),
+        HEADER_STR.format("-"),
+    )
     return docker_compose_path
 
 
@@ -281,6 +272,11 @@ def ops_docker_compose_file(
         ops_services_selection, ops_docker_compose, docker_compose_path
     )
 
+    print(
+        HEADER_STR.format(f"{docker_compose_path}"),
+        json.dumps(docker_compose_path.read_text()),
+        HEADER_STR.format("-"),
+    )
     return docker_compose_path
 
 

packages/pytest-simcore/src/pytest_simcore/helpers/constants.py

@@ -7,4 +7,4 @@
 
 
 # string templates
-HEADER_STR: str = "{:-^50}\n"
+HEADER_STR: str = "{:-^100}\n"

services/catalog/src/simcore_service_catalog/api/dependencies/services.py

@@ -93,8 +93,8 @@ async def get_service_from_manifest(
     """
     try:
         return await manifest.get_service(
-            service_key=service_key,
-            service_version=service_version,
+            key=service_key,
+            version=service_version,
             director_client=director_client,
         )
 

services/catalog/src/simcore_service_catalog/api/rpc/_services.py

@@ -19,7 +19,8 @@
 )
 
 from ...db.repositories.services import ServicesRepository
-from ...services import catalog
+from ...services import services_api
+from ..dependencies.director import get_director_api
 
 _logger = logging.getLogger(__name__)
 
@@ -38,8 +39,9 @@ async def list_services_paginated(
 ) -> PageRpcServicesGetV2:
     assert app.state.engine  # nosec
 
-    total_count, items = await catalog.list_services_paginated(
+    total_count, items = await services_api.list_services_paginated(
         repo=ServicesRepository(app.state.engine),
+        director_api=get_director_api(app),
         product_name=product_name,
         user_id=user_id,
         limit=limit,
@@ -69,8 +71,9 @@ async def get_service(
 ) -> ServiceGetV2:
     assert app.state.engine  # nosec
 
-    service = await catalog.get_service(
+    service = await services_api.get_service(
         repo=ServicesRepository(app.state.engine),
+        director_api=get_director_api(app),
         product_name=product_name,
         user_id=user_id,
         service_key=service_key,
@@ -98,8 +101,9 @@ async def update_service(
 
     assert app.state.engine  # nosec
 
-    service = await catalog.update_service(
+    service = await services_api.update_service(
         repo=ServicesRepository(app.state.engine),
+        director_api=get_director_api(app),
         product_name=product_name,
         user_id=user_id,
         service_key=service_key,

services/catalog/src/simcore_service_catalog/db/repositories/services.py

@@ -432,12 +432,9 @@ async def list_services_access_rights(
         )
         async with self.db_engine.connect() as conn:
             async for row in await conn.stream(query):
-                service_to_access_rights[
-                    (
-                        row[services_access_rights.c.key],
-                        row[services_access_rights.c.version],
-                    )
-                ].append(ServiceAccessRightsAtDB.from_orm(row))
+                service_to_access_rights[(row.key, row.version)].append(
+                    ServiceAccessRightsAtDB.from_orm(row)
+                )
         return service_to_access_rights
 
     async def upsert_service_access_rights(

services/catalog/src/simcore_service_catalog/services/director.py

@@ -11,7 +11,6 @@
 from models_library.services_metadata_published import ServiceMetaDataPublished
 from models_library.services_types import ServiceKey, ServiceVersion
 from models_library.utils.json_serialization import json_dumps
-from pydantic import parse_obj_as
 from servicelib.logging_utils import log_context
 from starlette import status
 from tenacity._asyncio import AsyncRetrying
@@ -140,12 +139,6 @@ async def is_responsive(self) -> bool:
         except (httpx.HTTPStatusError, httpx.RequestError, httpx.TimeoutException):
             return False
 
-    async def list_all_services(self) -> list[ServiceMetaDataPublished]:
-        # WARNING: this function probably raise ValidationError since director does NOT offer guarantees.
-        # SEE list_registered_services
-        data = await self.get("/services")
-        return parse_obj_as(list[ServiceMetaDataPublished], data)
-
     async def get_service(
         self, service_key: ServiceKey, service_version: ServiceVersion
     ) -> ServiceMetaDataPublished:

services/catalog/src/simcore_service_catalog/services/manifest.py

@@ -27,11 +27,14 @@
 import logging
 from typing import Any, TypeAlias, cast
 
+from aiocache import cached
 from models_library.function_services_catalog.api import iter_service_docker_data
 from models_library.services_metadata_published import ServiceMetaDataPublished
 from models_library.services_types import ServiceKey, ServiceVersion
 from pydantic import ValidationError
+from servicelib.utils import limited_gather
 
+from .._constants import DIRECTOR_CACHING_TTL
 from .director import DirectorApi
 from .function_services import get_function_service, is_function_service
 
@@ -80,18 +83,43 @@ async def get_services_map(
     return services
 
 
+@cached(
+    ttl=DIRECTOR_CACHING_TTL,
+    namespace=__name__,
+    key_builder=lambda f, *ag, **kw: f"{f.__name__}/{kw['key']}/{kw['version']}",
+)
 async def get_service(
-    service_key: ServiceKey,
-    service_version: ServiceVersion,
     director_client: DirectorApi,
+    *,
+    key: ServiceKey,
+    version: ServiceVersion,
 ) -> ServiceMetaDataPublished:
     """
     Retrieves service metadata from the docker registry via the director and accounting
+
+    raises if does not exist or if validation fails
     """
-    if is_function_service(service_key):
-        service = get_function_service(key=service_key, version=service_version)
+    if is_function_service(key):
+        service = get_function_service(key=key, version=version)
     else:
         service = await director_client.get_service(
-            service_key=service_key, service_version=service_version
+            service_key=key, service_version=version
         )
     return service
+
+
+async def get_batch_services(
+    selection: list[tuple[ServiceKey, ServiceVersion]],
+    director_client: DirectorApi,
+) -> list[ServiceMetaDataPublished | BaseException]:
+
+    batch: list[ServiceMetaDataPublished | BaseException] = await limited_gather(
+        *(
+            get_service(key=k, version=v, director_client=director_client)
+            for k, v in selection
+        ),
+        reraise=False,
+        log=_logger,
+        tasks_group_prefix="manifest.get_batch_services",
+    )
+    return batch

services/catalog/src/simcore_service_catalog/services/catalog.py renamed to services/catalog/src/simcore_service_catalog/services/services_api.py

@@ -7,8 +7,8 @@
 )
 from models_library.products import ProductName
 from models_library.rest_pagination import PageLimitInt
-from models_library.services_authoring import Author
 from models_library.services_enums import ServiceType
+from models_library.services_metadata_published import ServiceMetaDataPublished
 from models_library.services_types import ServiceKey, ServiceVersion
 from models_library.users import UserID
 from pydantic import NonNegativeInt
@@ -21,6 +21,8 @@
     ServiceMetaDataAtDB,
     ServiceWithHistoryFromDB,
 )
+from simcore_service_catalog.services import manifest
+from simcore_service_catalog.services.director import DirectorApi
 
 from ..db.repositories.services import ServicesRepository
 from .function_services import is_function_service
@@ -39,29 +41,33 @@ def _deduce_service_type_from(key: str) -> ServiceType:
 def _db_to_api_model(
     service_db: ServiceWithHistoryFromDB,
     access_rights_db: list[ServiceAccessRightsAtDB],
+    service_manifest: ServiceMetaDataPublished,
 ) -> ServiceGetV2:
+    assert (
+        _deduce_service_type_from(service_db.key) == service_manifest.service_type
+    )  # nosec
     return ServiceGetV2(
         key=service_db.key,
         version=service_db.version,
         name=service_db.name,
         thumbnail=service_db.thumbnail or None,
         description=service_db.description,
-        version_display=f"V{service_db.version}",  # rg.version_display,
-        type=_deduce_service_type_from(service_db.key),  # rg.service_type,
-        contact=Author.Config.schema_extra["examples"][0]["email"],  # rg.contact,
-        authors=Author.Config.schema_extra["examples"],
+        version_display=service_manifest.version_display,
+        type=service_manifest.service_type,
+        contact=service_manifest.contact,
+        authors=service_manifest.authors,
         owner=service_db.owner_email or None,
-        inputs={},  # rg.inputs,
-        outputs={},  # rg.outputs,
-        boot_options=None,  # rg.boot_options,
-        min_visible_inputs=None,  # rg.min_visible_inputs,
+        inputs=service_manifest.inputs or {},
+        outputs=service_manifest.outputs or {},
+        boot_options=service_manifest.boot_options,
+        min_visible_inputs=service_manifest.min_visible_inputs,
         access_rights={
             a.gid: ServiceGroupAccessRightsV2.construct(
                 execute=a.execute_access,
                 write=a.write_access,
             )
             for a in access_rights_db
-        },  # db.access_rights,
+        },
         classifiers=service_db.classifiers,
         quality=service_db.quality,
         history=[h.to_api_model() for h in service_db.history],
@@ -70,6 +76,7 @@ def _db_to_api_model(
 
 async def list_services_paginated(
     repo: ServicesRepository,
+    director_api: DirectorApi,
     product_name: ProductName,
     user_id: UserID,
     limit: PageLimitInt | None,
@@ -94,19 +101,31 @@ async def list_services_paginated(
             product_name=product_name,
         )
 
+    # get manifest of those with access rights
+    got = await manifest.get_batch_services(
+        [(s.key, s.version) for s in services if access_rights.get((s.key, s.version))],
+        director_api,
+    )
+    service_manifest = {
+        (s.key, s.version): s for s in got if isinstance(s, ServiceMetaDataPublished)
+    }
+
     # NOTE: aggregates published (i.e. not editable) is still missing in this version
     items = [
-        _db_to_api_model(s, ar)
+        _db_to_api_model(s, ar, sm)
         for s in services
-        if (ar := access_rights.get((s.key, s.version)))
+        if (
+            (ar := access_rights.get((s.key, s.version)))
+            and (sm := service_manifest.get((s.key, s.version)))
+        )
     ]
 
     return total_count, items
 
 
 async def get_service(
     repo: ServicesRepository,
-    # image_registry,
+    director_api: DirectorApi,
     product_name: ProductName,
     user_id: UserID,
     service_key: ServiceKey,
@@ -142,11 +161,18 @@ async def get_service(
             product_name=product_name,
         )
 
-    return _db_to_api_model(service, access_rights)
+    service_manifest = await manifest.get_service(
+        key=service_key,
+        version=service_version,
+        director_client=director_api,
+    )
+
+    return _db_to_api_model(service, access_rights, service_manifest)
 
 
 async def update_service(
     repo: ServicesRepository,
+    director_api: DirectorApi,
     *,
     product_name: ProductName,
     user_id: UserID,
@@ -229,6 +255,7 @@ async def update_service(
 
     return await get_service(
         repo=repo,
+        director_api=director_api,
         product_name=product_name,
         user_id=user_id,
         service_key=service_key,