♻️ Datcore-adapter refactoring (#7270)

Author: sanderegg

services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/__init__.py renamed to packages/models-library/src/models_library/api_schemas_datcore_adapter/__init__.py

@@ -0,0 +1,41 @@
+from datetime import datetime
+from enum import Enum, unique
+from pathlib import Path
+
+from pydantic import BaseModel, ByteSize
+
+
+class DatasetMetaData(BaseModel):
+    id: str
+    display_name: str
+
+
+@unique
+class DataType(str, Enum):
+    FILE = "FILE"
+    FOLDER = "FOLDER"
+
+
+class PackageMetaData(BaseModel):
+    path: Path
+    display_path: Path
+    package_id: str
+    name: str
+    filename: str
+    s3_bucket: str
+    size: ByteSize
+    created_at: datetime
+    updated_at: datetime
+
+
+class FileMetaData(BaseModel):
+    dataset_id: str
+    package_id: str
+    id: str
+    name: str
+    type: str
+    path: Path
+    size: int
+    created_at: datetime
+    last_modified_at: datetime
+    data_type: DataType

packages/models-library/src/models_library/api_schemas_datcore_adapter/datasets.py

@@ -32,7 +32,7 @@ def assert_status(
         response.status_code == expected_status_code
     ), f"received {response.status_code}: {response.text}, expected {get_code_display_name(expected_status_code)}"
 
-    # reponse
+    # response
     if expected_status_code == status.HTTP_204_NO_CONTENT:
         assert response.text == ""
         return None, None

packages/pytest-simcore/src/pytest_simcore/helpers/httpx_assert_checks.py

@@ -99,3 +99,17 @@ def set_app_default_http_error_handlers(app: FastAPI) -> None:
             envelope_error=True,
         ),
     )
+
+    # SEE https://docs.python.org/3/library/exceptions.html#exception-hierarchy
+    app.add_exception_handler(
+        NotImplementedError,
+        make_http_error_handler_for_exception(
+            status.HTTP_501_NOT_IMPLEMENTED, NotImplementedError, envelope_error=True
+        ),
+    )
+    app.add_exception_handler(
+        Exception,
+        make_http_error_handler_for_exception(
+            status.HTTP_500_INTERNAL_SERVER_ERROR, Exception, envelope_error=True
+        ),
+    )

packages/service-library/src/servicelib/fastapi/http_error.py

@@ -6,6 +6,8 @@
 
 
 asgi_lifespan
+botocore-stubs
+boto3-stubs
 coverage
 faker
 pytest
@@ -19,5 +21,3 @@ pytest-sugar
 pytest-xdist
 requests
 respx
-types-boto3
-types-botocore

services/datcore-adapter/requirements/_test.in

@@ -4,10 +4,12 @@ anyio==4.3.0
     #   httpx
 asgi-lifespan==2.1.0
     # via -r requirements/_test.in
+boto3-stubs==1.37.0
+    # via -r requirements/_test.in
 botocore-stubs==1.36.6
     # via
-    #   types-boto3
-    #   types-botocore
+    #   -r requirements/_test.in
+    #   boto3-stubs
 certifi==2024.2.2
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -110,17 +112,13 @@ termcolor==2.5.0
     # via pytest-sugar
 types-awscrt==0.23.7
     # via botocore-stubs
-types-boto3==1.36.6
-    # via -r requirements/_test.in
-types-botocore==1.0.2
-    # via -r requirements/_test.in
 types-s3transfer==0.11.2
-    # via types-boto3
+    # via boto3-stubs
 typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
+    #   boto3-stubs
     #   faker
-    #   types-boto3
 urllib3==2.2.3
     # via
     #   -c requirements/../../../requirements/constraints.txt

services/datcore-adapter/requirements/_test.txt

@@ -6,6 +6,6 @@
 assert get_app  # nosec
 
 __all__: tuple[str, ...] = (
-    "get_reverse_url_mapper",
     "get_app",
+    "get_reverse_url_mapper",
 )

services/datcore-adapter/src/simcore_service_datcore_adapter/api/dependencies/application.py

@@ -1,4 +1,4 @@
-from typing import cast
+from typing import Annotated, cast
 
 from fastapi import Depends, FastAPI
 from fastapi.requests import Request
@@ -11,7 +11,7 @@ def _get_app(request: Request) -> FastAPI:
 
 
 def get_pennsieve_api_client(
-    app: FastAPI = Depends(_get_app),
+    app: Annotated[FastAPI, Depends(_get_app)],
 ) -> PennsieveApiClient:
     client = PennsieveApiClient.get_instance(app)
     assert client  # nosec

services/datcore-adapter/src/simcore_service_datcore_adapter/api/dependencies/pennsieve.py

@@ -6,10 +6,13 @@
 from fastapi_pagination import Page, Params
 from fastapi_pagination.api import create_page, resolve_params
 from fastapi_pagination.bases import RawParams
+from models_library.api_schemas_datcore_adapter.datasets import (
+    DatasetMetaData,
+    FileMetaData,
+)
 from servicelib.fastapi.requests_decorators import cancel_on_disconnect
 from starlette import status
 
-from ...models.domains.datasets import DatasetsOut, FileMetaDataOut
 from ...modules.pennsieve import PennsieveApiClient
 from ..dependencies.pennsieve import get_pennsieve_api_client
 
@@ -26,7 +29,7 @@
     "/datasets",
     summary="list datasets",
     status_code=status.HTTP_200_OK,
-    response_model=Page[DatasetsOut],
+    response_model=Page[DatasetMetaData],
 )
 @cancel_on_disconnect
 @cached(
@@ -39,7 +42,7 @@ async def list_datasets(
     x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
     pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
     params: Annotated[Params, Depends()],
-) -> Page[DatasetsOut]:
+) -> Page[DatasetMetaData]:
     assert request  # nosec
     raw_params: RawParams = resolve_params(params).to_raw_params()
     assert raw_params.limit is not None  # nosec
@@ -57,7 +60,7 @@ async def list_datasets(
     "/datasets/{dataset_id}/files",
     summary="list top level files/folders in a dataset",
     status_code=status.HTTP_200_OK,
-    response_model=Page[FileMetaDataOut],
+    response_model=Page[FileMetaData],
 )
 @cancel_on_disconnect
 @cached(
@@ -71,7 +74,7 @@ async def list_dataset_top_level_files(
     x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
     pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
     params: Annotated[Params, Depends()],
-) -> Page[FileMetaDataOut]:
+) -> Page[FileMetaData]:
     assert request  # nosec
     raw_params: RawParams = resolve_params(params).to_raw_params()
 
@@ -91,7 +94,7 @@ async def list_dataset_top_level_files(
     "/datasets/{dataset_id}/files/{collection_id}",
     summary="list top level files/folders in a collection in a dataset",
     status_code=status.HTTP_200_OK,
-    response_model=Page[FileMetaDataOut],
+    response_model=Page[FileMetaData],
 )
 @cancel_on_disconnect
 @cached(
@@ -106,7 +109,7 @@ async def list_dataset_collection_files(
     x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
     pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
     params: Annotated[Params, Depends()],
-) -> Page[FileMetaDataOut]:
+) -> Page[FileMetaData]:
     assert request  # nosec
     raw_params: RawParams = resolve_params(params).to_raw_params()
     assert raw_params.limit is not None  # nosec
@@ -126,7 +129,7 @@ async def list_dataset_collection_files(
     "/datasets/{dataset_id}/files_legacy",
     summary="list all file meta data in dataset",
     status_code=status.HTTP_200_OK,
-    response_model=list[FileMetaDataOut],
+    response_model=list[FileMetaData],
 )
 @cancel_on_disconnect
 @cached(
@@ -139,7 +142,7 @@ async def list_dataset_files_legacy(
     x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
     x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
     pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
-) -> list[FileMetaDataOut]:
+) -> list[FileMetaData]:
     assert request  # nosec
     return await pennsieve_client.list_all_dataset_files(
         api_key=x_datcore_api_key,

services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/http_error.py

@@ -1,17 +1,18 @@
 import logging
-from typing import Annotated, Any
+from typing import Annotated
 
 from fastapi import APIRouter, Depends, Header, Request
+from models_library.api_schemas_datcore_adapter.datasets import PackageMetaData
 from pydantic import AnyUrl, TypeAdapter
 from servicelib.fastapi.requests_decorators import cancel_on_disconnect
 from starlette import status
 
-from ...models.domains.files import FileDownloadOut
+from ...models.files import FileDownloadOut
 from ...modules.pennsieve import PennsieveApiClient
 from ..dependencies.pennsieve import get_pennsieve_api_client
 
 router = APIRouter()
-log = logging.getLogger(__file__)
+_logger = logging.getLogger(__file__)
 
 
 @router.get(
@@ -62,7 +63,7 @@ async def delete_file(
     "/packages/{package_id}/files",
     summary="returns a package (i.e. a file)",
     status_code=status.HTTP_200_OK,
-    response_model=list[dict[str, Any]],
+    response_model=list[PackageMetaData],
 )
 @cancel_on_disconnect
 async def get_package(
@@ -71,12 +72,15 @@ async def get_package(
     x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
     x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
     pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
-) -> list[dict[str, Any]]:
+) -> list[PackageMetaData]:
     assert request  # nosec
-    return await pennsieve_client.get_package_files(
+
+    data = await pennsieve_client.get_package_files(
         api_key=x_datcore_api_key,
         api_secret=x_datcore_api_secret,
         package_id=package_id,
         limit=1,
         offset=0,
+        fill_path=True,
     )
+    return [_.to_api_model() for _ in data]

services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/validation_error.py

@@ -1,6 +1,6 @@
 import logging
 from collections.abc import Callable
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import Annotated
 
 from fastapi import APIRouter, Depends
@@ -24,7 +24,7 @@
     status_code=status.HTTP_200_OK,
 )
 async def get_service_alive():
-    return f"{__name__}@{datetime.now(timezone.utc).isoformat()}"
+    return f"{__name__}@{datetime.now(UTC).isoformat()}"
 
 
 @router.get("/ready", status_code=status.HTTP_200_OK, response_model=AppStatusCheck)