Merge branch 'master' into is5854/upgrade-to-py311

Author: giancarloromeo

.env-devel

@@ -318,6 +318,7 @@ WEBSERVER_DIAGNOSTICS={}
 WEBSERVER_EMAIL={}
 WEBSERVER_EXPORTER={}
 WEBSERVER_FRONTEND={}
+WEBSERVER_FOLDERS=1
 WEBSERVER_GARBAGE_COLLECTOR=null
 WEBSERVER_GROUPS=1
 WEBSERVER_GUNICORN_CMD_ARGS=--timeout=180

api/specs/web-server/_folders.py

@@ -15,7 +15,6 @@
     FolderGet,
     PutFolderBodyParams,
 )
-from models_library.api_schemas_webserver.wallets import WalletGet
 from models_library.generics import Envelope
 from models_library.rest_pagination import PageQueryParameters
 from pydantic import Json
@@ -39,7 +38,7 @@
 
 @router.post(
     "/folders",
-    response_model=Envelope[WalletGet],
+    response_model=Envelope[FolderGet],
     status_code=status.HTTP_201_CREATED,
 )
 async def create_folder(_body: CreateFolderBodyParams):
@@ -55,10 +54,10 @@ async def list_folders(
     order_by: Annotated[
         Json,
         Query(
-            description="Order by field (name|description) and direction (asc|desc). The default sorting order is ascending.",
+            description="Order by field (modified_at|name|description) and direction (asc|desc). The default sorting order is ascending.",
             example='{"field": "name", "direction": "desc"}',
         ),
-    ] = '{"field": "name", "direction": "desc"}',
+    ] = '{"field": "modified_at", "direction": "desc"}',
 ):
     ...
 
@@ -83,7 +82,7 @@ async def replace_folder(
 
 @router.delete(
     "/folders/{folder_id}",
-    response_model=Envelope[FolderGet],
+    status_code=status.HTTP_204_NO_CONTENT,
 )
 async def delete_folder(_path: Annotated[FoldersPathParams, Depends()]):
     ...

packages/models-library/src/models_library/api_schemas_webserver/folders.py

@@ -1,10 +1,12 @@
 from datetime import datetime
+from typing import NamedTuple
 
 from models_library.basic_types import IDStr
 from models_library.folders import FolderID
 from models_library.projects_access import AccessRights
 from models_library.users import GroupID
-from pydantic import Extra
+from models_library.utils.common_validators import null_or_none_str_to_none_validator
+from pydantic import Extra, PositiveInt, validator
 
 from ._base import InputSchema, OutputSchema
 
@@ -21,6 +23,11 @@ class FolderGet(OutputSchema):
     access_rights: dict[GroupID, AccessRights]
 
 
+class FolderGetPage(NamedTuple):
+    items: list[FolderGet]
+    total: PositiveInt
+
+
 class CreateFolderBodyParams(InputSchema):
     name: IDStr
     description: str
@@ -29,6 +36,10 @@ class CreateFolderBodyParams(InputSchema):
     class Config:
         extra = Extra.forbid
 
+    _null_or_none_str_to_none_validator = validator(
+        "parent_folder_id", allow_reuse=True, pre=True
+    )(null_or_none_str_to_none_validator)
+
 
 class PutFolderBodyParams(InputSchema):
     name: IDStr

packages/models-library/src/models_library/errors_classes.py

@@ -1,12 +1,21 @@
 from pydantic.errors import PydanticErrorMixin
 
 
+class _DefaultDict(dict):
+    def __missing__(self, key):
+        return f"'{key}=?'"
+
+
 class OsparcErrorMixin(PydanticErrorMixin):
     def __new__(cls, *args, **kwargs):
         if not hasattr(cls, "code"):
             cls.code = cls._get_full_class_name()
         return super().__new__(cls, *args, **kwargs)
 
+    def __str__(self) -> str:
+        # NOTE: safe. Does not raise KeyError
+        return self.msg_template.format_map(_DefaultDict(**self.__dict__))
+
     @classmethod
     def _get_full_class_name(cls) -> str:
         relevant_classes = [

packages/models-library/src/models_library/utils/common_validators.py

@@ -63,3 +63,9 @@ def ensure_unique_dict_values_validator(dict_data: dict) -> dict:
         msg = f"Dictionary values must be unique, provided: {dict_data}"
         raise ValueError(msg)
     return dict_data
+
+
+def null_or_none_str_to_none_validator(value: Any):
+    if isinstance(value, str) and value.lower() in ("null", "none"):
+        return None
+    return value

packages/models-library/tests/test_errors_classes.py

@@ -10,6 +10,7 @@
 
 import pytest
 from models_library.errors_classes import OsparcErrorMixin
+from pydantic.errors import PydanticErrorMixin
 
 
 def test_get_full_class_name():
@@ -134,3 +135,16 @@ class MyError(OsparcErrorMixin, ValueError):
 
     error = MyError(**ctx)
     assert str(error) == expected
+
+
+def test_missing_keys_in_msg_template_does_not_raise():
+    class MyErrorBefore(PydanticErrorMixin, ValueError):
+        msg_template = "{value} and {missing}"
+
+    with pytest.raises(KeyError, match="missing"):
+        str(MyErrorBefore(value=42))
+
+    class MyErrorAfter(OsparcErrorMixin, ValueError):
+        msg_template = "{value} and {missing}"
+
+    assert str(MyErrorAfter(value=42)) == "42 and 'missing=?'"

packages/models-library/tests/test_utils_common_validators.py

@@ -5,6 +5,7 @@
     create_enums_pre_validator,
     empty_str_to_none_pre_validator,
     none_to_empty_str_pre_validator,
+    null_or_none_str_to_none_validator,
 )
 from pydantic import BaseModel, ValidationError, validator
 
@@ -59,3 +60,30 @@ class Model(BaseModel):
 
     model = Model.parse_obj({"message": ""})
     assert model == Model.parse_obj({"message": None})
+
+
+def test_null_or_none_str_to_none_validator():
+    class Model(BaseModel):
+        message: str | None
+
+        _null_or_none_str_to_none_validator = validator(
+            "message", allow_reuse=True, pre=True
+        )(null_or_none_str_to_none_validator)
+
+    model = Model.parse_obj({"message": "none"})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": "null"})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": "NoNe"})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": "NuLl"})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": None})
+    assert model == Model.parse_obj({"message": None})
+
+    model = Model.parse_obj({"message": ""})
+    assert model == Model.parse_obj({"message": ""})

packages/postgres-database/src/simcore_postgres_database/utils_folders.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 from enum import Enum
 from functools import reduce
-from typing import Any, ClassVar, Final, TypeAlias
+from typing import Any, ClassVar, Final, TypeAlias, cast
 
 import sqlalchemy as sa
 from aiopg.sa.connection import SAConnection
@@ -20,12 +20,14 @@
     parse_obj_as,
 )
 from pydantic.errors import PydanticErrorMixin
+from simcore_postgres_database.utils_ordering import OrderByDict
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.sql.elements import ColumnElement
 from sqlalchemy.sql.selectable import ScalarSelect
 
 from .models.folders import folders, folders_access_rights, folders_to_projects
 from .models.groups import GroupType, groups
+from .utils_ordering import OrderDirection
 
 _ProductName: TypeAlias = str
 _ProjectID: TypeAlias = uuid.UUID
@@ -986,8 +988,11 @@ async def folder_list(
     *,
     offset: NonNegativeInt,
     limit: NonNegativeInt,
+    order_by: OrderByDict = OrderByDict(
+        field="modified", direction=OrderDirection.DESC
+    ),
     _required_permissions=_requires(_BasePermissions.LIST_FOLDERS),  # noqa: B008
-) -> list[FolderEntry]:
+) -> tuple[int, list[FolderEntry]]:
     """
     Raises:
         FolderNotFoundError
@@ -1015,7 +1020,7 @@ async def folder_list(
             access_via_gid = resolved_access_rights.gid
             access_via_folder_id = resolved_access_rights.folder_id
 
-        query = (
+        base_query = (
             sa.select(
                 folders,
                 folders_access_rights,
@@ -1047,14 +1052,30 @@ async def folder_list(
                 if folder_id is None
                 else True
             )
-            .offset(offset)
-            .limit(limit)
+            .where(folders.c.product_name == product_name)
         )
 
-        async for entry in connection.execute(query):
+        # Select total count from base_query
+        subquery = base_query.subquery()
+        count_query = sa.select(sa.func.count()).select_from(subquery)
+        count_result = await connection.execute(count_query)
+        total_count = await count_result.scalar()
+
+        # Ordering and pagination
+        if order_by["direction"] == OrderDirection.ASC:
+            list_query = base_query.order_by(
+                sa.asc(getattr(folders.c, order_by["field"]))
+            )
+        else:
+            list_query = base_query.order_by(
+                sa.desc(getattr(folders.c, order_by["field"]))
+            )
+        list_query = list_query.offset(offset).limit(limit)
+
+        async for entry in connection.execute(list_query):
             results.append(FolderEntry.from_orm(entry))  # noqa: PERF401s
 
-    return results
+    return cast(int, total_count), results
 
 
 async def folder_get(
@@ -1101,6 +1122,7 @@ async def folder_get(
                 if folder_id is None
                 else True
             )
+            .where(folders.c.product_name == product_name)
         )
 
         query_result: RowProxy | None = await (
@@ -1113,3 +1135,6 @@ async def folder_get(
         )
 
     return FolderEntry.from_orm(query_result)
+
+
+__all__ = ["OrderByDict"]

packages/postgres-database/src/simcore_postgres_database/utils_ordering.py

@@ -0,0 +1,19 @@
+from enum import Enum
+from typing import TypedDict
+
+
+class OrderDirection(str, Enum):
+    ASC = "asc"
+    DESC = "desc"
+
+
+class OrderByDict(TypedDict):
+    field: str
+    direction: OrderDirection
+
+
+# Example usage
+order_by_example: OrderByDict = {
+    "field": "example_field",
+    "direction": OrderDirection.ASC,
+}

packages/postgres-database/tests/test_utils_folders.py

@@ -1633,9 +1633,10 @@ async def _list_folder_as(
     limit: NonNegativeInt = ALL_IN_ONE_PAGE_LIMIT,
 ) -> list[FolderEntry]:
 
-    return await folder_list(
+    total_count, folders_db = await folder_list(
         connection, default_product_name, folder_id, gid, offset=offset, limit=limit
     )
+    return folders_db
 
 
 async def test_folder_list(

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py

@@ -54,7 +54,7 @@ async def _call(
             user_id=user_id,
             limit=limit,
             offset=offset,
-            timeout_s=4 * RPC_REQUEST_DEFAULT_TIMEOUT_S,
+            timeout_s=10 * RPC_REQUEST_DEFAULT_TIMEOUT_S,
         )
 
     result = await _call(

services/api-server/src/simcore_service_api_server/exceptions/service_errors_utils.py

@@ -58,13 +58,14 @@ def _get_http_exception_kwargs(
     service_name: str,
     service_error: httpx.HTTPStatusError,
     http_status_map: HttpStatusMap,
-    **detail_kwargs: Any,
+    **exception_ctx: Any,
 ):
     detail: str = ""
     headers: dict[str, str] = {}
 
     if exception_type := http_status_map.get(service_error.response.status_code):
-        raise exception_type(**detail_kwargs)
+        raise exception_type(**exception_ctx)
+
     if service_error.response.status_code in {
         status.HTTP_429_TOO_MANY_REQUESTS,
         status.HTTP_503_SERVICE_UNAVAILABLE,
@@ -92,9 +93,8 @@ def _get_http_exception_kwargs(
 def service_exception_handler(
     service_name: str,
     http_status_map: HttpStatusMap,
-    **endpoint_kwargs,
+    **context,
 ):
-    #
     status_code: int
     detail: str
     headers: dict[str, str] = {}
@@ -115,7 +115,7 @@ def service_exception_handler(
     except httpx.HTTPStatusError as exc:
 
         status_code, detail, headers = _get_http_exception_kwargs(
-            service_name, exc, http_status_map=http_status_map, **endpoint_kwargs
+            service_name, exc, http_status_map=http_status_map, **context
         )
         raise HTTPException(
             status_code=status_code, detail=detail, headers=headers
@@ -145,7 +145,7 @@ def _assert_correct_kwargs(func: Callable, status_map: HttpStatusMap):
         for name, param in signature(func).parameters.items()
         if param.kind == param.KEYWORD_ONLY
     }
-    for _, exc_type in status_map.items():
+    for exc_type in status_map.values():
         _exception_inputs = exc_type.named_fields()
         assert _exception_inputs.issubset(
             _required_kwargs

services/api-server/src/simcore_service_api_server/services/director_v2.py

@@ -192,9 +192,6 @@ async def get_computation_logs(
 
 
 def setup(app: FastAPI, settings: DirectorV2Settings) -> None:
-    if not settings:
-        settings = DirectorV2Settings()
-
     setup_client_instance(
         app,
         DirectorV2Api,

services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py

@@ -9,7 +9,7 @@
 import logging
 from collections.abc import Iterable
 from pprint import pprint
-from typing import AsyncIterable, Final
+from typing import Final
 
 import httpx
 import pytest
@@ -128,14 +128,14 @@ async def test_log_streaming(
 @pytest.fixture
 async def mock_job_not_found(
     mocked_directorv2_service_api_base: MockRouter,
-) -> AsyncIterable[MockRouter]:
+) -> MockRouter:
     def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response:
         return httpx.Response(status_code=status.HTTP_404_NOT_FOUND)
 
     mocked_directorv2_service_api_base.get(
         path__regex=r"/v2/computations/(?P<project_id>[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})"
     ).mock(side_effect=_get_computation)
-    yield mocked_directorv2_service_api_base
+    return mocked_directorv2_service_api_base
 
 
 async def test_logstreaming_job_not_found_exception(