🎨Storage: Increased maximum number of returned paths to 1000 (ITISFoundation#7305)

Author: sanderegg

api/specs/web-server/_storage.py

@@ -8,7 +8,6 @@
 from uuid import UUID
 
 from fastapi import APIRouter, Depends, Query, status
-from fastapi_pagination.cursor import CursorPage
 from models_library.api_schemas_storage.storage_schemas import (
     FileLocation,
     FileMetaDataGet,
@@ -32,6 +31,7 @@
 from models_library.projects_nodes_io import LocationID
 from models_library.users import UserID
 from pydantic import AnyUrl, ByteSize
+from servicelib.fastapi.rest_pagination import CustomizedPathsCursorPage
 from simcore_service_webserver._meta import API_VTAG
 from simcore_service_webserver.storage.schemas import DatasetMetaData, FileMetaData
 
@@ -59,7 +59,7 @@ async def list_storage_locations():
 
 @router.get(
     "/storage/locations/{location_id}/paths",
-    response_model=CursorPage[PathMetaDataGet],
+    response_model=CustomizedPathsCursorPage[PathMetaDataGet],
 )
 async def list_storage_paths(
     _path: Annotated[StorageLocationPathParams, Depends()],

packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py

@@ -9,7 +9,7 @@
 from datetime import datetime
 from enum import Enum
 from pathlib import Path
-from typing import Annotated, Any, Literal, Self, TypeAlias
+from typing import Annotated, Any, Final, Literal, Self, TypeAlias
 from uuid import UUID
 
 from pydantic import (
@@ -404,6 +404,10 @@ class SoftCopyBody(BaseModel):
     link_id: SimcoreS3FileID
 
 
+DEFAULT_NUMBER_OF_PATHS_PER_PAGE: Final[int] = 50
+MAX_NUMBER_OF_PATHS_PER_PAGE: Final[int] = 1000
+
+
 class PathMetaDataGet(BaseModel):
     path: Annotated[Path, Field(description="the path to the current path")]
     display_path: Annotated[

packages/models-library/src/models_library/api_schemas_webserver/storage.py

@@ -1,8 +1,12 @@
 from datetime import datetime
 from pathlib import Path
-from typing import Any
+from typing import Annotated, Any
 
-from pydantic import BaseModel
+from models_library.api_schemas_storage.storage_schemas import (
+    DEFAULT_NUMBER_OF_PATHS_PER_PAGE,
+    MAX_NUMBER_OF_PATHS_PER_PAGE,
+)
+from pydantic import BaseModel, Field
 
 from ..api_schemas_rpc_async_jobs.async_jobs import (
     AsyncJobGet,
@@ -13,7 +17,9 @@
 from ..api_schemas_storage.data_export_async_jobs import DataExportTaskStartInput
 from ..progress_bar import ProgressReport
 from ..projects_nodes_io import LocationID, StorageFileID
-from ..rest_pagination import CursorQueryParameters
+from ..rest_pagination import (
+    CursorQueryParameters,
+)
 from ._base import InputSchema, OutputSchema
 
 
@@ -24,6 +30,15 @@ class StorageLocationPathParams(BaseModel):
 class ListPathsQueryParams(InputSchema, CursorQueryParameters):
     file_filter: Path | None = None
 
+    size: Annotated[
+        int,
+        Field(
+            description="maximum number of items to return (pagination)",
+            ge=1,
+            lt=MAX_NUMBER_OF_PATHS_PER_PAGE,
+        ),
+    ] = DEFAULT_NUMBER_OF_PATHS_PER_PAGE
+
 
 class DataExportPost(InputSchema):
     paths: list[StorageFileID]

packages/service-library/src/servicelib/fastapi/rest_pagination.py

@@ -0,0 +1,28 @@
+from typing import TypeAlias, TypeVar
+
+from fastapi import Query
+from fastapi_pagination.cursor import CursorPage  # type: ignore[import-not-found]
+from fastapi_pagination.customization import (  # type: ignore[import-not-found]
+    CustomizedPage,
+    UseParamsFields,
+)
+from models_library.api_schemas_storage.storage_schemas import (
+    DEFAULT_NUMBER_OF_PATHS_PER_PAGE,
+    MAX_NUMBER_OF_PATHS_PER_PAGE,
+)
+
+_T = TypeVar("_T")
+
+CustomizedPathsCursorPage = CustomizedPage[
+    CursorPage[_T],
+    # Customizes the maximum value to fit frontend needs
+    UseParamsFields(
+        size=Query(
+            DEFAULT_NUMBER_OF_PATHS_PER_PAGE,
+            ge=1,
+            le=MAX_NUMBER_OF_PATHS_PER_PAGE,
+            description="Page size",
+        )
+    ),
+]
+CustomizedPathsCursorPageParams: TypeAlias = CustomizedPathsCursorPage.__params_type__  # type: ignore

services/api-server/openapi.json

@@ -805,9 +805,9 @@
             "required": false,
             "schema": {
               "type": "integer",
-              "maximum": 100,
+              "maximum": 50,
               "minimum": 1,
-              "default": 50,
+              "default": 20,
               "title": "Limit"
             }
           },
@@ -3352,9 +3352,9 @@
             "required": false,
             "schema": {
               "type": "integer",
-              "maximum": 100,
+              "maximum": 50,
               "minimum": 1,
-              "default": 50,
+              "default": 20,
               "title": "Limit"
             }
           },
@@ -4164,9 +4164,9 @@
             "required": false,
             "schema": {
               "type": "integer",
-              "maximum": 100,
+              "maximum": 50,
               "minimum": 1,
-              "default": 50,
+              "default": 20,
               "title": "Limit"
             }
           },
@@ -5351,9 +5351,9 @@
             "required": false,
             "schema": {
               "type": "integer",
-              "maximum": 100,
+              "maximum": 50,
               "minimum": 1,
-              "default": 50,
+              "default": 20,
               "title": "Limit"
             }
           },
@@ -5648,9 +5648,9 @@
             "required": false,
             "schema": {
               "type": "integer",
-              "maximum": 100,
+              "maximum": 50,
               "minimum": 1,
-              "default": 50,
+              "default": 20,
               "title": "Limit"
             }
           },

services/api-server/src/simcore_service_api_server/models/pagination.py

@@ -1,4 +1,4 @@
-""" Overrides models in fastapi_pagination
+"""Overrides models in fastapi_pagination
 
 Usage:
     from fastapi_pagination.api import create_page
@@ -11,7 +11,6 @@
 
 from fastapi import Query
 from fastapi_pagination.customization import CustomizedPage, UseName, UseParamsFields
-from fastapi_pagination.limit_offset import LimitOffsetParams as _LimitOffsetParams
 from fastapi_pagination.links import LimitOffsetPage as _LimitOffsetPage
 from models_library.rest_pagination import (
     DEFAULT_NUMBER_OF_ITEMS_PER_PAGE,
@@ -43,7 +42,7 @@
     UseName(name="Page"),
 ]
 
-PaginationParams: TypeAlias = _LimitOffsetParams
+PaginationParams: TypeAlias = Page.__params_type__  # type: ignore
 
 
 class OnePage(BaseModel, Generic[T]):

services/storage/openapi.json

@@ -1047,8 +1047,8 @@
             "required": false,
             "schema": {
               "type": "integer",
-              "maximum": 100,
-              "minimum": 0,
+              "maximum": 1000,
+              "minimum": 1,
               "default": 50,
               "title": "Size"
             }
@@ -1060,7 +1060,7 @@
             "content": {
               "application/json": {
                 "schema": {
-                  "$ref": "#/components/schemas/CursorPage_PathMetaDataGet_"
+                  "$ref": "#/components/schemas/CursorPage___T_Customized_PathMetaDataGet_"
                 }
               }
             }
@@ -1419,7 +1419,7 @@
         ],
         "title": "AppStatusCheck"
       },
-      "CursorPage_PathMetaDataGet_": {
+      "CursorPage___T_Customized_PathMetaDataGet_": {
         "properties": {
           "items": {
             "items": {
@@ -1493,7 +1493,7 @@
         "required": [
           "items"
         ],
-        "title": "CursorPage[PathMetaDataGet]"
+        "title": "CursorPage[~_T]Customized[PathMetaDataGet]"
       },
       "DatasetMetaDataGet": {
         "properties": {

services/storage/src/simcore_service_storage/api/rest/_paths.py

@@ -4,9 +4,12 @@
 
 from fastapi import APIRouter, Depends
 from fastapi_pagination import create_page
-from fastapi_pagination.cursor import CursorPage, CursorParams
 from models_library.api_schemas_storage.storage_schemas import PathMetaDataGet
 from models_library.users import UserID
+from servicelib.fastapi.rest_pagination import (
+    CustomizedPathsCursorPage,
+    CustomizedPathsCursorPageParams,
+)
 
 from ...dsm_factory import BaseDataManager
 from .dependencies.dsm_prodiver import get_data_manager
@@ -22,10 +25,10 @@
 
 @router.get(
     "/locations/{location_id}/paths",
-    response_model=CursorPage[PathMetaDataGet],
+    response_model=CustomizedPathsCursorPage[PathMetaDataGet],
 )
 async def list_paths(
-    page_params: Annotated[CursorParams, Depends()],
+    page_params: Annotated[CustomizedPathsCursorPageParams, Depends()],
     dsm: Annotated[BaseDataManager, Depends(get_data_manager)],
     user_id: UserID,
     file_filter: Path | None = None,

services/storage/tests/unit/test_handlers_paths.py

@@ -19,6 +19,7 @@
 from fastapi import FastAPI, status
 from fastapi_pagination.cursor import CursorPage
 from models_library.api_schemas_storage.storage_schemas import PathMetaDataGet
+from models_library.api_schemas_webserver.storage import MAX_NUMBER_OF_PATHS_PER_PAGE
 from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID
 from models_library.users import UserID
 from pydantic import ByteSize, TypeAdapter
@@ -203,6 +204,48 @@ async def test_list_paths_pagination(
         )
 
 
+@pytest.mark.parametrize(
+    "project_params",
+    [
+        ProjectWithFilesParams(
+            num_nodes=1,
+            allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("0b"),),
+            workspace_files_count=MAX_NUMBER_OF_PATHS_PER_PAGE,
+        )
+    ],
+    ids=str,
+)
+async def test_list_paths_pagination_large_page(
+    initialized_app: FastAPI,
+    client: httpx.AsyncClient,
+    location_id: LocationID,
+    user_id: UserID,
+    with_random_project_with_files: tuple[
+        dict[str, Any],
+        dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
+    ],
+):
+    project, list_of_files = with_random_project_with_files
+    selected_node_id = NodeID(random.choice(list(project["workbench"])))  # noqa: S311
+    selected_node_s3_keys = [
+        Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id]
+    ]
+    workspace_file_filter = Path(project["uuid"]) / f"{selected_node_id}" / "workspace"
+    expected_paths = _filter_and_group_paths_one_level_deeper(
+        selected_node_s3_keys, workspace_file_filter
+    )
+    await _assert_list_paths(
+        initialized_app,
+        client,
+        location_id,
+        user_id,
+        file_filter=workspace_file_filter,
+        expected_paths=expected_paths,
+        check_total=False,
+        limit=MAX_NUMBER_OF_PATHS_PER_PAGE,
+    )
+
+
 @pytest.mark.parametrize(
     "project_params, num_projects",
     [

services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml

@@ -6009,9 +6009,9 @@ paths:
           type: integer
           minimum: 1
           exclusiveMaximum: true
-          default: 20
+          default: 50
           title: Size
-          maximum: 50
+          maximum: 1000
       - name: cursor
         in: query
         required: false
@@ -6035,7 +6035,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/CursorPage_PathMetaDataGet_'
+                $ref: '#/components/schemas/CursorPage___T_Customized_PathMetaDataGet_'
   /v0/storage/locations/{location_id}/datasets:
     get:
       tags:
@@ -8404,7 +8404,7 @@ components:
       - usdPerCredit
       - minPaymentAmountUsd
       title: CreditPriceGet
-    CursorPage_PathMetaDataGet_:
+    CursorPage___T_Customized_PathMetaDataGet_:
       properties:
         items:
           items:
@@ -8444,7 +8444,7 @@ components:
       type: object
       required:
       - items
-      title: CursorPage[PathMetaDataGet]
+      title: CursorPage[~_T]Customized[PathMetaDataGet]
     DatCoreFileLink:
       properties:
         store: