♻️Storage: light refactoring to reduce noise (#7233)

Author: sanderegg

packages/pytest-simcore/src/pytest_simcore/file_extra.py

@@ -4,7 +4,7 @@
 
 import pytest
 from faker import Faker
-from pydantic import ByteSize
+from pydantic import ByteSize, NonNegativeInt
 from pytest_simcore.helpers.logging_tools import log_context
 
 
@@ -26,6 +26,37 @@ def _creator(size: ByteSize, name: str | None = None) -> Path:
     return _creator
 
 
+def _create_random_content(
+    faker: Faker,
+    *,
+    base_dir: Path,
+    file_min_size: ByteSize,
+    file_max_size: ByteSize,
+    remaining_size: ByteSize,
+    depth: NonNegativeInt | None,
+) -> ByteSize:
+    if remaining_size <= 0:
+        return remaining_size
+
+    file_size = ByteSize(
+        faker.pyint(
+            min_value=min(file_min_size, remaining_size),
+            max_value=min(remaining_size, file_max_size),
+        )
+    )
+    if depth is None:
+        depth = faker.pyint(0, 5)
+    file_path = base_dir / f"{faker.unique.file_path(depth=depth, absolute=False)}"
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+    assert not file_path.exists()
+    with file_path.open("wb") as fp:
+        fp.write(f"I am a {file_size.human_readable()} file".encode())
+        fp.truncate(file_size)
+    assert file_path.exists()
+
+    return ByteSize(remaining_size - file_size)
+
+
 @pytest.fixture
 def create_folder_of_size_with_multiple_files(
     tmp_path: Path, faker: Faker
@@ -34,33 +65,12 @@ def _create_folder_of_size_with_multiple_files(
         directory_size: ByteSize,
         file_min_size: ByteSize,
         file_max_size: ByteSize,
+        depth: NonNegativeInt | None = None,
     ) -> Path:
         # Helper function to create random files and directories
         assert file_min_size > 0
         assert file_min_size <= file_max_size
 
-        def create_random_content(base_dir: Path, remaining_size: ByteSize) -> ByteSize:
-            if remaining_size <= 0:
-                return remaining_size
-
-            # Decide to create a file or a subdirectory
-            # Create a file
-            file_size = ByteSize(
-                faker.pyint(
-                    min_value=min(file_min_size, remaining_size),
-                    max_value=min(remaining_size, file_max_size),
-                )
-            )  # max file size 1MB
-            file_path = base_dir / f"{faker.file_path(depth=4, absolute=False)}"
-            file_path.parent.mkdir(parents=True, exist_ok=True)
-            assert not file_path.exists()
-            with file_path.open("wb") as fp:
-                fp.write(f"I am a {file_size.human_readable()} file".encode())
-                fp.truncate(file_size)
-            assert file_path.exists()
-
-            return ByteSize(remaining_size - file_size)
-
         # Recursively create content in the temporary directory
         remaining_size = directory_size
         with log_context(
@@ -70,7 +80,14 @@ def create_random_content(base_dir: Path, remaining_size: ByteSize) -> ByteSize:
         ) as ctx:
             num_files_created = 0
             while remaining_size > 0:
-                remaining_size = create_random_content(tmp_path, remaining_size)
+                remaining_size = _create_random_content(
+                    faker,
+                    base_dir=tmp_path,
+                    file_min_size=file_min_size,
+                    file_max_size=file_max_size,
+                    remaining_size=remaining_size,
+                    depth=depth,
+                )
                 num_files_created += 1
             ctx.logger.info("created %s files", num_files_created)
         return tmp_path

packages/pytest-simcore/src/pytest_simcore/helpers/httpx_assert_checks.py

@@ -20,7 +20,7 @@ def assert_status(
     response_model: type[T] | None,
     *,
     expected_msg: str | None = None,
-    is_enveloped: bool = True,
+    expect_envelope: bool = True,
 ) -> tuple[T | None, Any]:
     """
     Asserts for enveloped responses
@@ -36,7 +36,7 @@ def assert_status(
     if expected_status_code == status.HTTP_204_NO_CONTENT:
         assert response.text == ""
         return None, None
-    if is_enveloped:
+    if expect_envelope:
         validated_response = TypeAdapter(Envelope[response_model]).validate_json(
             response.text
         )
@@ -49,6 +49,8 @@ def assert_status(
                 expected_status_code,
                 expected_msg,
             )
+        else:
+            assert data is not None
         return data, error
 
     if is_error(expected_status_code):

services/storage/tests/helpers/utils.py renamed to packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils.py

@@ -1,8 +1,10 @@
 import logging
 import os
-from typing import Any
+from pathlib import Path
+from typing import Any, TypedDict
 
 import sqlalchemy as sa
+from models_library.basic_types import SHA256Str
 from simcore_postgres_database.storage_models import projects
 from sqlalchemy.ext.asyncio import AsyncEngine
 
@@ -24,6 +26,10 @@ async def get_updated_project(
         result = await conn.execute(
             sa.select(projects).where(projects.c.uuid == project_id)
         )
-        row = result.fetchone()
-        assert row
+        row = result.one()
         return row._asdict()
+
+
+class FileIDDict(TypedDict):
+    path: Path
+    sha256_checksum: SHA256Str

services/storage/tests/helpers/utils_file_meta_data.py renamed to packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils_file_meta_data.py

@@ -7,24 +7,24 @@
 from contextlib import asynccontextmanager
 from pathlib import Path
 from random import choice, randint
-from typing import Any
+from typing import Any, cast
 
 import pytest
 import sqlalchemy as sa
 from faker import Faker
 from models_library.basic_types import SHA256Str
 from models_library.projects import ProjectID
-from models_library.projects_nodes_io import NodeID, SimcoreS3FileID
+from models_library.projects_nodes_io import NodeID, SimcoreS3FileID, StorageFileID
 from models_library.users import UserID
 from pydantic import ByteSize, TypeAdapter
-from pytest_simcore.helpers.faker_factories import random_project, random_user
 from servicelib.utils import limited_gather
 from simcore_postgres_database.models.project_to_groups import project_to_groups
 from simcore_postgres_database.storage_models import projects, users
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
 
-from ..helpers.utils import get_updated_project
+from .helpers.faker_factories import random_project, random_user
+from .helpers.storage_utils import FileIDDict, get_updated_project
 
 
 @asynccontextmanager
@@ -259,6 +259,39 @@ async def _creator(
     return _creator
 
 
+async def _upload_file_and_update_project(
+    project_id: ProjectID,
+    node_id: NodeID,
+    *,
+    file_name: str | None,
+    file_id: StorageFileID | None,
+    file_sizes: tuple[ByteSize, ...],
+    file_checksums: tuple[SHA256Str, ...],
+    node_to_files_mapping: dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
+    upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]],
+    create_simcore_file_id: Callable[
+        [ProjectID, NodeID, str, Path | None], SimcoreS3FileID
+    ],
+    faker: Faker,
+) -> None:
+    if file_name is None:
+        file_name = faker.file_name()
+        file_id = create_simcore_file_id(project_id, node_id, file_name, None)
+    checksum: SHA256Str = choice(file_checksums)  # noqa: S311
+    src_file, _ = await upload_file(
+        file_size=choice(file_sizes),  # noqa: S311
+        file_name=file_name,
+        file_id=file_id,
+        sha256_checksum=checksum,
+    )
+    assert file_name is not None
+    assert file_id is not None
+    node_to_files_mapping[node_id][file_id] = {
+        "path": src_file,
+        "sha256_checksum": checksum,
+    }
+
+
 @pytest.fixture
 async def random_project_with_files(
     sqlalchemy_async_engine: AsyncEngine,
@@ -271,11 +304,7 @@ async def random_project_with_files(
     faker: Faker,
 ) -> Callable[
     [int, tuple[ByteSize, ...], tuple[SHA256Str, ...]],
-    Awaitable[
-        tuple[
-            dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]]
-        ]
-    ],
+    Awaitable[tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]],
 ]:
     async def _creator(
         num_nodes: int = 12,
@@ -295,76 +324,67 @@ async def _creator(
                 "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3"
             ),
         ),
-    ) -> tuple[
-        dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]]
-    ]:
+    ) -> tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]:
         assert len(file_sizes) == len(file_checksums)
         project = await create_project(name="random-project")
-        src_projects_list: dict[
-            NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]
-        ] = {}
+        node_to_files_mapping: dict[NodeID, dict[SimcoreS3FileID, FileIDDict]] = {}
         upload_tasks: deque[Awaitable] = deque()
         for _node_index in range(num_nodes):
-            # NOTE: we put some more outputs in there to simulate a real case better
-            new_node_id = NodeID(f"{faker.uuid4()}")
+            # Create a node with outputs (files and others)
+            project_id = ProjectID(project["uuid"])
+            node_id = cast(NodeID, faker.uuid4(cast_to=None))
+            output3_file_name = faker.file_name()
             output3_file_id = create_simcore_file_id(
-                ProjectID(project["uuid"]),
-                new_node_id,
-                faker.file_name(),
-                Path("outputs/output3"),
+                project_id, node_id, output3_file_name, Path("outputs/output_3")
             )
-            src_node_id = await create_project_node(
+            created_node_id = await create_project_node(
                 ProjectID(project["uuid"]),
-                new_node_id,
+                node_id,
                 outputs={
                     "output_1": faker.pyint(),
                     "output_2": faker.pystr(),
                     "output_3": f"{output3_file_id}",
                 },
             )
-            assert src_node_id == new_node_id
-
-            # upload the output 3 and some random other files at the root of each node
-            src_projects_list[src_node_id] = {}
-            checksum: SHA256Str = choice(file_checksums)  # noqa: S311
-            src_file, _ = await upload_file(
-                file_size=choice(file_sizes),  # noqa: S311
-                file_name=Path(output3_file_id).name,
-                file_id=output3_file_id,
-                sha256_checksum=checksum,
-            )
-            src_projects_list[src_node_id][output3_file_id] = {
-                "path": src_file,
-                "sha256_checksum": checksum,
-            }
-
-            async def _upload_file_and_update_project(project, src_node_id):
-                src_file_name = faker.file_name()
-                src_file_uuid = create_simcore_file_id(
-                    ProjectID(project["uuid"]), src_node_id, src_file_name, None
-                )
-                checksum: SHA256Str = choice(file_checksums)  # noqa: S311
-                src_file, _ = await upload_file(
-                    file_size=choice(file_sizes),  # noqa: S311
-                    file_name=src_file_name,
-                    file_id=src_file_uuid,
-                    sha256_checksum=checksum,
+            assert created_node_id == node_id
+
+            node_to_files_mapping[created_node_id] = {}
+            upload_tasks.append(
+                _upload_file_and_update_project(
+                    project_id,
+                    node_id,
+                    file_name=output3_file_name,
+                    file_id=output3_file_id,
+                    file_sizes=file_sizes,
+                    file_checksums=file_checksums,
+                    upload_file=upload_file,
+                    create_simcore_file_id=create_simcore_file_id,
+                    faker=faker,
+                    node_to_files_mapping=node_to_files_mapping,
                 )
-                src_projects_list[src_node_id][src_file_uuid] = {
-                    "path": src_file,
-                    "sha256_checksum": checksum,
-                }
+            )
 
-            # add a few random files in the node storage
+            # add a few random files in the node workspace
             upload_tasks.extend(
                 [
-                    _upload_file_and_update_project(project, src_node_id)
+                    _upload_file_and_update_project(
+                        project_id,
+                        node_id,
+                        file_name=None,
+                        file_id=None,
+                        file_sizes=file_sizes,
+                        file_checksums=file_checksums,
+                        upload_file=upload_file,
+                        create_simcore_file_id=create_simcore_file_id,
+                        faker=faker,
+                        node_to_files_mapping=node_to_files_mapping,
+                    )
                     for _ in range(randint(0, 3))  # noqa: S311
                 ]
             )
         await limited_gather(*upload_tasks, limit=10)
 
         project = await get_updated_project(sqlalchemy_async_engine, project["uuid"])
-        return project, src_projects_list
+        return project, node_to_files_mapping
 
     return _creator

services/storage/tests/helpers/utils_project.py renamed to packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils_project.py

@@ -94,7 +94,7 @@ def set_app_default_http_error_handlers(app: FastAPI) -> None:
     app.add_exception_handler(
         ValidationError,
         make_http_error_handler_for_exception(
-            status.HTTP_422_UNPROCESSABLE_ENTITY,
+            status.HTTP_500_INTERNAL_SERVER_ERROR,
             ValidationError,
             envelope_error=True,
         ),

services/storage/tests/fixtures/data_models.py renamed to packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py

@@ -21,6 +21,7 @@ httpx
 opentelemetry-instrumentation-botocore
 packaging
 fastapi[all]
+fastapi-pagination
 orjson
 pydantic[dotenv]
 tenacity