Skip to content

Commit 821b822

Browse files
sandereggsanderegg
committed
ongoing
1 parent bdfd793 commit 821b822

File tree

3 files changed

+228
-12
lines changed

3 files changed

+228
-12
lines changed

‎services/storage/src/simcore_service_storage/api/rpc/_paths.py

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@
77
AsyncJobGet,
88
AsyncJobId,
99
)
10-
from models_library.api_schemas_storage.data_export_async_jobs import (
11-
AccessRightError,
12-
DataExportError,
13-
InvalidFileIdentifierError,
14-
)
1510
from models_library.projects_nodes_io import LocationID
1611
from models_library.users import UserID
1712
from servicelib.rabbitmq import RPCRouter
@@ -21,13 +16,7 @@
2116
router = RPCRouter()
2217

2318

24-
@router.expose(
25-
reraise_if_error_type=(
26-
InvalidFileIdentifierError,
27-
AccessRightError,
28-
DataExportError,
29-
)
30-
)
19+
@router.expose(reraise_if_error_type=())
3120
async def compute_path_size(
3221
app: FastAPI,
3322
user_id: UserID,

‎services/storage/tests/conftest.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
"pytest_simcore.openapi_specs",
9393
"pytest_simcore.postgres_service",
9494
"pytest_simcore.pytest_global_environs",
95+
"pytest_simcore.rabbit_service",
9596
"pytest_simcore.repository_paths",
9697
"pytest_simcore.simcore_storage_data_models",
9798
"pytest_simcore.simcore_storage_datcore_adapter",

‎services/storage/tests/unit/test_rpc_handlers_paths.py

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
# pylint:disable=no-name-in-module
2+
# pylint:disable=protected-access
3+
# pylint:disable=redefined-outer-name
4+
# pylint:disable=too-many-arguments
5+
# pylint:disable=too-many-positional-arguments
6+
# pylint:disable=unused-argument
7+
# pylint:disable=unused-variable
8+
9+
10+
import random
11+
from collections.abc import Awaitable, Callable
12+
from pathlib import Path
13+
from typing import Any, TypeAlias
14+
15+
import httpx
16+
import pytest
17+
from faker import Faker
18+
from fastapi import FastAPI
19+
from models_library.api_schemas_storage.storage_schemas import (
20+
PathTotalSizeCreate,
21+
)
22+
from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID
23+
from models_library.users import UserID
24+
from pydantic import ByteSize, TypeAdapter
25+
from pytest_simcore.helpers.storage_utils import FileIDDict, ProjectWithFilesParams
26+
from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient
27+
from servicelib.rabbitmq.rpc_interfaces.storage.paths import compute_path_size
28+
from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager
29+
30+
pytest_simcore_core_services_selection = ["postgres", "rabbit"]
31+
pytest_simcore_ops_services_selection = ["adminer"]
32+
33+
_IsFile: TypeAlias = bool
34+
35+
36+
@pytest.fixture
37+
async def storage_rabbitmq_rpc_client(
38+
rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]],
39+
) -> RabbitMQRPCClient:
40+
rpc_client = await rabbitmq_rpc_client("pytest_storage_rpc_client")
41+
assert rpc_client
42+
return rpc_client
43+
44+
45+
def _filter_and_group_paths_one_level_deeper(
46+
paths: list[Path], prefix: Path
47+
) -> list[tuple[Path, _IsFile]]:
48+
relative_paths = (path for path in paths if path.is_relative_to(prefix))
49+
return sorted(
50+
{
51+
(
52+
(path, len(path.relative_to(prefix).parts) == 1)
53+
if len(path.relative_to(prefix).parts) == 1
54+
else (prefix / path.relative_to(prefix).parts[0], False)
55+
)
56+
for path in relative_paths
57+
},
58+
key=lambda x: x[0],
59+
)
60+
61+
62+
async def _assert_compute_path_size(
63+
rpc_client: RabbitMQRPCClient,
64+
location_id: LocationID,
65+
user_id: UserID,
66+
*,
67+
path: Path,
68+
expected_total_size: int,
69+
) -> ByteSize:
70+
received = await compute_path_size(
71+
rpc_client, user_id=user_id, product_name="", location_id=location_id, path=path
72+
)
73+
74+
assert isinstance(received, PathTotalSizeCreate)
75+
76+
assert received
77+
assert received.path == path
78+
assert received.size == expected_total_size
79+
return received.size
80+
81+
82+
@pytest.mark.parametrize(
83+
"location_id",
84+
[SimcoreS3DataManager.get_location_id()],
85+
ids=[SimcoreS3DataManager.get_location_name()],
86+
indirect=True,
87+
)
88+
@pytest.mark.parametrize(
89+
"project_params",
90+
[
91+
ProjectWithFilesParams(
92+
num_nodes=5,
93+
allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),),
94+
workspace_files_count=10,
95+
)
96+
],
97+
ids=str,
98+
)
99+
async def test_path_compute_size(
100+
initialized_app: FastAPI,
101+
client: httpx.AsyncClient,
102+
storage_rabbitmq_rpc_client: RabbitMQRPCClient,
103+
location_id: LocationID,
104+
user_id: UserID,
105+
with_random_project_with_files: tuple[
106+
dict[str, Any],
107+
dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
108+
],
109+
project_params: ProjectWithFilesParams,
110+
):
111+
assert (
112+
len(project_params.allowed_file_sizes) == 1
113+
), "test preconditions are not filled! allowed file sizes should have only 1 option for this test"
114+
project, list_of_files = with_random_project_with_files
115+
116+
total_num_files = sum(
117+
len(files_in_node) for files_in_node in list_of_files.values()
118+
)
119+
120+
# get size of a full project
121+
expected_total_size = project_params.allowed_file_sizes[0] * total_num_files
122+
path = Path(project["uuid"])
123+
await _assert_compute_path_size(
124+
storage_rabbitmq_rpc_client,
125+
location_id,
126+
user_id,
127+
path=path,
128+
expected_total_size=expected_total_size,
129+
)
130+
131+
# get size of one of the nodes
132+
selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311
133+
path = Path(project["uuid"]) / f"{selected_node_id}"
134+
selected_node_s3_keys = [
135+
Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id]
136+
]
137+
expected_total_size = project_params.allowed_file_sizes[0] * len(
138+
selected_node_s3_keys
139+
)
140+
await _assert_compute_path_size(
141+
storage_rabbitmq_rpc_client,
142+
location_id,
143+
user_id,
144+
path=path,
145+
expected_total_size=expected_total_size,
146+
)
147+
148+
# get size of the outputs of one of the nodes
149+
path = Path(project["uuid"]) / f"{selected_node_id}" / "outputs"
150+
selected_node_s3_keys = [
151+
Path(s3_object_id)
152+
for s3_object_id in list_of_files[selected_node_id]
153+
if s3_object_id.startswith(f"{path}")
154+
]
155+
expected_total_size = project_params.allowed_file_sizes[0] * len(
156+
selected_node_s3_keys
157+
)
158+
await _assert_compute_path_size(
159+
storage_rabbitmq_rpc_client,
160+
location_id,
161+
user_id,
162+
path=path,
163+
expected_total_size=expected_total_size,
164+
)
165+
166+
# get size of workspace in one of the nodes (this is semi-cached in the DB)
167+
path = Path(project["uuid"]) / f"{selected_node_id}" / "workspace"
168+
selected_node_s3_keys = [
169+
Path(s3_object_id)
170+
for s3_object_id in list_of_files[selected_node_id]
171+
if s3_object_id.startswith(f"{path}")
172+
]
173+
expected_total_size = project_params.allowed_file_sizes[0] * len(
174+
selected_node_s3_keys
175+
)
176+
workspace_total_size = await _assert_compute_path_size(
177+
storage_rabbitmq_rpc_client,
178+
location_id,
179+
user_id,
180+
path=path,
181+
expected_total_size=expected_total_size,
182+
)
183+
184+
# get size of folders inside the workspace
185+
folders_inside_workspace = [
186+
p[0]
187+
for p in _filter_and_group_paths_one_level_deeper(selected_node_s3_keys, path)
188+
if p[1] is False
189+
]
190+
accumulated_subfolder_size = 0
191+
for workspace_subfolder in folders_inside_workspace:
192+
selected_node_s3_keys = [
193+
Path(s3_object_id)
194+
for s3_object_id in list_of_files[selected_node_id]
195+
if s3_object_id.startswith(f"{workspace_subfolder}")
196+
]
197+
expected_total_size = project_params.allowed_file_sizes[0] * len(
198+
selected_node_s3_keys
199+
)
200+
accumulated_subfolder_size += await _assert_compute_path_size(
201+
storage_rabbitmq_rpc_client,
202+
location_id,
203+
user_id,
204+
path=workspace_subfolder,
205+
expected_total_size=expected_total_size,
206+
)
207+
208+
assert workspace_total_size == accumulated_subfolder_size
209+
210+
211+
async def test_path_compute_size_inexistent_path(
212+
initialized_app: FastAPI,
213+
client: httpx.AsyncClient,
214+
storage_rabbitmq_rpc_client: RabbitMQRPCClient,
215+
location_id: LocationID,
216+
user_id: UserID,
217+
faker: Faker,
218+
fake_datcore_tokens: tuple[str, str],
219+
):
220+
await _assert_compute_path_size(
221+
storage_rabbitmq_rpc_client,
222+
location_id,
223+
user_id,
224+
path=Path(faker.file_path(absolute=False)),
225+
expected_total_size=0,
226+
)

0 commit comments

Comments
 (0)