diff --git a/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py b/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py index 560c4063250..8901e66a7ad 100644 --- a/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py +++ b/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py @@ -36,7 +36,6 @@ class AsyncJobResult(BaseModel): class AsyncJobGet(BaseModel): job_id: AsyncJobId - job_name: str class AsyncJobAbort(BaseModel): @@ -44,8 +43,8 @@ class AsyncJobAbort(BaseModel): job_id: AsyncJobId -class AsyncJobAccessData(BaseModel): +class AsyncJobNameData(BaseModel): """Data for controlling access to an async job""" - user_id: UserID | None + user_id: UserID product_name: str diff --git a/packages/models-library/src/models_library/api_schemas_storage/data_export_async_jobs.py b/packages/models-library/src/models_library/api_schemas_storage/data_export_async_jobs.py index 3645c918e99..a3db991452f 100644 --- a/packages/models-library/src/models_library/api_schemas_storage/data_export_async_jobs.py +++ b/packages/models-library/src/models_library/api_schemas_storage/data_export_async_jobs.py @@ -1,33 +1,33 @@ # pylint: disable=R6301 -from pathlib import Path from common_library.errors_classes import OsparcErrorMixin -from models_library.projects_nodes_io import LocationID -from models_library.users import UserID +from models_library.projects_nodes_io import LocationID, StorageFileID from pydantic import BaseModel, Field class DataExportTaskStartInput(BaseModel): - user_id: UserID - product_name: str location_id: LocationID - paths: list[Path] = Field(..., min_length=1) + file_and_folder_ids: list[StorageFileID] = Field(..., min_length=1) ### Exceptions -class StorageRpcError(OsparcErrorMixin, RuntimeError): +class StorageRpcBaseError(OsparcErrorMixin, RuntimeError): pass -class InvalidFileIdentifierError(StorageRpcError): +class InvalidLocationIdError(StorageRpcBaseError): + msg_template: str = "Invalid location_id {location_id}" + + +class InvalidFileIdentifierError(StorageRpcBaseError): msg_template: str = "Could not find the file {file_id}" -class AccessRightError(StorageRpcError): - msg_template: str = "User {user_id} does not have access to file {file_id}" +class AccessRightError(StorageRpcBaseError): + msg_template: str = "User {user_id} does not have access to file {file_id} with location {location_id}" -class DataExportError(StorageRpcError): +class DataExportError(StorageRpcBaseError): msg_template: str = "Could not complete data export job with id {job_id}" diff --git a/packages/models-library/src/models_library/api_schemas_webserver/storage.py b/packages/models-library/src/models_library/api_schemas_webserver/storage.py index 0721d153db5..9c793ad2ee7 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/storage.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/storage.py @@ -12,9 +12,8 @@ ) from ..api_schemas_storage.data_export_async_jobs import DataExportTaskStartInput from ..progress_bar import ProgressReport -from ..projects_nodes_io import LocationID +from ..projects_nodes_io import LocationID, StorageFileID from ..rest_pagination import CursorQueryParameters -from ..users import UserID from ._base import InputSchema, OutputSchema @@ -27,15 +26,11 @@ class ListPathsQueryParams(InputSchema, CursorQueryParameters): class DataExportPost(InputSchema): - paths: list[Path] + paths: list[StorageFileID] - def to_rpc_schema( - self, user_id: UserID, product_name: str, location_id: LocationID - ) -> DataExportTaskStartInput: + def to_rpc_schema(self, location_id: LocationID) -> DataExportTaskStartInput: return DataExportTaskStartInput( - paths=self.paths, - user_id=user_id, - product_name=product_name, + file_and_folder_ids=self.paths, location_id=location_id, ) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/async_jobs/async_jobs.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/async_jobs/async_jobs.py index 8daa9f674c4..a7246350773 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/async_jobs/async_jobs.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/async_jobs/async_jobs.py @@ -2,9 +2,9 @@ from models_library.api_schemas_rpc_async_jobs.async_jobs import ( AsyncJobAbort, - AsyncJobAccessData, AsyncJobGet, AsyncJobId, + AsyncJobNameData, AsyncJobResult, AsyncJobStatus, ) @@ -23,13 +23,13 @@ async def abort( *, rpc_namespace: RPCNamespace, job_id: AsyncJobId, - access_data: AsyncJobAccessData | None + job_id_data: AsyncJobNameData ) -> AsyncJobAbort: result = await rabbitmq_rpc_client.request( rpc_namespace, _RPC_METHOD_NAME_ADAPTER.validate_python("abort"), job_id=job_id, - access_data=access_data, + job_id_data=job_id_data, timeout_s=_DEFAULT_TIMEOUT_S, ) assert isinstance(result, AsyncJobAbort) @@ -41,13 +41,13 @@ async def get_status( *, rpc_namespace: RPCNamespace, job_id: AsyncJobId, - access_data: AsyncJobAccessData | None + job_id_data: AsyncJobNameData ) -> AsyncJobStatus: result = await rabbitmq_rpc_client.request( rpc_namespace, _RPC_METHOD_NAME_ADAPTER.validate_python("get_status"), job_id=job_id, - access_data=access_data, + job_id_data=job_id_data, timeout_s=_DEFAULT_TIMEOUT_S, ) assert isinstance(result, AsyncJobStatus) @@ -59,13 +59,13 @@ async def get_result( *, rpc_namespace: RPCNamespace, job_id: AsyncJobId, - access_data: AsyncJobAccessData | None + job_id_data: AsyncJobNameData ) -> AsyncJobResult: result = await rabbitmq_rpc_client.request( rpc_namespace, _RPC_METHOD_NAME_ADAPTER.validate_python("get_result"), job_id=job_id, - access_data=access_data, + job_id_data=job_id_data, timeout_s=_DEFAULT_TIMEOUT_S, ) assert isinstance(result, AsyncJobResult) @@ -73,12 +73,17 @@ async def get_result( async def list_jobs( - rabbitmq_rpc_client: RabbitMQRPCClient, *, rpc_namespace: RPCNamespace, filter_: str + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + rpc_namespace: RPCNamespace, + filter_: str, + job_id_data: AsyncJobNameData ) -> list[AsyncJobGet]: result: list[AsyncJobGet] = await rabbitmq_rpc_client.request( rpc_namespace, _RPC_METHOD_NAME_ADAPTER.validate_python("list_jobs"), filter_=filter_, + job_id_data=job_id_data, timeout_s=_DEFAULT_TIMEOUT_S, ) return result @@ -88,12 +93,14 @@ async def submit_job( rabbitmq_rpc_client: RabbitMQRPCClient, *, rpc_namespace: RPCNamespace, - job_name: str, + method_name: str, + job_id_data: AsyncJobNameData, **kwargs ) -> AsyncJobGet: result = await rabbitmq_rpc_client.request( rpc_namespace, - _RPC_METHOD_NAME_ADAPTER.validate_python(job_name), + _RPC_METHOD_NAME_ADAPTER.validate_python(method_name), + job_id_data=job_id_data, **kwargs, timeout_s=_DEFAULT_TIMEOUT_S, ) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py index c9e9699942a..c4aed1aa65d 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -5,9 +5,9 @@ from fastapi import FastAPI from models_library.api_schemas_rpc_async_jobs.async_jobs import ( AsyncJobAbort, - AsyncJobAccessData, AsyncJobGet, AsyncJobId, + AsyncJobNameData, AsyncJobResult, AsyncJobStatus, ) @@ -23,17 +23,19 @@ @router.expose() async def abort( - app: FastAPI, job_id: AsyncJobId, access_data: AsyncJobAccessData | None + app: FastAPI, job_id: AsyncJobId, job_id_data: AsyncJobNameData ) -> AsyncJobAbort: assert app # nosec + assert job_id_data # nosec return AsyncJobAbort(result=True, job_id=job_id) @router.expose(reraise_if_error_type=(StatusError,)) async def get_status( - app: FastAPI, job_id: AsyncJobId, access_data: AsyncJobAccessData | None + app: FastAPI, job_id: AsyncJobId, job_id_data: AsyncJobNameData ) -> AsyncJobStatus: assert app # nosec + assert job_id_data # nosec progress_report = ProgressReport(actual_value=0.5, total=1.0, attempt=1) return AsyncJobStatus( job_id=job_id, @@ -46,14 +48,17 @@ async def get_status( @router.expose(reraise_if_error_type=(ResultError,)) async def get_result( - app: FastAPI, job_id: AsyncJobId, access_data: AsyncJobAccessData | None + app: FastAPI, job_id: AsyncJobId, job_id_data: AsyncJobNameData ) -> AsyncJobResult: assert app # nosec assert job_id # nosec + assert job_id_data # nosec return AsyncJobResult(result="Here's your result.", error=None) @router.expose() -async def list_jobs(app: FastAPI, filter_: str) -> list[AsyncJobGet]: +async def list_jobs( + app: FastAPI, filter_: str, job_id_data: AsyncJobNameData +) -> list[AsyncJobGet]: assert app # nosec - return [AsyncJobGet(job_id=AsyncJobId(f"{uuid4()}"), job_name="myjob")] + return [AsyncJobGet(job_id=AsyncJobId(f"{uuid4()}"))] diff --git a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py index 644daf24586..7e31bddce38 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py @@ -1,7 +1,11 @@ from uuid import uuid4 from fastapi import FastAPI -from models_library.api_schemas_rpc_async_jobs.async_jobs import AsyncJobGet, AsyncJobId +from models_library.api_schemas_rpc_async_jobs.async_jobs import ( + AsyncJobGet, + AsyncJobId, + AsyncJobNameData, +) from models_library.api_schemas_storage.data_export_async_jobs import ( AccessRightError, DataExportError, @@ -10,6 +14,12 @@ ) from servicelib.rabbitmq import RPCRouter +from ...datcore_dsm import DatCoreDataManager +from ...dsm import get_dsm_provider +from ...exceptions.errors import FileAccessRightError +from ...modules.datcore_adapter.datcore_adapter_exceptions import DatcoreAdapterError +from ...simcore_s3_dsm import SimcoreS3DataManager + router = RPCRouter() @@ -21,10 +31,28 @@ ) ) async def start_data_export( - app: FastAPI, paths: DataExportTaskStartInput + app: FastAPI, + data_export_start: DataExportTaskStartInput, + job_id_data: AsyncJobNameData, ) -> AsyncJobGet: assert app # nosec + + dsm = get_dsm_provider(app).get(data_export_start.location_id) + + try: + for _id in data_export_start.file_and_folder_ids: + if isinstance(dsm, DatCoreDataManager): + _ = await dsm.get_file(user_id=job_id_data.user_id, file_id=_id) + elif isinstance(dsm, SimcoreS3DataManager): + await dsm.can_read_file(user_id=job_id_data.user_id, file_id=_id) + + except (FileAccessRightError, DatcoreAdapterError) as err: + raise AccessRightError( + user_id=job_id_data.user_id, + file_id=_id, + location_id=data_export_start.location_id, + ) from err + return AsyncJobGet( job_id=AsyncJobId(f"{uuid4()}"), - job_name=", ".join(str(p) for p in paths.paths), ) diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py index 4888b695ed0..8bb08788ac3 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py @@ -355,6 +355,12 @@ async def get_file(self, user_id: UserID, file_id: StorageFileID) -> FileMetaDat fmd = await self._update_database_from_storage(fmd) return convert_db_to_model(fmd) + async def can_read_file(self, user_id: UserID, file_id: StorageFileID): + async with self.engine.connect() as conn: + can = await get_file_access_rights(conn, int(user_id), file_id) + if not can.read: + raise FileAccessRightError(access_right="read", file_id=file_id) + async def create_file_upload_links( self, user_id: UserID, diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index a12a153dbd2..9c7ee085b08 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -61,12 +61,13 @@ from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings from simcore_service_storage.dsm import get_dsm_provider -from simcore_service_storage.models import S3BucketName +from simcore_service_storage.models import FileMetaData, FileMetaDataAtDB, S3BucketName from simcore_service_storage.modules.long_running_tasks import ( get_completed_upload_tasks, ) from simcore_service_storage.modules.s3 import get_s3_client from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager +from sqlalchemy import literal_column from sqlalchemy.ext.asyncio import AsyncEngine from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -849,3 +850,40 @@ async def with_random_project_with_files( faker: Faker, ) -> tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],]: return await random_project_with_files(project_params) + + +@pytest.fixture() +async def output_file( + user_id: UserID, project_id: str, sqlalchemy_async_engine: AsyncEngine, faker: Faker +) -> AsyncIterator[FileMetaData]: + node_id = "fd6f9737-1988-341b-b4ac-0614b646fa82" + + # pylint: disable=no-value-for-parameter + + file = FileMetaData.from_simcore_node( + user_id=user_id, + file_id=f"{project_id}/{node_id}/filename.txt", + bucket=TypeAdapter(S3BucketName).validate_python("master-simcore"), + location_id=SimcoreS3DataManager.get_location_id(), + location_name=SimcoreS3DataManager.get_location_name(), + sha256_checksum=faker.sha256(), + ) + file.entity_tag = "df9d868b94e53d18009066ca5cd90e9f" + file.file_size = ByteSize(12) + file.user_id = user_id + async with sqlalchemy_async_engine.begin() as conn: + stmt = ( + file_meta_data.insert() + .values(jsonable_encoder(FileMetaDataAtDB.model_validate(file))) + .returning(literal_column("*")) + ) + result = await conn.execute(stmt) + row = result.one() + assert row + + yield file + + async with sqlalchemy_async_engine.begin() as conn: + result = await conn.execute( + file_meta_data.delete().where(file_meta_data.c.file_id == row.file_id) + ) diff --git a/services/storage/tests/unit/test_data_export.py b/services/storage/tests/unit/test_db_data_export.py similarity index 50% rename from services/storage/tests/unit/test_data_export.py rename to services/storage/tests/unit/test_db_data_export.py index 7798621f1e8..1c9d3b01edb 100644 --- a/services/storage/tests/unit/test_data_export.py +++ b/services/storage/tests/unit/test_db_data_export.py @@ -2,6 +2,7 @@ # pylint: disable=W0613 from collections.abc import Awaitable, Callable from pathlib import Path +from typing import Any, Literal, NamedTuple import pytest from faker import Faker @@ -17,12 +18,18 @@ from models_library.api_schemas_storage.data_export_async_jobs import ( DataExportTaskStartInput, ) +from models_library.projects_nodes_io import NodeID, SimcoreS3FileID +from models_library.users import UserID +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.storage_utils import FileIDDict, ProjectWithFilesParams from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.async_jobs import async_jobs from settings_library.rabbit import RabbitSettings +from simcore_service_storage.api.rpc._async_jobs import AsyncJobNameData +from simcore_service_storage.api.rpc._data_export import AccessRightError from simcore_service_storage.core.settings import ApplicationSettings pytest_plugins = [ @@ -74,28 +81,102 @@ async def rpc_client( return await rabbitmq_rpc_client("client") -async def test_start_data_export(rpc_client: RabbitMQRPCClient, faker: Faker): +class UserWithFile(NamedTuple): + user: UserID + file: Path + + +@pytest.mark.parametrize( + "project_params,_type", + [ + ( + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ), + "file", + ), + ( + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ), + "folder", + ), + ], + ids=str, +) +async def test_start_data_export_success( + rpc_client: RabbitMQRPCClient, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], + user_id: UserID, + _type: Literal["file", "folder"], +): + + _, list_of_files = with_random_project_with_files + workspace_files = [ + p for p in list(list_of_files.values())[0].keys() if "/workspace/" in p + ] + assert len(workspace_files) > 0 + file_or_folder_id: SimcoreS3FileID + if _type == "file": + file_or_folder_id = workspace_files[0] + elif _type == "folder": + parts = Path(workspace_files[0]).parts + parts = parts[0 : parts.index("workspace") + 1] + assert len(parts) > 0 + folder = Path(*parts) + assert folder.name == "workspace" + file_or_folder_id = f"{folder}" + else: + pytest.fail("invalid parameter: to_check") + result = await async_jobs.submit_job( rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, - job_name="start_data_export", - paths=DataExportTaskStartInput( - user_id=1, - product_name="osparc", + method_name="start_data_export", + job_id_data=AsyncJobNameData(user_id=user_id, product_name="osparc"), + data_export_start=DataExportTaskStartInput( location_id=0, - paths=[Path(faker.file_path())], + file_and_folder_ids=[file_or_folder_id], ), ) assert isinstance(result, AsyncJobGet) +async def test_start_data_export_fail( + rpc_client: RabbitMQRPCClient, user_id: UserID, faker: Faker +): + + with pytest.raises(AccessRightError): + _ = await async_jobs.submit_job( + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + method_name="start_data_export", + job_id_data=AsyncJobNameData(user_id=user_id, product_name="osparc"), + data_export_start=DataExportTaskStartInput( + location_id=0, + file_and_folder_ids=[ + f"{faker.uuid4()}/{faker.uuid4()}/{faker.file_name()}" + ], + ), + ) + + async def test_abort_data_export(rpc_client: RabbitMQRPCClient, faker: Faker): _job_id = AsyncJobId(faker.uuid4()) result = await async_jobs.abort( rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id_data=AsyncJobNameData( + user_id=faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), job_id=_job_id, - access_data=None, ) assert isinstance(result, AsyncJobAbort) assert result.job_id == _job_id @@ -107,7 +188,9 @@ async def test_get_data_export_status(rpc_client: RabbitMQRPCClient, faker: Fake rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id=_job_id, - access_data=None, + job_id_data=AsyncJobNameData( + user_id=faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), ) assert isinstance(result, AsyncJobStatus) assert result.job_id == _job_id @@ -119,14 +202,21 @@ async def test_get_data_export_result(rpc_client: RabbitMQRPCClient, faker: Fake rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id=_job_id, - access_data=None, + job_id_data=AsyncJobNameData( + user_id=faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), ) assert isinstance(result, AsyncJobResult) async def test_list_jobs(rpc_client: RabbitMQRPCClient, faker: Faker): result = await async_jobs.list_jobs( - rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, filter_="" + rpc_client, + rpc_namespace=STORAGE_RPC_NAMESPACE, + job_id_data=AsyncJobNameData( + user_id=faker.pyint(min_value=1, max_value=100), product_name="osparc" + ), + filter_="", ) assert isinstance(result, list) assert all(isinstance(elm, AsyncJobGet) for elm in result) diff --git a/services/storage/tests/unit/test_dsm_soft_links.py b/services/storage/tests/unit/test_dsm_soft_links.py index da0d363482d..aa2d1be9161 100644 --- a/services/storage/tests/unit/test_dsm_soft_links.py +++ b/services/storage/tests/unit/test_dsm_soft_links.py @@ -3,63 +3,16 @@ # pylint: disable=unused-variable import uuid -from collections.abc import AsyncIterator from functools import lru_cache -import pytest -from faker import Faker -from models_library.api_schemas_storage.storage_schemas import S3BucketName from models_library.projects_nodes_io import SimcoreS3FileID -from models_library.users import UserID -from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import ByteSize, TypeAdapter -from simcore_postgres_database.storage_models import file_meta_data -from simcore_service_storage.models import FileMetaData, FileMetaDataAtDB +from simcore_service_storage.models import FileMetaData from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager -from sqlalchemy.ext.asyncio import AsyncEngine -from sqlalchemy.sql.expression import literal_column pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] -@pytest.fixture() -async def output_file( - user_id: UserID, project_id: str, sqlalchemy_async_engine: AsyncEngine, faker: Faker -) -> AsyncIterator[FileMetaData]: - node_id = "fd6f9737-1988-341b-b4ac-0614b646fa82" - - # pylint: disable=no-value-for-parameter - - file = FileMetaData.from_simcore_node( - user_id=user_id, - file_id=f"{project_id}/{node_id}/filename.txt", - bucket=TypeAdapter(S3BucketName).validate_python("master-simcore"), - location_id=SimcoreS3DataManager.get_location_id(), - location_name=SimcoreS3DataManager.get_location_name(), - sha256_checksum=faker.sha256(), - ) - file.entity_tag = "df9d868b94e53d18009066ca5cd90e9f" - file.file_size = ByteSize(12) - file.user_id = user_id - async with sqlalchemy_async_engine.begin() as conn: - stmt = ( - file_meta_data.insert() - .values(jsonable_encoder(FileMetaDataAtDB.model_validate(file))) - .returning(literal_column("*")) - ) - result = await conn.execute(stmt) - row = result.one() - assert row - - yield file - - async with sqlalchemy_async_engine.begin() as conn: - result = await conn.execute( - file_meta_data.delete().where(file_meta_data.c.file_id == row.file_id) - ) - - def create_reverse_dns(*resource_name_parts) -> str: """ Returns a name for the resource following the reverse domain name notation diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 63c43cff9d3..42573752b6f 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -8488,8 +8488,11 @@ components: properties: paths: items: - type: string - format: path + anyOf: + - type: string + pattern: ^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\/(.+)$ + - type: string + pattern: ^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$ type: array title: Paths type: object diff --git a/services/web/server/src/simcore_service_webserver/storage/_rest.py b/services/web/server/src/simcore_service_webserver/storage/_rest.py index d65a15fefc6..a3839db9d55 100644 --- a/services/web/server/src/simcore_service_webserver/storage/_rest.py +++ b/services/web/server/src/simcore_service_webserver/storage/_rest.py @@ -3,14 +3,13 @@ Mostly resolves and redirect to storage API """ -import json import logging import urllib.parse from typing import Any, Final, NamedTuple from urllib.parse import quote, unquote from aiohttp import ClientTimeout, web -from models_library.api_schemas_rpc_async_jobs.async_jobs import AsyncJobAccessData +from models_library.api_schemas_rpc_async_jobs.async_jobs import AsyncJobNameData from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE from models_library.api_schemas_storage.storage_schemas import ( FileUploadCompleteResponse, @@ -424,10 +423,11 @@ class _PathParams(BaseModel): async_job_rpc_get = await submit_job( rabbitmq_rpc_client=rabbitmq_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, - job_name="start_data_export", - paths=data_export_post.to_rpc_schema( - user_id=_req_ctx.user_id, - product_name=_req_ctx.product_name, + method_name="start_data_export", + job_id_data=AsyncJobNameData( + user_id=_req_ctx.user_id, product_name=_req_ctx.product_name + ), + data_export_start=data_export_post.to_rpc_schema( location_id=_path_params.location_id, ), ) @@ -452,9 +452,10 @@ async def get_async_jobs(request: web.Request) -> web.Response: user_async_jobs = await list_jobs( rabbitmq_rpc_client=rabbitmq_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, - filter_=json.dumps( - {"user_id": _req_ctx.user_id, "product_name": _req_ctx.product_name} + job_id_data=AsyncJobNameData( + user_id=_req_ctx.user_id, product_name=_req_ctx.product_name ), + filter_="", ) return create_data_response( [StorageAsyncJobGet.from_rpc_schema(job) for job in user_async_jobs], @@ -478,7 +479,7 @@ async def get_async_job_status(request: web.Request) -> web.Response: rabbitmq_rpc_client=rabbitmq_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id=async_job_get.job_id, - access_data=AsyncJobAccessData( + job_id_data=AsyncJobNameData( user_id=_req_ctx.user_id, product_name=_req_ctx.product_name ), ) @@ -504,7 +505,7 @@ async def abort_async_job(request: web.Request) -> web.Response: rabbitmq_rpc_client=rabbitmq_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id=async_job_get.job_id, - access_data=AsyncJobAccessData( + job_id_data=AsyncJobNameData( user_id=_req_ctx.user_id, product_name=_req_ctx.product_name ), ) @@ -531,7 +532,7 @@ async def get_async_job_result(request: web.Request) -> web.Response: rabbitmq_rpc_client=rabbitmq_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id=async_job_get.job_id, - access_data=AsyncJobAccessData( + job_id_data=AsyncJobNameData( user_id=_req_ctx.user_id, product_name=_req_ctx.product_name ), ) diff --git a/services/web/server/tests/unit/with_dbs/01/storage/test_storage_rpc.py b/services/web/server/tests/unit/with_dbs/01/storage/test_storage_rpc.py index e68f3ad1ef9..910836c0245 100644 --- a/services/web/server/tests/unit/with_dbs/01/storage/test_storage_rpc.py +++ b/services/web/server/tests/unit/with_dbs/01/storage/test_storage_rpc.py @@ -65,7 +65,7 @@ def side_effect(*args, **kwargs): @pytest.mark.parametrize( "backend_result_or_exception", [ - AsyncJobGet(job_id=AsyncJobId(_faker.uuid4()), job_name=_faker.text()), + AsyncJobGet(job_id=AsyncJobId(f"{_faker.uuid4()}")), InvalidFileIdentifierError(file_id=Path("/my/file")), AccessRightError(user_id=_faker.pyint(min_value=0), file_id=Path("/my/file")), DataExportError(job_id=_faker.pyint(min_value=0)), @@ -85,7 +85,9 @@ async def test_data_export( backend_result_or_exception, ) - _body = DataExportPost(paths=[Path(".")]) + _body = DataExportPost( + paths=[f"{faker.uuid4()}/{faker.uuid4()}/{faker.file_name()}"] + ) response = await client.post( "/v0/storage/locations/0/export-data", data=_body.model_dump_json() ) @@ -106,7 +108,7 @@ async def test_data_export( "backend_result_or_exception", [ AsyncJobStatus( - job_id=_faker.uuid4(), + job_id=f"{_faker.uuid4()}", progress=ProgressReport(actual_value=0.5, total=1.0), done=False, started=datetime.now(),