diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py index 46b8444fde9..b23f65b290c 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py @@ -45,16 +45,18 @@ async def complete_file_upload( uploaded_parts: list[UploadedPart], upload_completion_link: AnyUrl, client_session: ClientSession | None = None, -) -> ETag: + is_directory: bool = False, +) -> ETag | None: async with ClientSessionContextManager(client_session) as session: e_tag: ETag | None = await _filemanager_utils.complete_upload( session=session, upload_completion_link=upload_completion_link, parts=uploaded_parts, - is_directory=False, + is_directory=is_directory, ) # should not be None because a file is being uploaded - assert e_tag is not None # nosec + if not is_directory: + assert e_tag is not None # nosec return e_tag @@ -278,8 +280,7 @@ class UploadedFile: @dataclass -class UploadedFolder: - ... +class UploadedFolder: ... async def _generate_checksum( diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index b7c0befbb47..dfafb2a25f7 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -449,7 +449,15 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ClientFile" + "anyOf": [ + { + "$ref": "#/components/schemas/UserFileToProgramJob" + }, + { + "$ref": "#/components/schemas/UserFile" + } + ], + "title": "Client File" } } } @@ -1248,6 +1256,189 @@ } } }, + "/v0/programs": { + "get": { + "tags": [ + "programs" + ], + "summary": "List Programs", + "description": "Lists all available solvers (latest version)\n\nSEE get_solvers_page for paginated version of this function", + "operationId": "list_programs", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/Program" + }, + "type": "array", + "title": "Response List Programs V0 Programs Get" + } + } + } + } + }, + "security": [ + { + "HTTPBasic": [] + } + ] + } + }, + "/v0/programs/{program_key}/releases/{version}": { + "get": { + "tags": [ + "programs" + ], + "summary": "Get Program Release", + "description": "Gets a specific release of a solver", + "operationId": "get_program_release", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "program_key", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^simcore/services/dynamic/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", + "title": "Program Key" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", + "title": "Version" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Program" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/programs/{program_key}/releases/{version}/jobs": { + "post": { + "tags": [ + "programs" + ], + "summary": "Create Program Job", + "description": "Creates a job in a specific release with given inputs.\n\nNOTE: This operation does **not** start the job", + "operationId": "create_program_job", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "program_key", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^simcore/services/dynamic/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", + "title": "Program Key" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", + "title": "Version" + } + }, + { + "name": "x-simcore-parent-project-uuid", + "in": "header", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "X-Simcore-Parent-Project-Uuid" + } + }, + { + "name": "x-simcore-parent-node-id", + "in": "header", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "X-Simcore-Parent-Node-Id" + } + } + ], + "responses": { + "201": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Job" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v0/solvers": { "get": { "tags": [ @@ -2014,9 +2205,9 @@ "tags": [ "solvers" ], - "summary": "Create Job", + "summary": "Create Solver Job", "description": "Creates a job in a specific release with given inputs.\n\nNOTE: This operation does **not** start the job", - "operationId": "create_job", + "operationId": "create_solver_job", "security": [ { "HTTPBasic": [] @@ -5855,7 +6046,15 @@ "Body_abort_multipart_upload_v0_files__file_id__abort_post": { "properties": { "client_file": { - "$ref": "#/components/schemas/ClientFile" + "anyOf": [ + { + "$ref": "#/components/schemas/UserFileToProgramJob" + }, + { + "$ref": "#/components/schemas/UserFile" + } + ], + "title": "Client File" } }, "type": "object", @@ -5867,7 +6066,15 @@ "Body_complete_multipart_upload_v0_files__file_id__complete_post": { "properties": { "client_file": { - "$ref": "#/components/schemas/ClientFile" + "anyOf": [ + { + "$ref": "#/components/schemas/UserFileToProgramJob" + }, + { + "$ref": "#/components/schemas/UserFile" + } + ], + "title": "Client File" }, "uploaded_parts": { "$ref": "#/components/schemas/FileUploadCompletionBody" @@ -5894,35 +6101,6 @@ ], "title": "Body_upload_file_v0_files_content_put" }, - "ClientFile": { - "properties": { - "filename": { - "type": "string", - "title": "Filename", - "description": "File name" - }, - "filesize": { - "type": "integer", - "minimum": 0, - "title": "Filesize", - "description": "File size in bytes" - }, - "sha256_checksum": { - "type": "string", - "pattern": "^[a-fA-F0-9]{64}$", - "title": "Sha256 Checksum", - "description": "SHA256 checksum" - } - }, - "type": "object", - "required": [ - "filename", - "filesize", - "sha256_checksum" - ], - "title": "ClientFile", - "description": "Represents a file stored on the client side" - }, "ClientFileUploadData": { "properties": { "file_id": { @@ -7522,6 +7700,72 @@ "type": "object", "title": "ProfileUpdate" }, + "Program": { + "properties": { + "id": { + "type": "string", + "title": "Id", + "description": "Resource identifier" + }, + "version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", + "title": "Version", + "description": "Semantic version number of the resource" + }, + "title": { + "type": "string", + "maxLength": 100, + "title": "Title", + "description": "Human readable name" + }, + "description": { + "anyOf": [ + { + "type": "string", + "maxLength": 500 + }, + { + "type": "null" + } + ], + "title": "Description", + "description": "Description of the resource" + }, + "url": { + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], + "title": "Url", + "description": "Link to get this resource" + } + }, + "type": "object", + "required": [ + "id", + "version", + "title", + "url" + ], + "title": "Program", + "description": "A released program with a specific version", + "example": { + "description": "Simulation framework", + "id": "simcore/services/dynamic/sim4life", + "maintainer": "info@itis.swiss", + "title": "Sim4life", + "url": "https://api.osparc.io/v0/solvers/simcore%2Fservices%2Fdynamic%2Fsim4life/releases/8.0.0", + "version": "8.0.0" + } + }, "RunningState": { "type": "string", "enum": [ @@ -7591,35 +7835,33 @@ "properties": { "id": { "type": "string", - "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Id", - "description": "Solver identifier" + "description": "Resource identifier" }, "version": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version", - "description": "semantic version number of the node" + "description": "Semantic version number of the resource" }, "title": { "type": "string", + "maxLength": 100, "title": "Title", "description": "Human readable name" }, "description": { "anyOf": [ { - "type": "string" + "type": "string", + "maxLength": 500 }, { "type": "null" } ], - "title": "Description" - }, - "maintainer": { - "type": "string", - "title": "Maintainer" + "title": "Description", + "description": "Description of the resource" }, "url": { "anyOf": [ @@ -7635,6 +7877,11 @@ ], "title": "Url", "description": "Link to get this resource" + }, + "maintainer": { + "type": "string", + "title": "Maintainer", + "description": "Maintainer of the solver" } }, "type": "object", @@ -7642,8 +7889,8 @@ "id", "version", "title", - "maintainer", - "url" + "url", + "maintainer" ], "title": "Solver", "description": "A released solver with a specific version", @@ -7852,6 +8099,93 @@ ], "title": "UploadedPart" }, + "UserFile": { + "properties": { + "filename": { + "type": "string", + "title": "Filename", + "description": "File name" + }, + "filesize": { + "type": "integer", + "minimum": 0, + "title": "Filesize", + "description": "File size in bytes" + }, + "sha256_checksum": { + "type": "string", + "pattern": "^[a-fA-F0-9]{64}$", + "title": "Sha256 Checksum", + "description": "SHA256 checksum" + } + }, + "type": "object", + "required": [ + "filename", + "filesize", + "sha256_checksum" + ], + "title": "UserFile", + "description": "Represents a file stored on the client side" + }, + "UserFileToProgramJob": { + "properties": { + "filename": { + "type": "string", + "pattern": ".+", + "title": "Filename", + "description": "File name" + }, + "filesize": { + "type": "integer", + "minimum": 0, + "title": "Filesize", + "description": "File size in bytes" + }, + "sha256_checksum": { + "type": "string", + "pattern": "^[a-fA-F0-9]{64}$", + "title": "Sha256 Checksum", + "description": "SHA256 checksum" + }, + "program_key": { + "type": "string", + "pattern": "^simcore/services/dynamic/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", + "title": "Program Key", + "description": "Program identifier" + }, + "program_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", + "title": "Program Version", + "description": "Program version" + }, + "job_id": { + "type": "string", + "format": "uuid", + "title": "Job Id", + "description": "Job identifier" + }, + "workspace_path": { + "type": "string", + "pattern": "^workspace/.*", + "format": "path", + "title": "Workspace Path", + "description": "The file's relative path within the job's workspace directory. E.g. 'workspace/myfile.txt'" + } + }, + "type": "object", + "required": [ + "filename", + "filesize", + "sha256_checksum", + "program_key", + "program_version", + "job_id", + "workspace_path" + ], + "title": "UserFileToProgramJob" + }, "UserRoleEnum": { "type": "string", "enum": [ diff --git a/services/api-server/src/simcore_service_api_server/_service.py b/services/api-server/src/simcore_service_api_server/_service.py new file mode 100644 index 00000000000..bc826b1a310 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/_service.py @@ -0,0 +1,58 @@ +import logging +from collections.abc import Callable + +from models_library.api_schemas_webserver.projects import ProjectCreateNew, ProjectGet +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID +from pydantic import HttpUrl +from simcore_service_api_server.models.schemas.jobs import Job, JobInputs +from simcore_service_api_server.models.schemas.programs import Program +from simcore_service_api_server.models.schemas.solvers import Solver +from simcore_service_api_server.services_http.solver_job_models_converters import ( + create_job_from_project, + create_new_project_for_job, +) +from simcore_service_api_server.services_http.webserver import AuthSession + +_logger = logging.getLogger(__name__) + + +async def create_solver_or_program_job( + *, + webserver_api: AuthSession, + solver_or_program: Solver | Program, + inputs: JobInputs, + parent_project_uuid: ProjectID | None, + parent_node_id: NodeID | None, + url_for: Callable[..., HttpUrl], + hidden: bool +) -> tuple[Job, ProjectGet]: + # creates NEW job as prototype + pre_job = Job.create_job_from_solver_or_program( + solver_or_program_name=solver_or_program.name, inputs=inputs + ) + _logger.debug("Creating Job '%s'", pre_job.name) + + project_in: ProjectCreateNew = create_new_project_for_job( + solver_or_program, pre_job, inputs + ) + new_project: ProjectGet = await webserver_api.create_project( + project_in, + is_hidden=hidden, + parent_project_uuid=parent_project_uuid, + parent_node_id=parent_node_id, + ) + assert new_project # nosec + assert new_project.uuid == pre_job.id # nosec + + # for consistency, it rebuild job + job = create_job_from_project( + solver_or_program=solver_or_program, project=new_project, url_for=url_for + ) + assert job.id == pre_job.id # nosec + assert job.name == pre_job.name # nosec + assert job.name == Job.compose_resource_name( + parent_name=solver_or_program.resource_name, + job_id=job.id, + ) + return job, new_project diff --git a/services/api-server/src/simcore_service_api_server/api/root.py b/services/api-server/src/simcore_service_api_server/api/root.py index c50511dda88..5654601d403 100644 --- a/services/api-server/src/simcore_service_api_server/api/root.py +++ b/services/api-server/src/simcore_service_api_server/api/root.py @@ -10,6 +10,7 @@ health, licensed_items, meta, + programs, solvers, solvers_jobs, solvers_jobs_getters, @@ -33,6 +34,7 @@ def create_router(settings: ApplicationSettings): router.include_router(meta.router, tags=["meta"], prefix="/meta") router.include_router(users.router, tags=["users"], prefix="/me") router.include_router(files.router, tags=["files"], prefix="/files") + router.include_router(programs.router, tags=["programs"], prefix="/programs") router.include_router(solvers.router, tags=["solvers"], prefix=_SOLVERS_PREFIX) router.include_router(solvers_jobs.router, tags=["solvers"], prefix=_SOLVERS_PREFIX) router.include_router( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/files.py b/services/api-server/src/simcore_service_api_server/api/routes/files.py index 0821d81abab..04e35bc00f9 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/files.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/files.py @@ -16,6 +16,7 @@ LinkType, ) from models_library.basic_types import SHA256Str +from models_library.projects_nodes_io import NodeID from pydantic import AnyUrl, ByteSize, PositiveInt, TypeAdapter, ValidationError from servicelib.fastapi.requests_decorators import cancel_on_disconnect from simcore_sdk.node_ports_common.constants import SIMCORE_LOCATION @@ -31,17 +32,25 @@ from starlette.datastructures import URL from starlette.responses import RedirectResponse +from ...api.dependencies.webserver_http import ( + get_webserver_session, +) from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES +from ...models.domain.files import File as DomainFile from ...models.pagination import Page, PaginationParams from ...models.schemas.errors import ErrorGet from ...models.schemas.files import ( - ClientFile, ClientFileUploadData, - File, +) +from ...models.schemas.files import File as OutputFile +from ...models.schemas.files import ( FileUploadData, UploadLinks, + UserFile, ) +from ...models.schemas.jobs import UserFileToProgramJob from ...services_http.storage import StorageApi, StorageFileMetaData, to_file_api_model +from ...services_http.webserver import AuthSession from ..dependencies.authentication import get_current_user_id from ..dependencies.services import get_api_client from ._common import API_SERVER_DEV_FEATURES_ENABLED @@ -70,14 +79,14 @@ async def _get_file( file_id: UUID, storage_client: StorageApi, user_id: int, -): +) -> DomainFile: """Gets metadata for a given file resource""" try: - stored_files: list[ - StorageFileMetaData - ] = await storage_client.search_owned_files( - user_id=user_id, file_id=file_id, limit=1 + stored_files: list[StorageFileMetaData] = ( + await storage_client.search_owned_files( + user_id=user_id, file_id=file_id, limit=1 + ) ) if not stored_files: msg = "Not found in storage" @@ -98,7 +107,32 @@ async def _get_file( ) from err -@router.get("", response_model=list[File], responses=_FILE_STATUS_CODES) +async def _create_domain_file( + *, + webserver_api: AuthSession, + file_id: UUID | None, + client_file: UserFile | UserFileToProgramJob, +) -> DomainFile: + if isinstance(client_file, UserFile): + file = client_file.to_domain_model(file_id=file_id) + elif isinstance(client_file, UserFileToProgramJob): + project = await webserver_api.get_project(project_id=client_file.job_id) + if len(project.workbench) > 1: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Job_id {project.uuid} is not a valid program job.", + ) + node_id = next(iter(project.workbench.keys())) + file = client_file.to_domain_model( + project_id=project.uuid, node_id=NodeID(node_id) + ) + else: + err_msg = f"Invalid client_file type passed: {type(client_file)=}" + raise TypeError(err_msg) + return file + + +@router.get("", response_model=list[OutputFile], responses=_FILE_STATUS_CODES) async def list_files( storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], user_id: Annotated[int, Depends(get_current_user_id)], @@ -113,12 +147,12 @@ async def list_files( ) # Adapts storage API model to API model - all_files: list[File] = [] + all_files: list[OutputFile] = [] for stored_file_meta in stored_files: try: assert stored_file_meta.file_id # nosec - file_meta: File = to_file_api_model(stored_file_meta) + file_meta = to_file_api_model(stored_file_meta) except (ValidationError, ValueError, AttributeError) as err: _logger.warning( @@ -129,14 +163,14 @@ async def list_files( ) else: - all_files.append(file_meta) + all_files.append(OutputFile.from_domain_model(file_meta)) return all_files @router.get( "/page", - response_model=Page[File], + response_model=Page[OutputFile], include_in_schema=API_SERVER_DEV_FEATURES_ENABLED, status_code=status.HTTP_501_NOT_IMPLEMENTED, ) @@ -161,7 +195,7 @@ def _get_spooled_file_size(file_io: IO) -> int: @router.put( "/content", - response_model=File, + response_model=OutputFile, responses=_FILE_STATUS_CODES, ) @cancel_on_disconnect @@ -187,7 +221,7 @@ async def upload_file( None, _get_spooled_file_size, file.file ) # assign file_id. - file_meta: File = await File.create_from_uploaded( + file_meta = await DomainFile.create_from_uploaded( file, file_size=file_size, created_at=datetime.datetime.now(datetime.UTC).isoformat(), @@ -216,7 +250,7 @@ async def upload_file( assert isinstance(upload_result, UploadedFile) # nosec file_meta.e_tag = upload_result.etag - return file_meta + return OutputFile.from_domain_model(file_meta) # NOTE: MaG suggested a single function that can upload one or multiple files instead of having @@ -239,14 +273,14 @@ async def upload_files(files: list[UploadFile] = FileParam(...)): @cancel_on_disconnect async def get_upload_links( request: Request, - client_file: ClientFile, + client_file: UserFileToProgramJob | UserFile, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], ): """Get upload links for uploading a file to storage""" assert request # nosec - file_meta: File = await File.create_from_client_file( - client_file, - datetime.datetime.now(datetime.UTC).isoformat(), + file_meta = await _create_domain_file( + webserver_api=webserver_api, file_id=None, client_file=client_file ) _, upload_links = await get_upload_links_from_s3( user_id=user_id, @@ -275,7 +309,7 @@ async def get_upload_links( @router.get( "/{file_id}", - response_model=File, + response_model=OutputFile, responses=_FILE_STATUS_CODES, ) async def get_file( @@ -294,7 +328,7 @@ async def get_file( @router.get( ":search", - response_model=Page[File], + response_model=Page[OutputFile], responses=_FILE_STATUS_CODES, ) async def search_files_page( @@ -317,8 +351,11 @@ async def search_files_page( raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Not found in storage" ) + file_list = [ + OutputFile.from_domain_model(to_file_api_model(fmd)) for fmd in stored_files + ] return create_page( - [to_file_api_model(fmd) for fmd in stored_files], + file_list, total=len(stored_files), params=page_params, ) @@ -333,7 +370,7 @@ async def delete_file( user_id: Annotated[int, Depends(get_current_user_id)], storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], ): - file: File = await _get_file( + file = await _get_file( file_id=file_id, storage_client=storage_client, user_id=user_id, @@ -350,17 +387,17 @@ async def delete_file( async def abort_multipart_upload( request: Request, file_id: UUID, - client_file: Annotated[ClientFile, Body(..., embed=True)], + client_file: Annotated[UserFileToProgramJob | UserFile, Body(..., embed=True)], storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], ): + assert file_id # nosec assert request # nosec assert user_id # nosec - file: File = File( - id=file_id, - filename=client_file.filename, - checksum=client_file.sha256_checksum, - e_tag=None, + + file = await _create_domain_file( + webserver_api=webserver_api, file_id=file_id, client_file=client_file ) abort_link: URL = await storage_client.create_abort_upload_link( file=file, query={"user_id": str(user_id)} @@ -372,35 +409,34 @@ async def abort_multipart_upload( @router.post( "/{file_id}:complete", - response_model=File, + response_model=OutputFile, responses=_FILE_STATUS_CODES, ) @cancel_on_disconnect async def complete_multipart_upload( request: Request, file_id: UUID, - client_file: Annotated[ClientFile, Body(...)], + client_file: Annotated[UserFileToProgramJob | UserFile, Body(...)], uploaded_parts: Annotated[FileUploadCompletionBody, Body(...)], storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], ): + assert file_id # nosec assert request # nosec assert user_id # nosec - - file: File = File( - id=file_id, - filename=client_file.filename, - checksum=client_file.sha256_checksum, - e_tag=None, + file = await _create_domain_file( + webserver_api=webserver_api, file_id=file_id, client_file=client_file ) complete_link: URL = await storage_client.create_complete_upload_link( file=file, query={"user_id": str(user_id)} ) - e_tag: ETag = await complete_file_upload( + e_tag: ETag | None = await complete_file_upload( uploaded_parts=uploaded_parts.parts, upload_completion_link=TypeAdapter(AnyUrl).validate_python(f"{complete_link}"), ) + assert e_tag is not None # nosec file.e_tag = e_tag return file @@ -429,7 +465,7 @@ async def download_file( ): # NOTE: application/octet-stream is defined as "arbitrary binary data" in RFC 2046, # gets meta - file_meta: File = await get_file(file_id, storage_client, user_id) + file_meta = await get_file(file_id, storage_client, user_id) # download from S3 using pre-signed link presigned_download_link = await storage_client.get_download_link( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/programs.py b/services/api-server/src/simcore_service_api_server/api/routes/programs.py new file mode 100644 index 00000000000..5f69580e06b --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/api/routes/programs.py @@ -0,0 +1,162 @@ +import logging +from collections.abc import Callable +from operator import attrgetter +from typing import Annotated + +from fastapi import APIRouter, Depends, Header, HTTPException, status +from httpx import HTTPStatusError +from models_library.api_schemas_storage.storage_schemas import ( + LinkType, +) +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID +from pydantic import ByteSize, PositiveInt, ValidationError +from servicelib.fastapi.dependencies import get_reverse_url_mapper +from simcore_sdk.node_ports_common.constants import SIMCORE_LOCATION +from simcore_sdk.node_ports_common.filemanager import ( + complete_file_upload, + get_upload_links_from_s3, +) +from simcore_service_api_server._service import create_solver_or_program_job +from simcore_service_api_server.api.dependencies.webserver_http import ( + get_webserver_session, +) +from simcore_service_api_server.services_http.webserver import AuthSession + +from ...models.basic_types import VersionStr +from ...models.schemas.jobs import Job, JobInputs +from ...models.schemas.programs import Program, ProgramKeyId +from ...services_http.catalog import CatalogApi +from ..dependencies.authentication import get_current_user_id, get_product_name +from ..dependencies.services import get_api_client + +_logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.get("", response_model=list[Program]) +async def list_programs( + user_id: Annotated[int, Depends(get_current_user_id)], + catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + product_name: Annotated[str, Depends(get_product_name)], +): + """Lists all available solvers (latest version) + + SEE get_solvers_page for paginated version of this function + """ + services = await catalog_client.list_services( + user_id=user_id, + product_name=product_name, + predicate=None, + type_filter="DYNAMIC", + ) + + programs = [service.to_program() for service in services] + + for program in programs: + program.url = url_for( + "get_program_release", program_key=program.id, version=program.version + ) + + return sorted(programs, key=attrgetter("id")) + + +@router.get( + "/{program_key:path}/releases/{version}", + response_model=Program, +) +async def get_program_release( + program_key: ProgramKeyId, + version: VersionStr, + user_id: Annotated[int, Depends(get_current_user_id)], + catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + product_name: Annotated[str, Depends(get_product_name)], +) -> Program: + """Gets a specific release of a solver""" + try: + program = await catalog_client.get_program( + user_id=user_id, + name=program_key, + version=version, + product_name=product_name, + ) + + program.url = url_for( + "get_program_release", program_key=program.id, version=program.version + ) + return program + + except ( + ValueError, + IndexError, + ValidationError, + HTTPStatusError, + ) as err: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Program {program_key}:{version} not found", + ) from err + + +@router.post( + "/{program_key:path}/releases/{version}/jobs", + response_model=Job, + status_code=status.HTTP_201_CREATED, +) +async def create_program_job( + program_key: ProgramKeyId, + version: VersionStr, + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + product_name: Annotated[str, Depends(get_product_name)], + x_simcore_parent_project_uuid: Annotated[ProjectID | None, Header()] = None, + x_simcore_parent_node_id: Annotated[NodeID | None, Header()] = None, +): + """Creates a job in a specific release with given inputs. + + NOTE: This operation does **not** start the job + """ + + # ensures user has access to solver + inputs = JobInputs(values={}) + program = await catalog_client.get_program( + user_id=user_id, + name=program_key, + version=version, + product_name=product_name, + ) + + job, project = await create_solver_or_program_job( + webserver_api=webserver_api, + solver_or_program=program, + inputs=inputs, + parent_project_uuid=x_simcore_parent_project_uuid, + parent_node_id=x_simcore_parent_node_id, + url_for=url_for, + hidden=False, + ) + # create workspace directory so files can be uploaded to it + assert len(project.workbench) > 0 # nosec + node_id = next(iter(project.workbench)) + + _, file_upload_schema = await get_upload_links_from_s3( + user_id=user_id, + store_name=None, + store_id=SIMCORE_LOCATION, + s3_object=f"{project.uuid}/{node_id}/workspace", + link_type=LinkType.PRESIGNED, + client_session=None, + file_size=ByteSize(0), + is_directory=True, + sha256_checksum=None, + ) + await complete_file_upload( + uploaded_parts=[], + upload_completion_link=file_upload_schema.links.complete_upload, + is_directory=True, + ) + return job diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index f9f80b37c4c..89b220d2d5f 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -97,9 +97,13 @@ async def list_solvers_releases( """ assert await catalog_client.is_responsive() # nosec - solvers: list[Solver] = await catalog_client.list_solvers( - user_id=user_id, product_name=product_name + services = await catalog_client.list_services( + user_id=user_id, + product_name=product_name, + predicate=None, + type_filter="COMPUTATIONAL", ) + solvers = [service.to_solver() for service in services] for solver in solvers: solver.url = url_for( @@ -140,7 +144,9 @@ async def get_solver( # otherwise, {solver_key:path} will override and consume any of the paths that follow. try: solver = await catalog_client.get_latest_release( - user_id=user_id, solver_key=solver_key, product_name=product_name + user_id=user_id, + solver_key=solver_key, + product_name=product_name, ) solver.url = url_for( "get_solver_release", solver_key=solver.id, version=solver.version @@ -171,8 +177,10 @@ async def list_solver_releases( SEE get_solver_releases_page for a paginated version of this function """ - releases: list[Solver] = await catalog_client.list_solver_releases( - user_id=user_id, solver_key=solver_key, product_name=product_name + releases: list[Solver] = await catalog_client.list_service_releases( + user_id=user_id, + solver_key=solver_key, + product_name=product_name, ) for solver in releases: @@ -212,7 +220,7 @@ async def get_solver_release( ) -> Solver: """Gets a specific release of a solver""" try: - solver: Solver = await catalog_client.get_service( + solver: Solver = await catalog_client.get_solver( user_id=user_id, name=solver_key, version=version, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index b0737356e7f..3fc1b1109ad 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -7,12 +7,12 @@ from fastapi import APIRouter, Depends, Header, Query, Request, status from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse -from models_library.api_schemas_webserver.projects import ProjectCreateNew, ProjectGet from models_library.clusters import ClusterID from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from pydantic.types import PositiveInt +from ..._service import create_solver_or_program_job from ...exceptions.backend_errors import ProjectAlreadyStartedError from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES from ...models.basic_types import VersionStr @@ -30,9 +30,7 @@ from ...services_http.director_v2 import DirectorV2Api from ...services_http.jobs import replace_custom_metadata, start_project, stop_project from ...services_http.solver_job_models_converters import ( - create_job_from_project, create_jobstatus_from_task, - create_new_project_for_job, ) from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.application import get_reverse_url_mapper @@ -92,7 +90,7 @@ def _compose_job_resource_name(solver_key, solver_version, job_id) -> str: status_code=status.HTTP_201_CREATED, responses=JOBS_STATUS_CODES, ) -async def create_job( +async def create_solver_job( solver_key: SolverKeyId, version: VersionStr, inputs: JobInputs, @@ -112,46 +110,28 @@ async def create_job( """ # ensures user has access to solver - solver = await catalog_client.get_service( + solver = await catalog_client.get_solver( user_id=user_id, name=solver_key, version=version, product_name=product_name, ) - - # creates NEW job as prototype - pre_job = Job.create_solver_job(solver=solver, inputs=inputs) - _logger.debug("Creating Job '%s'", pre_job.name) - - project_in: ProjectCreateNew = create_new_project_for_job(solver, pre_job, inputs) - new_project: ProjectGet = await webserver_api.create_project( - project_in, - is_hidden=hidden, + job, project = await create_solver_or_program_job( + webserver_api=webserver_api, + solver_or_program=solver, + inputs=inputs, + url_for=url_for, + hidden=hidden, parent_project_uuid=x_simcore_parent_project_uuid, parent_node_id=x_simcore_parent_node_id, ) - assert new_project # nosec - assert new_project.uuid == pre_job.id # nosec - - # for consistency, it rebuild job - job = create_job_from_project( - solver_key=solver.id, - solver_version=solver.version, - project=new_project, - url_for=url_for, - ) - assert job.id == pre_job.id # nosec - assert job.name == pre_job.name # nosec - assert job.name == _compose_job_resource_name(solver_key, version, job.id) # nosec - await wb_api_rpc.mark_project_as_job( product_name=product_name, user_id=user_id, - project_uuid=new_project.uuid, + project_uuid=project.uuid, job_parent_resource_name=job.runner_name, ) - return job diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index 3c2bd479b0a..e156863c90b 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -22,9 +22,10 @@ from ...exceptions.custom_errors import InsufficientCreditsError, MissingWalletError from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES from ...models.basic_types import LogStreamingResponse, VersionStr +from ...models.domain.files import File as DomainFile from ...models.pagination import Page, PaginationParams from ...models.schemas.errors import ErrorGet -from ...models.schemas.files import File +from ...models.schemas.files import File as SchemaFile from ...models.schemas.jobs import ( ArgumentTypes, Job, @@ -133,7 +134,7 @@ async def list_jobs( - SEE `get_jobs_page` for paginated version of this function """ - solver = await catalog_client.get_service( + solver = await catalog_client.get_solver( user_id=user_id, name=solver_key, version=version, @@ -147,7 +148,9 @@ async def list_jobs( jobs: deque[Job] = deque() for prj in projects_page.data: - job = create_job_from_project(solver_key, version, prj, url_for) + job = create_job_from_project( + solver_or_program=solver, project=prj, url_for=url_for + ) assert job.id == prj.uuid # nosec assert job.name == prj.name # nosec @@ -178,7 +181,7 @@ async def get_jobs_page( # NOTE: Different entry to keep backwards compatibility with list_jobs. # Eventually use a header with agent version to switch to new interface - solver = await catalog_client.get_service( + solver = await catalog_client.get_solver( user_id=user_id, name=solver_key, version=version, @@ -191,7 +194,7 @@ async def get_jobs_page( ) jobs: list[Job] = [ - create_job_from_project(solver_key, version, prj, url_for) + create_job_from_project(solver_or_program=solver, project=prj, url_for=url_for) for prj in projects_page.data ] @@ -211,7 +214,10 @@ async def get_job( solver_key: SolverKeyId, version: VersionStr, job_id: JobID, + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + product_name: Annotated[str, Depends(get_product_name)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], ): """Gets job of a given solver""" @@ -219,9 +225,17 @@ async def get_job( "Getting Job '%s'", _compose_job_resource_name(solver_key, version, job_id) ) + solver = await catalog_client.get_solver( + user_id=user_id, + name=solver_key, + version=version, + product_name=product_name, + ) project: ProjectGet = await webserver_api.get_project(project_id=job_id) - job = create_job_from_project(solver_key, version, project, url_for) + job = create_job_from_project( + solver_or_program=solver, project=project, url_for=url_for + ) assert job.id == job_id # nosec return job # nosec @@ -269,19 +283,21 @@ async def get_job_outputs( results: dict[str, ArgumentTypes] = {} for name, value in outputs.items(): if isinstance(value, BaseFileLink): - file_id: UUID = File.create_id(*value.path.split("/")) + file_id: UUID = DomainFile.create_id(*value.path.split("/")) found = await storage_client.search_owned_files( user_id=user_id, file_id=file_id, limit=1 ) if found: assert len(found) == 1 # nosec - results[name] = to_file_api_model(found[0]) + results[name] = SchemaFile.from_domain_model( + to_file_api_model(found[0]) + ) else: - api_file: File = await storage_client.create_soft_link( + api_file = await storage_client.create_soft_link( user_id=user_id, target_s3_path=value.path, as_file_id=file_id ) - results[name] = api_file + results[name] = SchemaFile.from_domain_model(api_file) else: results[name] = value diff --git a/services/api-server/src/simcore_service_api_server/models/domain/files.py b/services/api-server/src/simcore_service_api_server/models/domain/files.py new file mode 100644 index 00000000000..82b73cb4456 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/models/domain/files.py @@ -0,0 +1,147 @@ +from pathlib import Path +from typing import Annotated +from urllib.parse import quote as _quote +from urllib.parse import unquote as _unquote +from uuid import UUID, uuid3 + +import aiofiles +from fastapi import UploadFile +from models_library.api_schemas_storage.storage_schemas import ETag +from models_library.basic_types import SHA256Str +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID, StorageFileID +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StringConstraints, + TypeAdapter, +) +from servicelib.file_utils import create_sha256_checksum + +FileName = Annotated[str, StringConstraints(strip_whitespace=True)] +NAMESPACE_FILEID_KEY = UUID("aa154444-d22d-4290-bb15-df37dba87865") + + +class FileInProgramJobData(BaseModel): + """Represents a file stored on the client side""" + + project_id: Annotated[ProjectID, Field(..., description="Project identifier")] + node_id: Annotated[NodeID, Field(..., description="Node identifier")] + workspace_path: Annotated[ + Path, + StringConstraints(pattern=r"^workspace/.*"), + Field(..., description="File path within the workspace"), + ] + + +class File(BaseModel): + """Represents a file stored on the server side i.e. a unique reference to a file in the cloud.""" + + # WARNING: from pydantic import File as FileParam + # NOTE: see https://ant.apache.org/manual/Tasks/checksum.html + + id: UUID = Field(..., description="Resource identifier") + + filename: str = Field(..., description="Name of the file with extension") + content_type: str | None = Field( + default=None, + description="Guess of type content [EXPERIMENTAL]", + validate_default=True, + ) + sha256_checksum: SHA256Str | None = Field( + default=None, + description="SHA256 hash of the file's content", + alias="checksum", # alias for backwards compatibility + ) + e_tag: ETag | None = Field(default=None, description="S3 entity tag") + program_job_file_path: Annotated[ + FileInProgramJobData | None, + Field( + ..., + description="Destination information in case the file is uploaded directly to a program job", + ), + ] = None + + model_config = ConfigDict( + populate_by_name=True, + json_schema_extra={ + "examples": [ + # complete + { + "id": "f0e1fb11-208d-3ed2-b5ef-cab7a7398f78", + "filename": "Architecture-of-Scalable-Distributed-ETL-System-whitepaper.pdf", + "content_type": "application/pdf", + "checksum": "1a512547e3ce3427482da14e8c914ecf61da76ad5f749ff532efe906e6bba128", + }, + # minimum + { + "id": "f0e1fb11-208d-3ed2-b5ef-cab7a7398f78", + "filename": "whitepaper.pdf", + }, + ] + }, + ) + + @classmethod + async def create_from_path(cls, path: Path) -> "File": + async with aiofiles.open(path, mode="rb") as file: + sha256check = await create_sha256_checksum(file) + + return cls( + id=cls.create_id(sha256check, path.name), + filename=path.name, + checksum=SHA256Str(sha256check), + ) + + @classmethod + async def create_from_file_link(cls, s3_object_path: str, e_tag: str) -> "File": + filename = Path(s3_object_path).name + return cls( + id=cls.create_id(e_tag, filename), + filename=filename, + e_tag=e_tag, + ) + + @classmethod + async def create_from_uploaded( + cls, file: UploadFile, *, file_size=None, created_at=None + ) -> "File": + sha256check = await create_sha256_checksum(file) + # WARNING: UploadFile wraps a stream and wil checkt its cursor position: file.file.tell() != 0 + # WARNING: await file.seek(0) might introduce race condition if not done carefuly + + return cls( + id=cls.create_id(sha256check or file_size, file.filename, created_at), + filename=file.filename or "Undefined", + content_type=file.content_type, + checksum=SHA256Str(sha256check), + ) + + @classmethod + async def create_from_quoted_storage_id(cls, quoted_storage_id: str) -> "File": + storage_file_id: StorageFileID = TypeAdapter(StorageFileID).validate_python( + _unquote(quoted_storage_id) + ) + _, fid, fname = Path(storage_file_id).parts + return cls(id=UUID(fid), filename=fname, checksum=None) + + @classmethod + def create_id(cls, *keys) -> UUID: + return uuid3(NAMESPACE_FILEID_KEY, ":".join(map(str, keys))) + + @property + def storage_file_id(self) -> StorageFileID: + """Get the StorageFileId associated with this file""" + if program_path := self.program_job_file_path: + return TypeAdapter(StorageFileID).validate_python( + f"{program_path.project_id}/{program_path.node_id}/{program_path.workspace_path}" + ) + return TypeAdapter(StorageFileID).validate_python( + f"api/{self.id}/{self.filename}" + ) + + @property + def quoted_storage_file_id(self) -> str: + """Quoted version of the StorageFileId""" + return _quote(self.storage_file_id, safe="") diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/_base.py b/services/api-server/src/simcore_service_api_server/models/schemas/_base.py new file mode 100644 index 00000000000..dec37f87f90 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/models/schemas/_base.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import Annotated + +import packaging.version +from models_library.utils.change_case import camel_to_snake +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, StringConstraints + +from ...models._utils_pydantic import UriSchema +from ..basic_types import VersionStr + + +class ApiServerOutputSchema(BaseModel): + model_config = ConfigDict( + alias_generator=camel_to_snake, + populate_by_name=True, + extra="ignore", # Used to prune extra fields from internal data + frozen=True, + ) + + +class ApiServerInputSchema(BaseModel): + model_config = ConfigDict( + alias_generator=camel_to_snake, + populate_by_name=True, + extra="ignore", # Used to prune extra fields from internal data + frozen=True, + ) + + +class BaseService(BaseModel): + id: Annotated[str, Field(..., description="Resource identifier")] + version: Annotated[ + VersionStr, Field(..., description="Semantic version number of the resource") + ] + title: Annotated[ + str, + StringConstraints(max_length=100), + Field(..., description="Human readable name"), + ] + description: Annotated[ + str | None, + StringConstraints(max_length=500), + Field(default=None, description="Description of the resource"), + ] + url: Annotated[ + HttpUrl | None, UriSchema(), Field(..., description="Link to get this resource") + ] + + @property + def pep404_version(self) -> packaging.version.Version: + """Rich version type that can be used e.g. to compare""" + return packaging.version.parse(self.version) + + @property + def url_friendly_id(self) -> str: + """Use to pass id as parameter in URLs""" + return urllib.parse.quote_plus(self.id) + + @property + def resource_name(self) -> str: + """Relative resource name""" + return self.compose_resource_name(self.id, self.version) + + @property + def name(self) -> str: + """API standards notation (see api_resources.py)""" + return self.resource_name + + @classmethod + def compose_resource_name(cls, key: str, version: str) -> str: + raise NotImplementedError("Subclasses must implement this method") diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/files.py b/services/api-server/src/simcore_service_api_server/models/schemas/files.py index 29cc9aacf0a..ebfee726adb 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/files.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/files.py @@ -1,64 +1,77 @@ +import datetime from mimetypes import guess_type -from pathlib import Path from typing import Annotated -from urllib.parse import quote as _quote -from urllib.parse import unquote as _unquote -from uuid import UUID, uuid3 +from uuid import UUID -import aiofiles -from fastapi import UploadFile from models_library.api_schemas_storage.storage_schemas import ETag from models_library.basic_types import SHA256Str -from models_library.projects_nodes_io import StorageFileID from pydantic import ( AnyHttpUrl, BaseModel, ConfigDict, Field, NonNegativeInt, - StringConstraints, - TypeAdapter, ValidationInfo, field_validator, ) -from servicelib.file_utils import create_sha256_checksum from .._utils_pydantic import UriSchema +from ..domain.files import File as DomainFile +from ..domain.files import FileName +from ._base import ApiServerInputSchema, ApiServerOutputSchema -_NAMESPACE_FILEID_KEY = UUID("aa154444-d22d-4290-bb15-df37dba87865") - -FileName = Annotated[str, StringConstraints(strip_whitespace=True)] - - -class ClientFile(BaseModel): +class UserFile(ApiServerInputSchema): """Represents a file stored on the client side""" - filename: FileName = Field(..., description="File name") - filesize: NonNegativeInt = Field(..., description="File size in bytes") - sha256_checksum: SHA256Str = Field(..., description="SHA256 checksum") + filename: Annotated[FileName, Field(..., description="File name")] + filesize: Annotated[NonNegativeInt, Field(..., description="File size in bytes")] + sha256_checksum: Annotated[SHA256Str, Field(..., description="SHA256 checksum")] + + def to_domain_model( + self, + file_id: UUID | None = None, + ) -> DomainFile: + return DomainFile( + id=( + file_id + if file_id + else DomainFile.create_id( + self.filesize, + self.filename, + datetime.datetime.now(datetime.UTC).isoformat(), + ) + ), + filename=self.filename, + checksum=self.sha256_checksum, + program_job_file_path=None, + ) -class File(BaseModel): +class File(ApiServerOutputSchema): """Represents a file stored on the server side i.e. a unique reference to a file in the cloud.""" - # WARNING: from pydantic import File as FileParam - # NOTE: see https://ant.apache.org/manual/Tasks/checksum.html - - id: UUID = Field(..., description="Resource identifier") - - filename: str = Field(..., description="Name of the file with extension") - content_type: str | None = Field( - default=None, - description="Guess of type content [EXPERIMENTAL]", - validate_default=True, - ) - sha256_checksum: SHA256Str | None = Field( - default=None, - description="SHA256 hash of the file's content", - alias="checksum", # alias for backwards compatibility + id: Annotated[UUID, Field(..., description="Resource identifier")] + filename: Annotated[str, Field(..., description="Name of the file with extension")] + content_type: Annotated[ + str | None, + Field( + default=None, + description="Guess of type content [EXPERIMENTAL]", + validate_default=True, + ), + ] = None + sha256_checksum: Annotated[ + SHA256Str | None, + Field( + default=None, + description="SHA256 hash of the file's content", + alias="checksum", # alias for backwards compatibility + ), + ] = None + e_tag: Annotated[ETag | None, Field(default=None, description="S3 entity tag")] = ( + None ) - e_tag: ETag | None = Field(default=None, description="S3 entity tag") model_config = ConfigDict( populate_by_name=True, @@ -91,76 +104,15 @@ def guess_content_type(cls, v, info: ValidationInfo): return v @classmethod - async def create_from_path(cls, path: Path) -> "File": - async with aiofiles.open(path, mode="rb") as file: - sha256check = await create_sha256_checksum(file) - - return cls( - id=cls.create_id(sha256check, path.name), - filename=path.name, - checksum=SHA256Str(sha256check), - ) - - @classmethod - async def create_from_file_link(cls, s3_object_path: str, e_tag: str) -> "File": - filename = Path(s3_object_path).name - return cls( - id=cls.create_id(e_tag, filename), - filename=filename, - e_tag=e_tag, - ) - - @classmethod - async def create_from_uploaded( - cls, file: UploadFile, *, file_size=None, created_at=None - ) -> "File": - sha256check = await create_sha256_checksum(file) - # WARNING: UploadFile wraps a stream and wil checkt its cursor position: file.file.tell() != 0 - # WARNING: await file.seek(0) might introduce race condition if not done carefuly - + def from_domain_model(cls, file: DomainFile) -> "File": return cls( - id=cls.create_id(sha256check or file_size, file.filename, created_at), - filename=file.filename or "Undefined", + id=file.id, + filename=file.filename, content_type=file.content_type, - checksum=SHA256Str(sha256check), - ) - - @classmethod - async def create_from_client_file( - cls, - client_file: ClientFile, - created_at: str, - ) -> "File": - return cls( - id=cls.create_id(client_file.filesize, client_file.filename, created_at), - filename=client_file.filename, - checksum=client_file.sha256_checksum, + sha256_checksum=file.sha256_checksum, + e_tag=file.e_tag, ) - @classmethod - async def create_from_quoted_storage_id(cls, quoted_storage_id: str) -> "File": - storage_file_id: StorageFileID = TypeAdapter(StorageFileID).validate_python( - _unquote(quoted_storage_id) - ) - _, fid, fname = Path(storage_file_id).parts - return cls(id=UUID(fid), filename=fname, checksum=None) - - @classmethod - def create_id(cls, *keys) -> UUID: - return uuid3(_NAMESPACE_FILEID_KEY, ":".join(map(str, keys))) - - @property - def storage_file_id(self) -> StorageFileID: - """Get the StorageFileId associated with this file""" - return TypeAdapter(StorageFileID).validate_python( - f"api/{self.id}/{self.filename}" - ) - - @property - def quoted_storage_file_id(self) -> str: - """Quoted version of the StorageFileId""" - return _quote(self.storage_file_id, safe="") - class UploadLinks(BaseModel): abort_upload: str diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py index b0616170501..19899f57ef6 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py @@ -1,21 +1,27 @@ import datetime import hashlib import logging +from collections.abc import Callable +from pathlib import Path from typing import Annotated, TypeAlias from uuid import UUID, uuid4 +from models_library.basic_types import SHA256Str from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.projects_state import RunningState +from models_library.services_types import FileName from pydantic import ( BaseModel, ConfigDict, Field, HttpUrl, + NonNegativeInt, PositiveInt, StrictBool, StrictFloat, StrictInt, + StringConstraints, TypeAdapter, ValidationError, ValidationInfo, @@ -24,14 +30,27 @@ from servicelib.logging_utils import LogLevelInt, LogMessageStr from starlette.datastructures import Headers -from ...models.schemas.files import File -from ...models.schemas.solvers import Solver +from ...models.schemas.files import File, UserFile from .._utils_pydantic import UriSchema from ..api_resources import ( RelativeResourceName, compose_resource_name, split_resource_name, ) +from ..basic_types import VersionStr +from ..domain.files import File as DomainFile +from ..domain.files import FileInProgramJobData +from ..schemas.files import UserFile +from ._base import ApiServerInputSchema + +# JOB SUB-RESOURCES ---------- +# +# - Wrappers for input/output values +# - Input/outputs are defined in service metadata +# - custom metadata +# +from .programs import Program, ProgramKeyId +from .solvers import Solver JobID: TypeAlias = UUID @@ -55,12 +74,42 @@ def _compute_keyword_arguments_checksum(kwargs: KeywordArguments): return hashlib.sha256(_dump_str.encode("utf-8")).hexdigest() -# JOB SUB-RESOURCES ---------- -# -# - Wrappers for input/output values -# - Input/outputs are defined in service metadata -# - custom metadata -# +class UserFileToProgramJob(ApiServerInputSchema): + filename: Annotated[FileName, Field(..., description="File name")] + filesize: Annotated[NonNegativeInt, Field(..., description="File size in bytes")] + sha256_checksum: Annotated[SHA256Str, Field(..., description="SHA256 checksum")] + program_key: Annotated[ProgramKeyId, Field(..., description="Program identifier")] + program_version: Annotated[VersionStr, Field(..., description="Program version")] + job_id: Annotated[JobID, Field(..., description="Job identifier")] + workspace_path: Annotated[ + Path, + StringConstraints(pattern=r"^workspace/.*"), + Field( + ..., + description="The file's relative path within the job's workspace directory. E.g. 'workspace/myfile.txt'", + ), + ] + + def to_domain_model(self, *, project_id: ProjectID, node_id: NodeID) -> DomainFile: + return DomainFile( + id=DomainFile.create_id( + self.filesize, + self.filename, + datetime.datetime.now(datetime.UTC).isoformat(), + ), + filename=self.filename, + checksum=self.sha256_checksum, + program_job_file_path=FileInProgramJobData( + project_id=project_id, + node_id=node_id, + workspace_path=self.workspace_path, + ), + ) + + +assert set(UserFile.model_fields.keys()).issubset( + set(UserFileToProgramJob.model_fields.keys()) +) # nosec class JobInputs(BaseModel): @@ -255,9 +304,11 @@ def create_now( ) @classmethod - def create_solver_job(cls, *, solver: Solver, inputs: JobInputs): + def create_job_from_solver_or_program( + cls, *, solver_or_program_name: str, inputs: JobInputs + ): return Job.create_now( - parent_name=solver.name, + parent_name=solver_or_program_name, inputs_checksum=inputs.compute_checksum(), ) @@ -280,6 +331,44 @@ def resource_name(self) -> str: return self.name +def get_url( + solver_or_program: Solver | Program, url_for: Callable[..., HttpUrl], job_id: JobID +) -> HttpUrl | None: + if isinstance(solver_or_program, Solver): + return url_for( + "get_job", + solver_key=solver_or_program.id, + version=solver_or_program.version, + job_id=job_id, + ) + return None + + +def get_runner_url( + solver_or_program: Solver | Program, url_for: Callable[..., HttpUrl] +) -> HttpUrl | None: + if isinstance(solver_or_program, Solver): + return url_for( + "get_solver_release", + solver_key=solver_or_program.id, + version=solver_or_program.version, + ) + return None + + +def get_outputs_url( + solver_or_program: Solver | Program, url_for: Callable[..., HttpUrl], job_id: JobID +) -> HttpUrl | None: + if isinstance(solver_or_program, Solver): + return url_for( + "get_job_outputs", + solver_key=solver_or_program.id, + version=solver_or_program.version, + job_id=job_id, + ) + return None + + PercentageInt: TypeAlias = Annotated[int, Field(ge=0, le=100)] diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/programs.py b/services/api-server/src/simcore_service_api_server/models/schemas/programs.py new file mode 100644 index 00000000000..25d1d1f5cf2 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/models/schemas/programs.py @@ -0,0 +1,61 @@ +from typing import Annotated + +from models_library.services import ServiceMetaDataPublished +from models_library.services_regex import DYNAMIC_SERVICE_KEY_RE +from pydantic import ConfigDict, StringConstraints +from simcore_service_api_server.models.schemas._base import ( + ApiServerOutputSchema, + BaseService, +) + +from ..api_resources import compose_resource_name +from ..basic_types import VersionStr + +# - API will add flexibility to identify solver resources using aliases. Analogously to docker images e.g. a/b == a/b:latest == a/b:2.3 +# +LATEST_VERSION = "latest" + + +# SOLVER ---------- +# +PROGRAM_RESOURCE_NAME_RE = r"^programs/([^\s/]+)/releases/([\d\.]+)$" + + +ProgramKeyId = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=DYNAMIC_SERVICE_KEY_RE) +] + + +class Program(BaseService, ApiServerOutputSchema): + """A released program with a specific version""" + + model_config = ConfigDict( + extra="ignore", + json_schema_extra={ + "example": { + "id": "simcore/services/dynamic/sim4life", + "version": "8.0.0", + "title": "Sim4life", + "description": "Simulation framework", + "maintainer": "info@itis.swiss", + "url": "https://api.osparc.io/v0/solvers/simcore%2Fservices%2Fdynamic%2Fsim4life/releases/8.0.0", + } + }, + ) + + @classmethod + def create_from_image(cls, image_meta: ServiceMetaDataPublished) -> "Program": + data = image_meta.model_dump( + include={"name", "key", "version", "description", "contact"}, + ) + return cls( + id=data.pop("key"), + version=data.pop("version"), + title=data.pop("name"), + url=None, + **data, + ) + + @classmethod + def compose_resource_name(cls, key: ProgramKeyId, version: VersionStr) -> str: + return compose_resource_name("programs", key, "releases", version) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py index f3be20211cd..6cae1156a7d 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py @@ -1,16 +1,12 @@ -import urllib.parse from typing import Annotated, Any, Literal -import packaging.version from models_library.basic_regex import PUBLIC_VARIABLE_NAME_RE from models_library.services import ServiceMetaDataPublished from models_library.services_regex import COMPUTATIONAL_SERVICE_KEY_RE -from packaging.version import Version -from pydantic import BaseModel, ConfigDict, Field, HttpUrl, StringConstraints +from pydantic import BaseModel, ConfigDict, Field, StringConstraints -from ...models._utils_pydantic import UriSchema +from ...models.schemas._base import BaseService from ..api_resources import compose_resource_name -from ..basic_types import VersionStr # NOTE: # - API does NOT impose prefix (simcore)/(services)/comp because does not know anything about registry deployed. This constraint @@ -36,27 +32,10 @@ ] -class Solver(BaseModel): +class Solver(BaseService): """A released solver with a specific version""" - id: SolverKeyId = Field(..., description="Solver identifier") - version: VersionStr = Field( - ..., - description="semantic version number of the node", - ) - - # Human readables Identifiers - title: str = Field(..., description="Human readable name") - description: str | None = None - maintainer: str - # TODO: consider released: Optional[datetime] required? - # TODO: consider version_aliases: list[str] = [] # remaining tags - - # Get links to other resources - url: Annotated[ - Annotated[HttpUrl, UriSchema()] | None, - Field(..., description="Link to get this resource"), - ] + maintainer: str = Field(..., description="Maintainer of the solver") model_config = ConfigDict( extra="ignore", @@ -77,7 +56,6 @@ def create_from_image(cls, image_meta: ServiceMetaDataPublished) -> "Solver": data = image_meta.model_dump( include={"name", "key", "version", "description", "contact"}, ) - return cls( id=data.pop("key"), version=data.pop("version"), @@ -87,29 +65,9 @@ def create_from_image(cls, image_meta: ServiceMetaDataPublished) -> "Solver": **data, ) - @property - def pep404_version(self) -> Version: - """Rich version type that can be used e.g. to compare""" - return packaging.version.parse(self.version) - - @property - def url_friendly_id(self) -> str: - """Use to pass id as parameter in urls""" - return urllib.parse.quote_plus(self.id) - - @property - def resource_name(self) -> str: - """Relative resource name""" - return self.compose_resource_name(self.id, self.version) - - @property - def name(self) -> str: - """API standards notation (see api_resources.py)""" - return self.resource_name - @classmethod - def compose_resource_name(cls, solver_key, solver_version) -> str: - return compose_resource_name("solvers", solver_key, "releases", solver_version) + def compose_resource_name(cls, key: str, version: str) -> str: + return compose_resource_name("solvers", key, "releases", version) PortKindStr = Literal["input", "output"] diff --git a/services/api-server/src/simcore_service_api_server/services_http/catalog.py b/services/api-server/src/simcore_service_api_server/services_http/catalog.py index b6607342724..d9a8b26adb4 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/catalog.py +++ b/services/api-server/src/simcore_service_api_server/services_http/catalog.py @@ -5,14 +5,14 @@ from dataclasses import dataclass from functools import partial from operator import attrgetter -from typing import Final +from typing import Final, Literal from fastapi import FastAPI, status from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished, ServiceType from models_library.users import UserID -from pydantic import ConfigDict, TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter from settings_library.catalog import CatalogSettings from settings_library.tracing import TracingSettings @@ -22,6 +22,7 @@ ) from ..exceptions.service_errors_utils import service_exception_mapper from ..models.basic_types import VersionStr +from ..models.schemas.programs import Program, ProgramKeyId from ..models.schemas.solvers import LATEST_VERSION, Solver, SolverKeyId, SolverPort from ..utils.client_base import BaseServiceClientApi, setup_client_instance @@ -64,6 +65,20 @@ def to_solver(self) -> Solver: **data, ) + def to_program(self) -> Program: + data = self.model_dump( + include={"name", "key", "version", "description", "contact", "owner"}, + ) + return Program( + id=data.pop("key"), + version=data.pop("version"), + title=data.pop("name"), + url=None, + **data, + ) + + +ServiceTypes = Literal["COMPUTATIONAL", "DYNAMIC"] # API CLASS --------------------------------------------- # @@ -96,13 +111,14 @@ class CatalogApi(BaseServiceClientApi): @_exception_mapper( http_status_map={status.HTTP_404_NOT_FOUND: ListSolversOrStudiesError} ) - async def list_solvers( + async def list_services( self, *, - user_id: UserID, - product_name: ProductName, - predicate: Callable[[Solver], bool] | None = None, - ) -> list[Solver]: + user_id: int, + product_name: str, + predicate: Callable[[TruncatedCatalogServiceOut], bool] | None = None, + type_filter: ServiceTypes, + ) -> list[TruncatedCatalogServiceOut]: response = await self.client.get( "/services", @@ -119,29 +135,17 @@ async def list_solvers( TruncatedCatalogServiceOutListAdapter, response, ) - solvers = [] - for service in services: - try: - if service.service_type == ServiceType.COMPUTATIONAL: - solver = service.to_solver() - if predicate is None or predicate(solver): - solvers.append(solver) - - except ValidationError as err: - # NOTE: For the moment, this is necessary because there are no guarantees - # at the image registry. Therefore we exclude and warn - # invalid items instead of returning error - _logger.warning( - "Skipping invalid service returned by catalog '%s': %s", - service.model_dump_json(), - err, - ) - return solvers - @_exception_mapper( - http_status_map={status.HTTP_404_NOT_FOUND: SolverOrStudyNotFoundError} - ) - async def get_service( + services = [ + service + for service in services + if service.service_type == ServiceType[type_filter] + ] + if predicate is not None: + services = [service for service in services if predicate(service)] + return services + + async def get_solver( self, *, user_id: UserID, @@ -149,6 +153,40 @@ async def get_service( version: VersionStr, product_name: ProductName, ) -> Solver: + service = await self._get_service( + user_id=user_id, name=name, version=version, product_name=product_name + ) + assert ( # nosec + service.service_type == ServiceType.COMPUTATIONAL + ), "Expected by SolverName regex" + + solver: Solver = service.to_solver() + return solver + + async def get_program( + self, + *, + user_id: int, + name: ProgramKeyId, + version: VersionStr, + product_name: str, + ) -> Program: + service = await self._get_service( + user_id=user_id, name=name, version=version, product_name=product_name + ) + assert ( # nosec + service.service_type == ServiceType.DYNAMIC + ), "Expected by ProgramName regex" + + program = service.to_program() + return program + + @_exception_mapper( + http_status_map={status.HTTP_404_NOT_FOUND: SolverOrStudyNotFoundError} + ) + async def _get_service( + self, *, user_id: int, name: SolverKeyId, version: VersionStr, product_name: str + ) -> TruncatedCatalogServiceOut: assert version != LATEST_VERSION # nosec @@ -167,12 +205,7 @@ async def get_service( ) = await asyncio.get_event_loop().run_in_executor( None, _parse_response, TruncatedCatalogServiceOutAdapter, response ) - assert ( # nosec - service.service_type == ServiceType.COMPUTATIONAL - ), "Expected by SolverName regex" - - solver: Solver = service.to_solver() - return solver + return service @_exception_mapper( http_status_map={status.HTTP_404_NOT_FOUND: SolverOrStudyNotFoundError} @@ -204,9 +237,13 @@ async def get_service_ports( async def list_latest_releases( self, *, user_id: UserID, product_name: ProductName ) -> list[Solver]: - solvers: list[Solver] = await self.list_solvers( - user_id=user_id, product_name=product_name + services = await self.list_services( + user_id=user_id, + product_name=product_name, + predicate=None, + type_filter="COMPUTATIONAL", ) + solvers = [service.to_solver() for service in services] latest_releases: dict[SolverKeyId, Solver] = {} for solver in solvers: @@ -216,22 +253,36 @@ async def list_latest_releases( return list(latest_releases.values()) - async def list_solver_releases( - self, *, user_id: UserID, solver_key: SolverKeyId, product_name: ProductName + async def list_service_releases( + self, + *, + user_id: int, + solver_key: SolverKeyId, + product_name: str, ) -> list[Solver]: def _this_solver(solver: Solver) -> bool: return solver.id == solver_key - releases: list[Solver] = await self.list_solvers( - user_id=user_id, predicate=_this_solver, product_name=product_name + services = await self.list_services( + user_id=user_id, + predicate=None, + product_name=product_name, + type_filter="COMPUTATIONAL", ) - return releases + solvers = [service.to_solver() for service in services] + return [solver for solver in solvers if _this_solver(solver)] async def get_latest_release( - self, *, user_id: UserID, solver_key: SolverKeyId, product_name: ProductName + self, + *, + user_id: int, + solver_key: SolverKeyId, + product_name: str, ) -> Solver: - releases = await self.list_solver_releases( - user_id=user_id, solver_key=solver_key, product_name=product_name + releases = await self.list_service_releases( + user_id=user_id, + solver_key=solver_key, + product_name=product_name, ) # raises IndexError if None diff --git a/services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py b/services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py index 86c3f8bc53d..81ea1683a78 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py +++ b/services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py @@ -1,6 +1,6 @@ """ - Helper functions to convert models used in - services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +Helper functions to convert models used in +services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py """ import urllib.parse @@ -13,10 +13,10 @@ from models_library.api_schemas_webserver.projects import ProjectCreateNew, ProjectGet from models_library.api_schemas_webserver.projects_ui import StudyUI from models_library.basic_types import KeyIDStr +from models_library.projects import Project from models_library.projects_nodes import InputID -from pydantic import TypeAdapter +from pydantic import HttpUrl, TypeAdapter -from ..models.basic_types import VersionStr from ..models.domain.projects import InputTypes, Node, SimCoreFileLink from ..models.schemas.files import File from ..models.schemas.jobs import ( @@ -25,8 +25,12 @@ JobInputs, JobStatus, PercentageInt, + get_outputs_url, + get_runner_url, + get_url, ) -from ..models.schemas.solvers import Solver, SolverKeyId +from ..models.schemas.programs import Program +from ..models.schemas.solvers import Solver from .director_v2 import ComputationTaskGet # UTILS ------ @@ -115,7 +119,7 @@ def get_node_id(project_id, solver_id) -> str: def create_new_project_for_job( - solver: Solver, job: Job, inputs: JobInputs + solver_or_program: Solver | Program, job: Job, inputs: JobInputs ) -> ProjectCreateNew: """ Creates a project for a solver's job @@ -131,7 +135,7 @@ def create_new_project_for_job( raises ValidationError """ project_id = job.id - solver_id = get_node_id(project_id, solver.id) + solver_id = get_node_id(project_id, solver_or_program.id) # map Job inputs with solveri nputs # TODO: ArgumentType -> InputTypes dispatcher and reversed @@ -140,9 +144,9 @@ def create_new_project_for_job( ) solver_service = Node( - key=solver.id, - version=solver.version, - label=solver.title, + key=solver_or_program.id, + version=solver_or_program.version, + label=solver_or_program.title, inputs=solver_inputs, inputs_units={}, ) @@ -176,10 +180,10 @@ def create_new_project_for_job( def create_job_from_project( - solver_key: SolverKeyId, - solver_version: VersionStr, - project: ProjectGet, - url_for: Callable, + *, + solver_or_program: Solver | Program, + project: ProjectGet | Project, + url_for: Callable[..., HttpUrl], ) -> Job: """ Given a project, creates a job @@ -190,8 +194,8 @@ def create_job_from_project( raise ValidationError """ assert len(project.workbench) == 1 # nosec - assert solver_version in project.name # nosec - assert urllib.parse.quote_plus(solver_key) in project.name # nosec + assert solver_or_program.version in project.name # nosec + assert urllib.parse.quote_plus(solver_or_program.id) in project.name # nosec # get solver node node_id = next(iter(project.workbench.keys())) @@ -201,7 +205,7 @@ def create_job_from_project( ) # create solver's job - solver_name = Solver.compose_resource_name(solver_key, solver_version) + solver_or_program_name = solver_or_program.resource_name job_id = project.uuid @@ -210,30 +214,16 @@ def create_job_from_project( name=project.name, inputs_checksum=job_inputs.compute_checksum(), created_at=project.creation_date, # type: ignore[arg-type] - runner_name=solver_name, - url=url_for( - "get_job", - solver_key=solver_key, - version=solver_version, - job_id=job_id, + runner_name=solver_or_program_name, + url=get_url( + solver_or_program=solver_or_program, url_for=url_for, job_id=job_id ), - runner_url=url_for( - "get_solver_release", - solver_key=solver_key, - version=solver_version, - ), - outputs_url=url_for( - "get_job_outputs", - solver_key=solver_key, - version=solver_version, - job_id=job_id, + runner_url=get_runner_url(solver_or_program=solver_or_program, url_for=url_for), + outputs_url=get_outputs_url( + solver_or_program=solver_or_program, url_for=url_for, job_id=job_id ), ) - assert all( - getattr(job, f) for f in job.model_fields.keys() if f.endswith("url") - ) # nosec - return job diff --git a/services/api-server/src/simcore_service_api_server/services_http/storage.py b/services/api-server/src/simcore_service_api_server/services_http/storage.py index 52d3c8e8ddb..31690297612 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/storage.py +++ b/services/api-server/src/simcore_service_api_server/services_http/storage.py @@ -8,7 +8,9 @@ from fastapi import FastAPI from fastapi.encoders import jsonable_encoder -from models_library.api_schemas_storage.storage_schemas import FileMetaDataArray +from models_library.api_schemas_storage.storage_schemas import ( + FileMetaDataArray, +) from models_library.api_schemas_storage.storage_schemas import ( FileMetaDataGet as StorageFileMetaData, ) @@ -24,16 +26,17 @@ from ..core.settings import StorageSettings from ..exceptions.service_errors_utils import service_exception_mapper -from ..models.schemas.files import File +from ..models.domain.files import File from ..utils.client_base import BaseServiceClientApi, setup_client_instance _logger = logging.getLogger(__name__) _exception_mapper = partial(service_exception_mapper, service_name="Storage") -_FILE_ID_PATTERN = re.compile(r"^api\/(?P[\w-]+)\/(?P.+)$") AccessRight = Literal["read", "write"] +_FILE_ID_PATTERN = re.compile(r"^api\/(?P[\w-]+)\/(?P.+)$") + def to_file_api_model(stored_file_meta: StorageFileMetaData) -> File: # extracts fields from api/{file_id}/{filename} @@ -47,7 +50,8 @@ def to_file_api_model(stored_file_meta: StorageFileMetaData) -> File: return File( id=file_id, # type: ignore filename=filename, - content_type=guess_type(filename)[0] or "application/octet-stream", + content_type=guess_type(stored_file_meta.file_name)[0] + or "application/octet-stream", e_tag=stored_file_meta.entity_tag, checksum=stored_file_meta.sha256_checksum, ) @@ -105,7 +109,7 @@ async def search_owned_files( { "kind": "owned", "user_id": f"{user_id}", - "startswith": None if file_id is None else f"api/{file_id}", + "startswith": "api/" if file_id is None else f"api/{file_id}", "sha256_checksum": sha256_checksum, "limit": limit, "offset": offset, diff --git a/services/api-server/src/simcore_service_api_server/services_http/study_job_models_converters.py b/services/api-server/src/simcore_service_api_server/services_http/study_job_models_converters.py index 1ab18a85c0d..99bb5a59ae9 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/study_job_models_converters.py +++ b/services/api-server/src/simcore_service_api_server/services_http/study_job_models_converters.py @@ -1,7 +1,8 @@ """ - Helper functions to convert models used in - services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +Helper functions to convert models used in +services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py """ + from typing import Any, NamedTuple from uuid import UUID @@ -15,8 +16,8 @@ from models_library.projects_nodes_io import LinkToFileTypes, NodeID, SimcoreS3FileID from pydantic import TypeAdapter +from ..models.domain.files import File from ..models.domain.projects import InputTypes, SimCoreFileLink -from ..models.schemas.files import File from ..models.schemas.jobs import Job, JobInputs, JobOutputs from ..models.schemas.studies import Study, StudyID from .storage import StorageApi, to_file_api_model diff --git a/services/api-server/tests/unit/test__fastapi.py b/services/api-server/tests/unit/test__fastapi.py index 4eaddee4437..739d67cedb8 100644 --- a/services/api-server/tests/unit/test__fastapi.py +++ b/services/api-server/tests/unit/test__fastapi.py @@ -26,10 +26,10 @@ from fastapi import APIRouter, FastAPI, status from fastapi.testclient import TestClient from simcore_service_api_server._meta import API_VTAG +from simcore_service_api_server.models.schemas.programs import VersionStr from simcore_service_api_server.models.schemas.solvers import ( Solver, SolverKeyId, - VersionStr, ) diff --git a/services/api-server/tests/unit/test_api_files.py b/services/api-server/tests/unit/test_api_files.py index 8f44f00371e..78a150c6c61 100644 --- a/services/api-server/tests/unit/test_api_files.py +++ b/services/api-server/tests/unit/test_api_files.py @@ -31,11 +31,11 @@ ) from respx import MockRouter from simcore_service_api_server._meta import API_VTAG +from simcore_service_api_server.models.domain.files import File from simcore_service_api_server.models.pagination import Page from simcore_service_api_server.models.schemas.files import ( - ClientFile, ClientFileUploadData, - File, + UserFile, ) _FAKER = Faker() @@ -66,8 +66,8 @@ def file(cls) -> File: ) @classmethod - def client_file(cls) -> ClientFile: - return TypeAdapter(ClientFile).validate_python( + def client_file(cls) -> UserFile: + return TypeAdapter(UserFile).validate_python( { "filename": cls._file_name, "filesize": cls._file_size, diff --git a/services/api-server/tests/unit/test_models_schemas_files.py b/services/api-server/tests/unit/test_models_schemas_files.py index bd7cfddfaf8..30d4ead053c 100644 --- a/services/api-server/tests/unit/test_models_schemas_files.py +++ b/services/api-server/tests/unit/test_models_schemas_files.py @@ -17,7 +17,7 @@ from models_library.basic_types import SHA256Str from models_library.projects_nodes_io import StorageFileID from pydantic import TypeAdapter, ValidationError -from simcore_service_api_server.models.schemas.files import File +from simcore_service_api_server.models.domain.files import File from simcore_service_api_server.services_http.storage import to_file_api_model FILE_CONTENT = "This is a test" diff --git a/services/api-server/tests/unit/test_models_schemas_jobs.py b/services/api-server/tests/unit/test_models_schemas_jobs.py index 73486cf0f55..f11f8a494db 100644 --- a/services/api-server/tests/unit/test_models_schemas_jobs.py +++ b/services/api-server/tests/unit/test_models_schemas_jobs.py @@ -58,7 +58,7 @@ def _deepcopy_and_shuffle(src): ), f"{inputs1}!={inputs2}" -def test_job_resouce_names_has_associated_url(app: FastAPI): +def test_job_resource_names_has_associated_url(app: FastAPI): solver_key = "z43/name with spaces/isolve" solver_version = "1.0.3" job_id = uuid4() diff --git a/services/api-server/tests/unit/test_models_schemas_solvers.py b/services/api-server/tests/unit/test_models_schemas_solvers.py index 0e81e60aa86..121520ea646 100644 --- a/services/api-server/tests/unit/test_models_schemas_solvers.py +++ b/services/api-server/tests/unit/test_models_schemas_solvers.py @@ -5,7 +5,8 @@ from operator import attrgetter from faker import Faker -from simcore_service_api_server.models.schemas.solvers import Solver, Version +from packaging.version import Version +from simcore_service_api_server.models.schemas.solvers import Solver def test_solvers_sorting_by_name_and_version(faker: Faker): diff --git a/services/api-server/tests/unit/test_services_solver_job_models_converters.py b/services/api-server/tests/unit/test_services_solver_job_models_converters.py index 40d60359477..1e926f09c86 100644 --- a/services/api-server/tests/unit/test_services_solver_job_models_converters.py +++ b/services/api-server/tests/unit/test_services_solver_job_models_converters.py @@ -6,7 +6,7 @@ from faker import Faker from models_library.projects import Project from models_library.projects_nodes import InputsDict, InputTypes, SimCoreFileLink -from pydantic import RootModel, TypeAdapter, create_model +from pydantic import HttpUrl, RootModel, TypeAdapter, create_model from simcore_service_api_server.models.schemas.files import File from simcore_service_api_server.models.schemas.jobs import ArgumentTypes, Job, JobInputs from simcore_service_api_server.models.schemas.solvers import Solver @@ -48,7 +48,9 @@ def test_create_project_model_for_job(faker: Faker): print(inputs.model_dump_json(indent=2)) - job = Job.create_solver_job(solver=solver, inputs=inputs) + job = Job.create_job_from_solver_or_program( + solver_or_program_name=solver.name, inputs=inputs + ) # body of create project! createproject_body = create_new_project_for_job(solver, job, inputs) @@ -197,11 +199,20 @@ def test_create_job_from_project(faker: Faker): solver_key = "simcore/services/comp/itis/sleeper" solver_version = "2.0.2" - def fake_url_for(*args, **kwargs): - return faker.url() + def fake_url_for(*args, **kwargs) -> HttpUrl: + return HttpUrl(faker.url()) + + solver = Solver( + id=solver_key, + version=solver_version, + title=faker.text(max_nb_chars=20), + maintainer=faker.name(), + description=faker.text(max_nb_chars=100), + url=None, + ) job = create_job_from_project( - solver_key, solver_version, project, url_for=fake_url_for + solver_or_program=solver, project=project, url_for=fake_url_for ) assert job.id == project.uuid