From 61daf3ceb27d064fe62445296518758452cd9deb Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 26 Mar 2025 11:31:10 +0100 Subject: [PATCH 01/39] minor --- .../src/simcore_service_api_server/services_http/webserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/services_http/webserver.py b/services/api-server/src/simcore_service_api_server/services_http/webserver.py index 3f9d93c7455..c86ca6e1a4f 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services_http/webserver.py @@ -1,6 +1,5 @@ # pylint: disable=too-many-public-methods -import json import logging import urllib.parse from collections.abc import Mapping @@ -10,6 +9,7 @@ from uuid import UUID import httpx +from common_library.json_serialization import json_dumps from cryptography import fernet from fastapi import FastAPI, status from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet @@ -194,7 +194,7 @@ async def _page_projects( optional: dict[str, Any] = {} if search_by_project_name is not None: filters_dict = {"search_by_project_name": search_by_project_name} - filters_json = json.dumps(filters_dict) + filters_json = json_dumps(filters_dict) optional["filters"] = filters_json with service_exception_handler( From 55071675a23a25745fa1e3c30dda669b89f0a2c6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 26 Mar 2025 11:48:45 +0100 Subject: [PATCH 02/39] table --- .../models/projects_to_jobs.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py new file mode 100644 index 00000000000..ff0a77c04de --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py @@ -0,0 +1,40 @@ +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from ._common import RefActions +from .base import metadata +from .projects import projects + +projects_to_jobs = sa.Table( + "projects_to_jobs", + metadata, + sa.Column( + "id", + sa.BigInteger, + primary_key=True, + autoincrement=True, + doc="Identifier index", + ), + sa.Column( + "project_uuid", + UUID(as_uuid=True), + sa.ForeignKey( + projects.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_projects_to_jobs_project_uuid", + ), + nullable=False, + doc="Foreign key to projects.uuid", + ), + sa.Column( + "job_name", + sa.String, + nullable=False, + doc="Identifier for the job associated with the project", + ), + sa.UniqueConstraint( + "project_uuid", "job_name", name="uq_projects_to_jobs_project_uuid_job_name" + ), + comment="Maps projects.uuid to job_name", +) From 4b8d6dd9f5048f63346f66e24872a9d9cb05d93d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 26 Mar 2025 11:57:05 +0100 Subject: [PATCH 03/39] new table --- .../fbc04f6860cd_new_projects_to_job_map.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/fbc04f6860cd_new_projects_to_job_map.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/fbc04f6860cd_new_projects_to_job_map.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/fbc04f6860cd_new_projects_to_job_map.py new file mode 100644 index 00000000000..9b2e958122e --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/fbc04f6860cd_new_projects_to_job_map.py @@ -0,0 +1,46 @@ +"""new projects to job map + +Revision ID: fbc04f6860cd +Revises: 8403acca8759 +Create Date: 2025-03-26 10:49:21.206239+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "fbc04f6860cd" +down_revision = "8403acca8759" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "projects_to_jobs", + sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("project_uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("job_name", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["project_uuid"], + ["projects.uuid"], + name="fk_projects_to_jobs_project_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "project_uuid", "job_name", name="uq_projects_to_jobs_project_uuid_job_name" + ), + comment="Maps projects.uuid to job_name", + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("projects_to_jobs") + # ### end Alembic commands ### From f6f0c416a46b89b05a3ba4de4548cd48ddcd2d9e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 26 Mar 2025 12:24:06 +0100 Subject: [PATCH 04/39] adds tests --- .../tests/test_models_projects_to_jobs.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 packages/postgres-database/tests/test_models_projects_to_jobs.py diff --git a/packages/postgres-database/tests/test_models_projects_to_jobs.py b/packages/postgres-database/tests/test_models_projects_to_jobs.py new file mode 100644 index 00000000000..57a66d09968 --- /dev/null +++ b/packages/postgres-database/tests/test_models_projects_to_jobs.py @@ -0,0 +1,83 @@ +import sqlalchemy as sa +import sqlalchemy.engine +from faker import Faker +from pytest_simcore.helpers.faker_factories import random_project +from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs +from sqlalchemy.dialects.postgresql import insert + + +def populate_projects_to_jobs(connection): + """ + Populates the projects_to_jobs table by analyzing the projects table. + + + NOTE: tested here but will be used in migration script + """ + query = sa.text( + """ + INSERT INTO projects_to_jobs (project_uuid, job_name, job_info) + SELECT + uuid AS project_uuid, + regexp_replace(name, '^.*jobs/([^/]+)$', '\\1') AS job_name, + FROM projects + WHERE name ~* '^solvers/.+/jobs/.+$' OR name ~* '^studies/.+/jobs/.+$'; + """ + ) + connection.execute(query) + + +def test_populate_projects_to_jobs( + pg_sa_engine: sqlalchemy.engine.Engine, faker: Faker +): + + sample_projects = [ + random_project( + faker, + uuid="cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + name="solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + description=( + "Study associated to solver job:" + """{ + "id": "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + "name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c2c-85fd-0d951d7dcd5a", + "inputs_checksum": "015ba4cd5cf00c511a8217deb65c242e3b15dc6ae4b1ecf94982d693887d9e8a", + "created_at": "2025-01-27T13:12:58.676564Z" + } + """ + ), + ), + random_project( + faker, + uuid="bf204942-007b-11ef-befd-0242ac114f07", + name="studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07", + description="Valid project 2", + ), + random_project( + faker, + uuid="33333333-3333-3333-3333-333333333333", + name="invalid/project/name", + description="Invalid project", + ), + ] + + # Insert sample projects into the projects table + with pg_sa_engine.connect() as conn: + conn.execute(insert(projects).values(sample_projects)) + + # Run the populate_projects_to_jobs function + populate_projects_to_jobs(conn) + + # Query the projects_to_jobs table + result = conn.execute(sa.select(projects_to_jobs)).fetchall() + + # Assert only valid projects are added + assert len(result) == 2 + assert { + "project_uuid": "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + "job_name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + } in result + assert { + "project_uuid": "bf204942-007b-11ef-befd-0242ac114f07", + "job_name": "studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07", + } in result From 4c7c602bf1602f0a7fe3c4342881d2021588a471 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 26 Mar 2025 12:58:52 +0100 Subject: [PATCH 05/39] unique --- .../fbc04f6860cd_new_projects_to_job_map.py | 46 ------------------- .../models/projects_to_jobs.py | 9 ++-- 2 files changed, 3 insertions(+), 52 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/fbc04f6860cd_new_projects_to_job_map.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/fbc04f6860cd_new_projects_to_job_map.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/fbc04f6860cd_new_projects_to_job_map.py deleted file mode 100644 index 9b2e958122e..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/fbc04f6860cd_new_projects_to_job_map.py +++ /dev/null @@ -1,46 +0,0 @@ -"""new projects to job map - -Revision ID: fbc04f6860cd -Revises: 8403acca8759 -Create Date: 2025-03-26 10:49:21.206239+00:00 - -""" - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = "fbc04f6860cd" -down_revision = "8403acca8759" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "projects_to_jobs", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), - sa.Column("project_uuid", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("job_name", sa.String(), nullable=False), - sa.ForeignKeyConstraint( - ["project_uuid"], - ["projects.uuid"], - name="fk_projects_to_jobs_project_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint( - "project_uuid", "job_name", name="uq_projects_to_jobs_project_uuid_job_name" - ), - comment="Maps projects.uuid to job_name", - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("projects_to_jobs") - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py index ff0a77c04de..7a2d0052a5f 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py @@ -1,5 +1,4 @@ import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import UUID from ._common import RefActions from .base import metadata @@ -17,7 +16,7 @@ ), sa.Column( "project_uuid", - UUID(as_uuid=True), + sa.String, sa.ForeignKey( projects.c.uuid, onupdate=RefActions.CASCADE, @@ -25,16 +24,14 @@ name="fk_projects_to_jobs_project_uuid", ), nullable=False, + unique=True, doc="Foreign key to projects.uuid", ), sa.Column( "job_name", sa.String, nullable=False, + unique=True, doc="Identifier for the job associated with the project", ), - sa.UniqueConstraint( - "project_uuid", "job_name", name="uq_projects_to_jobs_project_uuid_job_name" - ), - comment="Maps projects.uuid to job_name", ) From 90ea8fb20b9866118dfd0eafa1ca339b553d8797 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 26 Mar 2025 13:03:31 +0100 Subject: [PATCH 06/39] cleanup --- .../48604dfdc5f4_new_projects_to_job_map.py | 56 +++++++++++++++++++ .../tests/test_models_projects_to_jobs.py | 22 +++----- 2 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py new file mode 100644 index 00000000000..5345a9fbbfa --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py @@ -0,0 +1,56 @@ +"""new projects to job map + +Revision ID: 48604dfdc5f4 +Revises: 8403acca8759 +Create Date: 2025-03-26 12:00:14.763439+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "48604dfdc5f4" +down_revision = "8403acca8759" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "projects_to_jobs", + sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("project_uuid", sa.String(), nullable=False), + sa.Column("job_name", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["project_uuid"], + ["projects.uuid"], + name="fk_projects_to_jobs_project_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("job_name"), + sa.UniqueConstraint("project_uuid"), + ) + + # Populate the new table + op.execute( + sa.text( + r""" +INSERT INTO projects_to_jobs (project_uuid, job_name) +SELECT + uuid AS project_uuid, + regexp_replace(name, '\s+', '', 'g') AS job_name -- no spaces +FROM projects +WHERE name ~* '^solvers/.+/jobs/.+$' OR name ~* '^studies/.+/jobs/.+$'; + """ + ) + ) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("projects_to_jobs") + # ### end Alembic commands ### diff --git a/packages/postgres-database/tests/test_models_projects_to_jobs.py b/packages/postgres-database/tests/test_models_projects_to_jobs.py index 57a66d09968..519d702a904 100644 --- a/packages/postgres-database/tests/test_models_projects_to_jobs.py +++ b/packages/postgres-database/tests/test_models_projects_to_jobs.py @@ -8,20 +8,16 @@ def populate_projects_to_jobs(connection): - """ - Populates the projects_to_jobs table by analyzing the projects table. - - - NOTE: tested here but will be used in migration script - """ + # Populates the projects_to_jobs table by analyzing the projects table. + # NOTE: tested here but will be used in migration script query = sa.text( - """ - INSERT INTO projects_to_jobs (project_uuid, job_name, job_info) - SELECT - uuid AS project_uuid, - regexp_replace(name, '^.*jobs/([^/]+)$', '\\1') AS job_name, - FROM projects - WHERE name ~* '^solvers/.+/jobs/.+$' OR name ~* '^studies/.+/jobs/.+$'; + r""" +INSERT INTO projects_to_jobs (project_uuid, job_name) +SELECT + uuid AS project_uuid, + regexp_replace(name, '\s+', '', 'g') AS job_name -- no spaces +FROM projects +WHERE name ~* '^solvers/.+/jobs/.+$' OR name ~* '^studies/.+/jobs/.+$'; """ ) connection.execute(query) From 5b6334b91dc8c19b989e082acdc4458669ddc608 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 26 Mar 2025 13:17:48 +0100 Subject: [PATCH 07/39] modify test to emulate migration --- .../tests/test_models_projects_to_jobs.py | 122 +++++++++++------- 1 file changed, 74 insertions(+), 48 deletions(-) diff --git a/packages/postgres-database/tests/test_models_projects_to_jobs.py b/packages/postgres-database/tests/test_models_projects_to_jobs.py index 519d702a904..3a5876c9795 100644 --- a/packages/postgres-database/tests/test_models_projects_to_jobs.py +++ b/packages/postgres-database/tests/test_models_projects_to_jobs.py @@ -1,69 +1,95 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +from collections.abc import Iterator + +import pytest +import simcore_postgres_database.cli import sqlalchemy as sa import sqlalchemy.engine from faker import Faker +from pytest_simcore.helpers import postgres_tools from pytest_simcore.helpers.faker_factories import random_project from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs from sqlalchemy.dialects.postgresql import insert -def populate_projects_to_jobs(connection): - # Populates the projects_to_jobs table by analyzing the projects table. - # NOTE: tested here but will be used in migration script - query = sa.text( - r""" -INSERT INTO projects_to_jobs (project_uuid, job_name) -SELECT - uuid AS project_uuid, - regexp_replace(name, '\s+', '', 'g') AS job_name -- no spaces -FROM projects -WHERE name ~* '^solvers/.+/jobs/.+$' OR name ~* '^studies/.+/jobs/.+$'; - """ +@pytest.fixture +def sync_engine( + sync_engine: sqlalchemy.engine.Engine, db_metadata: sa.MetaData +) -> Iterator[sqlalchemy.engine.Engine]: + # EXTENDS sync_engine fixture to include cleanup and parare migration + + # cleanup tables + db_metadata.drop_all(sync_engine) + + # prepare migration upgrade + assert simcore_postgres_database.cli.discover.callback + assert simcore_postgres_database.cli.upgrade.callback + + dsn = sync_engine.url + simcore_postgres_database.cli.discover.callback( + user=dsn.username, + password=dsn.password, + host=dsn.host, + database=dsn.database, + port=dsn.port, ) - connection.execute(query) + yield sync_engine -def test_populate_projects_to_jobs( - pg_sa_engine: sqlalchemy.engine.Engine, faker: Faker + # cleanup tables + postgres_tools.force_drop_all_tables(sync_engine) + + +def test_populate_projects_to_jobs_during_migration( + sync_engine: sqlalchemy.engine.Engine, faker: Faker ): + assert simcore_postgres_database.cli.discover.callback + assert simcore_postgres_database.cli.upgrade.callback + + # UPGRADE just one before 48604dfdc5f4_new_projects_to_job_map.py + simcore_postgres_database.cli.upgrade.callback("8403acca8759") - sample_projects = [ - random_project( - faker, - uuid="cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", - name="solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", - description=( - "Study associated to solver job:" - """{ - "id": "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", - "name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c2c-85fd-0d951d7dcd5a", - "inputs_checksum": "015ba4cd5cf00c511a8217deb65c242e3b15dc6ae4b1ecf94982d693887d9e8a", - "created_at": "2025-01-27T13:12:58.676564Z" - } - """ + with sync_engine.connect() as conn: + sample_projects = [ + random_project( + faker, + uuid="cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + name="solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + description=( + "Study associated to solver job:" + """{ + "id": "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + "name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c2c-85fd-0d951d7dcd5a", + "inputs_checksum": "015ba4cd5cf00c511a8217deb65c242e3b15dc6ae4b1ecf94982d693887d9e8a", + "created_at": "2025-01-27T13:12:58.676564Z" + } + """ + ), + ), + random_project( + faker, + uuid="bf204942-007b-11ef-befd-0242ac114f07", + name="studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07", + description="Valid project 2", + ), + random_project( + faker, + uuid="33333333-3333-3333-3333-333333333333", + name="invalid/project/name", + description="Invalid project", ), - ), - random_project( - faker, - uuid="bf204942-007b-11ef-befd-0242ac114f07", - name="studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07", - description="Valid project 2", - ), - random_project( - faker, - uuid="33333333-3333-3333-3333-333333333333", - name="invalid/project/name", - description="Invalid project", - ), - ] - - # Insert sample projects into the projects table - with pg_sa_engine.connect() as conn: + ] conn.execute(insert(projects).values(sample_projects)) - # Run the populate_projects_to_jobs function - populate_projects_to_jobs(conn) + # Run upgrade to head! to populate + simcore_postgres_database.cli.upgrade.callback("head") + with sync_engine.connect() as conn: # Query the projects_to_jobs table result = conn.execute(sa.select(projects_to_jobs)).fetchall() From 4d52f26b47f9af2d221f68a24f9767dd75d4d5bf Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 26 Mar 2025 13:43:43 +0100 Subject: [PATCH 08/39] cleanup --- .../tests/test_models_projects_to_jobs.py | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/packages/postgres-database/tests/test_models_projects_to_jobs.py b/packages/postgres-database/tests/test_models_projects_to_jobs.py index 3a5876c9795..bff9bac5247 100644 --- a/packages/postgres-database/tests/test_models_projects_to_jobs.py +++ b/packages/postgres-database/tests/test_models_projects_to_jobs.py @@ -9,12 +9,13 @@ import simcore_postgres_database.cli import sqlalchemy as sa import sqlalchemy.engine +import sqlalchemy.exc from faker import Faker from pytest_simcore.helpers import postgres_tools -from pytest_simcore.helpers.faker_factories import random_project +from pytest_simcore.helpers.faker_factories import random_project, random_user from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs -from sqlalchemy.dialects.postgresql import insert +from simcore_postgres_database.models.users import users @pytest.fixture @@ -55,7 +56,24 @@ def test_populate_projects_to_jobs_during_migration( simcore_postgres_database.cli.upgrade.callback("8403acca8759") with sync_engine.connect() as conn: - sample_projects = [ + + # Ensure the projects_to_jobs table does NOT exist yet + with pytest.raises(sqlalchemy.exc.ProgrammingError) as exc_info: + conn.execute( + sa.select(sa.func.count()).select_from(projects_to_jobs) + ).scalar() + assert "psycopg2.errors.UndefinedTable" in f"{exc_info.value}" + + # INSERT data (emulates data in-place) + user_data = random_user( + faker, name="test_populate_projects_to_jobs_during_migration" + ) + user_id = conn.execute( + sa.insert(users).values(user_data).returning(users.c.id) + ).scalar() + + SPACES = " " * 3 + projects_data = [ random_project( faker, uuid="cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", @@ -70,36 +88,42 @@ def test_populate_projects_to_jobs_during_migration( } """ ), + prj_owner=user_id, ), random_project( faker, uuid="bf204942-007b-11ef-befd-0242ac114f07", - name="studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07", + name=f"studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07{SPACES}", description="Valid project 2", + prj_owner=user_id, ), random_project( faker, uuid="33333333-3333-3333-3333-333333333333", name="invalid/project/name", description="Invalid project", + prj_owner=user_id, ), ] - conn.execute(insert(projects).values(sample_projects)) + for prj in projects_data: + conn.execute(sa.insert(projects).values(prj)) - # Run upgrade to head! to populate + # MIGRATE UPGRADE: this should populate simcore_postgres_database.cli.upgrade.callback("head") with sync_engine.connect() as conn: # Query the projects_to_jobs table - result = conn.execute(sa.select(projects_to_jobs)).fetchall() + result = conn.execute( + sa.select(projects_to_jobs.c.project_uuid, projects_to_jobs.c.job_name) + ).fetchall() # Assert only valid projects are added assert len(result) == 2 - assert { - "project_uuid": "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", - "job_name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", - } in result - assert { - "project_uuid": "bf204942-007b-11ef-befd-0242ac114f07", - "job_name": "studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07", - } in result + assert ( + "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + ) in result + assert ( + "bf204942-007b-11ef-befd-0242ac114f07", + "studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07", + ) in result From 20116b065607b6cf4cb8650e9f98e0b1c1fc944b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 26 Mar 2025 13:48:52 +0100 Subject: [PATCH 09/39] cleanup --- .../postgres-database/tests/test_models_projects_to_jobs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/postgres-database/tests/test_models_projects_to_jobs.py b/packages/postgres-database/tests/test_models_projects_to_jobs.py index bff9bac5247..be069d77403 100644 --- a/packages/postgres-database/tests/test_models_projects_to_jobs.py +++ b/packages/postgres-database/tests/test_models_projects_to_jobs.py @@ -68,9 +68,9 @@ def test_populate_projects_to_jobs_during_migration( user_data = random_user( faker, name="test_populate_projects_to_jobs_during_migration" ) - user_id = conn.execute( - sa.insert(users).values(user_data).returning(users.c.id) - ).scalar() + stmt = sa.insert(users).values(**user_data).returning(users.c.id) + result = conn.execute(stmt) + user_id = result.scalar() SPACES = " " * 3 projects_data = [ From 2c95ab46b39399cc3360a0179b8a9ead1130ad50 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 27 Mar 2025 17:10:52 +0100 Subject: [PATCH 10/39] pass parent resource in the header --- .../src/servicelib/common_headers.py | 9 +++++--- .../api/routes/solvers_jobs.py | 1 + .../api/routes/studies.py | 1 + .../api/routes/studies_jobs.py | 3 +++ .../models/api_resources.py | 21 +++++++++++++++++++ .../services_http/webserver.py | 18 +++++++++++----- .../tests/unit/test_models_api_resources.py | 19 +++++++++++++++++ 7 files changed, 64 insertions(+), 8 deletions(-) diff --git a/packages/service-library/src/servicelib/common_headers.py b/packages/service-library/src/servicelib/common_headers.py index 543ef593fe5..0dc279b8956 100644 --- a/packages/service-library/src/servicelib/common_headers.py +++ b/packages/service-library/src/servicelib/common_headers.py @@ -1,9 +1,12 @@ from typing import Final +UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE: Final[str] = "undefined" X_DYNAMIC_SIDECAR_REQUEST_DNS: Final[str] = "X-Dynamic-Sidecar-Request-DNS" X_DYNAMIC_SIDECAR_REQUEST_SCHEME: Final[str] = "X-Dynamic-Sidecar-Request-Scheme" X_FORWARDED_PROTO: Final[str] = "X-Forwarded-Proto" -X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent" -X_SIMCORE_PARENT_PROJECT_UUID: Final[str] = "X-Simcore-Parent-Project-Uuid" +X_SIMCORE_API_JOB_PARENT_RESOURCE_NAME: Final[str] = ( + "X-Simcore-Api-Job-Parent-Resource-Name" +) X_SIMCORE_PARENT_NODE_ID: Final[str] = "X-Simcore-Parent-Node-Id" -UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE: Final[str] = "undefined" +X_SIMCORE_PARENT_PROJECT_UUID: Final[str] = "X-Simcore-Parent-Project-Uuid" +X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent" diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 609bf0b56a6..abefb98bd4e 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -122,6 +122,7 @@ async def create_job( new_project: ProjectGet = await webserver_api.create_project( project_in, is_hidden=hidden, + job_parent_resource_name=pre_job.runner_name, parent_project_uuid=x_simcore_parent_project_uuid, parent_node_id=x_simcore_parent_node_id, ) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies.py b/services/api-server/src/simcore_service_api_server/api/routes/studies.py index d5f3e2c821e..53fb3a818f5 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies.py @@ -90,6 +90,7 @@ async def clone_study( hidden=False, parent_project_uuid=x_simcore_parent_project_uuid, parent_node_id=x_simcore_parent_node_id, + job_parent_resource_name=None, # this resource is NOT a job ) return _create_study_from_project(project) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index f1e5513414b..101aff22488 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -94,11 +94,14 @@ async def create_study_job( """ hidden -- if True (default) hides project from UI """ + study_name = Study.compose_resource_name(f"{study_id}") + project = await webserver_api.clone_project( project_id=study_id, hidden=hidden, parent_project_uuid=x_simcore_parent_project_uuid, parent_node_id=x_simcore_parent_node_id, + job_parent_resource_name=study_name, ) job = create_job_from_study( study_key=study_id, project=project, job_inputs=job_inputs diff --git a/services/api-server/src/simcore_service_api_server/models/api_resources.py b/services/api-server/src/simcore_service_api_server/models/api_resources.py index 9a2221034ad..aac72872a45 100644 --- a/services/api-server/src/simcore_service_api_server/models/api_resources.py +++ b/services/api-server/src/simcore_service_api_server/models/api_resources.py @@ -46,6 +46,7 @@ def parse_last_resource_id(resource_name: RelativeResourceName) -> str: def compose_resource_name(*collection_or_resource_ids) -> RelativeResourceName: + # NOTE: collection_or_resource_ids expected de quoted_parts = [ urllib.parse.quote_plus(f"{_id}".lstrip("/")) for _id in collection_or_resource_ids @@ -56,3 +57,23 @@ def compose_resource_name(*collection_or_resource_ids) -> RelativeResourceName: def split_resource_name(resource_name: RelativeResourceName) -> list[str]: quoted_parts = resource_name.split("/") return [f"{urllib.parse.unquote_plus(p)}" for p in quoted_parts] + + +def split_resource_name_as_dict( + resource_name: RelativeResourceName, +) -> dict[str, str | None]: + """Returns a map with + resource_ids[Collection-ID] == Resource-ID + """ + parts = split_resource_name(resource_name) + return dict(zip(parts[::2], parts[1::2], strict=False)) + + +def parse_collections_ids(resource_name: RelativeResourceName) -> list[str]: + parts = split_resource_name(resource_name) + return parts[::2] + + +def parse_resources_ids(resource_name: RelativeResourceName) -> list[str]: + parts = split_resource_name(resource_name) + return parts[1::2] diff --git a/services/api-server/src/simcore_service_api_server/services_http/webserver.py b/services/api-server/src/simcore_service_api_server/services_http/webserver.py index c86ca6e1a4f..6a260b3f8d5 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services_http/webserver.py @@ -43,6 +43,7 @@ from pydantic import PositiveInt from servicelib.aiohttp.long_running_tasks.server import TaskStatus from servicelib.common_headers import ( + X_SIMCORE_API_JOB_PARENT_RESOURCE_NAME, X_SIMCORE_PARENT_NODE_ID, X_SIMCORE_PARENT_PROJECT_UUID, ) @@ -294,18 +295,22 @@ async def create_project( project: ProjectCreateNew, *, is_hidden: bool, + job_parent_resource_name: str, parent_project_uuid: ProjectID | None, parent_node_id: NodeID | None, ) -> ProjectGet: # POST /projects --> 202 Accepted - _headers = { + query_params = {"hidden": is_hidden} + headers = { X_SIMCORE_PARENT_PROJECT_UUID: parent_project_uuid, X_SIMCORE_PARENT_NODE_ID: parent_node_id, + X_SIMCORE_API_JOB_PARENT_RESOURCE_NAME: job_parent_resource_name, } + response = await self.client.post( "/projects", - params={"hidden": is_hidden}, - headers={k: f"{v}" for k, v in _headers.items() if v is not None}, + params=query_params, + headers={k: f"{v}" for k, v in headers.items() if v is not None}, json=jsonable_encoder(project, by_alias=True, exclude={"state"}), cookies=self.session_cookies, ) @@ -321,17 +326,20 @@ async def clone_project( hidden: bool, parent_project_uuid: ProjectID | None, parent_node_id: NodeID | None, + job_parent_resource_name: str | None = None, ) -> ProjectGet: - query = {"from_study": project_id, "hidden": hidden} + # POST /projects --> 202 Accepted + query_params = {"from_study": project_id, "hidden": hidden} _headers = { X_SIMCORE_PARENT_PROJECT_UUID: parent_project_uuid, X_SIMCORE_PARENT_NODE_ID: parent_node_id, + X_SIMCORE_API_JOB_PARENT_RESOURCE_NAME: job_parent_resource_name, } response = await self.client.post( "/projects", cookies=self.session_cookies, - params=query, + params=query_params, headers={k: f"{v}" for k, v in _headers.items() if v is not None}, ) response.raise_for_status() diff --git a/services/api-server/tests/unit/test_models_api_resources.py b/services/api-server/tests/unit/test_models_api_resources.py index 46f31b3ecd7..39137bcb8d0 100644 --- a/services/api-server/tests/unit/test_models_api_resources.py +++ b/services/api-server/tests/unit/test_models_api_resources.py @@ -7,8 +7,11 @@ from simcore_service_api_server.models.api_resources import ( compose_resource_name, + parse_collections_ids, parse_last_resource_id, + parse_resources_ids, split_resource_name, + split_resource_name_as_dict, ) @@ -38,3 +41,19 @@ def test_parse_resource_id(): assert ( parse_last_resource_id(resource_name) == split_resource_name(resource_name)[-1] ) + + collection_to_resource_id_map = split_resource_name_as_dict(resource_name) + # Collection-ID -> Resource-ID + assert list(collection_to_resource_id_map.keys()) == parse_collections_ids( + resource_name + ) + assert list(collection_to_resource_id_map.values()) == parse_resources_ids( + resource_name + ) + + assert collection_to_resource_id_map["solvers"] == "simcore/services/comp/isolve" + assert collection_to_resource_id_map["releases"] == "1.3.4" + assert ( + collection_to_resource_id_map["jobs"] == "f622946d-fd29-35b9-a193-abdd1095167c" + ) + assert collection_to_resource_id_map["outputs"] == "output 22" From 7ed03330676575ddda4edc7ff61e9b06165072b6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 27 Mar 2025 17:22:52 +0100 Subject: [PATCH 11/39] saves parent --- .../48604dfdc5f4_new_projects_to_job_map.py | 18 +++++++++++++----- .../models/projects_to_jobs.py | 14 ++++++++++---- .../tests/test_models_projects_to_jobs.py | 4 ++-- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py index 5345a9fbbfa..bbec001bd22 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py @@ -22,7 +22,12 @@ def upgrade(): "projects_to_jobs", sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), sa.Column("project_uuid", sa.String(), nullable=False), - sa.Column("job_name", sa.String(), nullable=False), + sa.Column( + "job_parent_resource_name", + sa.String(), + nullable=False, + doc="Prefix for the job resource name. For example, if the relative resource name is shelves/shelf1/books/book2, the parent resource name is shelves/shelf1.", + ), sa.ForeignKeyConstraint( ["project_uuid"], ["projects.uuid"], @@ -31,18 +36,21 @@ def upgrade(): ondelete="CASCADE", ), sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("job_name"), - sa.UniqueConstraint("project_uuid"), + sa.UniqueConstraint( + "project_uuid", + "job_parent_resource_name", + name="uq_projects_to_jobs_project_uuid_job_parent_resource_name", + ), ) # Populate the new table op.execute( sa.text( r""" -INSERT INTO projects_to_jobs (project_uuid, job_name) +INSERT INTO projects_to_jobs (project_uuid, job_parent_resource_name) SELECT uuid AS project_uuid, - regexp_replace(name, '\s+', '', 'g') AS job_name -- no spaces + regexp_replace(name, '/jobs/.+$', '', 'g') AS job_parent_resource_name -- trim /jobs/.+$ FROM projects WHERE name ~* '^solvers/.+/jobs/.+$' OR name ~* '^studies/.+/jobs/.+$'; """ diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py index 7a2d0052a5f..85a7ea14c0e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py @@ -5,6 +5,7 @@ from .projects import projects projects_to_jobs = sa.Table( + # Maps projects used as jobs in the public-api "projects_to_jobs", metadata, sa.Column( @@ -24,14 +25,19 @@ name="fk_projects_to_jobs_project_uuid", ), nullable=False, - unique=True, doc="Foreign key to projects.uuid", ), sa.Column( - "job_name", + "job_parent_resource_name", sa.String, nullable=False, - unique=True, - doc="Identifier for the job associated with the project", + doc="Prefix for the job resource name use in the public-api. For example, if " + "the relative resource name is shelves/shelf1/jobs/job2, " + "the parent resource name is shelves/shelf1.", + ), + sa.UniqueConstraint( + "project_uuid", + "job_parent_resource_name", + name="uq_projects_to_jobs_project_uuid_job_parent_resource_name", ), ) diff --git a/packages/postgres-database/tests/test_models_projects_to_jobs.py b/packages/postgres-database/tests/test_models_projects_to_jobs.py index be069d77403..e5bfae2e30a 100644 --- a/packages/postgres-database/tests/test_models_projects_to_jobs.py +++ b/packages/postgres-database/tests/test_models_projects_to_jobs.py @@ -121,9 +121,9 @@ def test_populate_projects_to_jobs_during_migration( assert len(result) == 2 assert ( "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", - "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1", ) in result assert ( "bf204942-007b-11ef-befd-0242ac114f07", - "studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07", + "studies/4b7a704a-007a-11ef-befd-0242ac114f07", ) in result From fe732f07539eb4afbff983ee5f16b5839def6ce8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 27 Mar 2025 17:33:03 +0100 Subject: [PATCH 12/39] extending test --- .../tests/test_models_projects_to_jobs.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/tests/test_models_projects_to_jobs.py b/packages/postgres-database/tests/test_models_projects_to_jobs.py index e5bfae2e30a..d6f2879694d 100644 --- a/packages/postgres-database/tests/test_models_projects_to_jobs.py +++ b/packages/postgres-database/tests/test_models_projects_to_jobs.py @@ -68,7 +68,7 @@ def test_populate_projects_to_jobs_during_migration( user_data = random_user( faker, name="test_populate_projects_to_jobs_during_migration" ) - stmt = sa.insert(users).values(**user_data).returning(users.c.id) + stmt = users.insert().values(**user_data).returning(users.c.id) result = conn.execute(stmt) user_id = result.scalar() @@ -114,7 +114,10 @@ def test_populate_projects_to_jobs_during_migration( with sync_engine.connect() as conn: # Query the projects_to_jobs table result = conn.execute( - sa.select(projects_to_jobs.c.project_uuid, projects_to_jobs.c.job_name) + sa.select( + projects_to_jobs.c.project_uuid, + projects_to_jobs.c.job_parent_resource_name, + ) ).fetchall() # Assert only valid projects are added @@ -127,3 +130,23 @@ def test_populate_projects_to_jobs_during_migration( "bf204942-007b-11ef-befd-0242ac114f07", "studies/4b7a704a-007a-11ef-befd-0242ac114f07", ) in result + + # Query project name and description for projects also in projects_to_jobs + result = conn.execute( + sa.select( + projects.c.name, + projects.c.uuid, + projects_to_jobs.c.job_parent_resource_name, + ).select_from( + projects.join( + projects_to_jobs, projects.c.uuid == projects_to_jobs.c.project_uuid + ) + ) + ).fetchall() + + # Print or assert the result as needed + for project_name, project_uuid, job_parent_resource_name in result: + assert ( + f"{job_parent_resource_name}/jobs/{project_uuid}" + == project_name.strip() + ) From 1c34cb5869e5fe32c2866c7cc2265c3312b365fa Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 27 Mar 2025 18:45:44 +0100 Subject: [PATCH 13/39] cleanup --- .../services_http/webserver.py | 14 +- .../projects/_controller/projects_rest.py | 2 + .../_controller/projects_rest_schemas.py | 169 ++++++++++-------- 3 files changed, 109 insertions(+), 76 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/services_http/webserver.py b/services/api-server/src/simcore_service_api_server/services_http/webserver.py index 6a260b3f8d5..10fda838463 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services_http/webserver.py @@ -187,16 +187,16 @@ async def _page_projects( limit: int, offset: int, show_hidden: bool, - search_by_project_name: str | None = None, + job_parent_resource_name: str | None = None, ) -> Page[ProjectGet]: assert 1 <= limit <= MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE # nosec assert offset >= 0 # nosec optional: dict[str, Any] = {} - if search_by_project_name is not None: - filters_dict = {"search_by_project_name": search_by_project_name} - filters_json = json_dumps(filters_dict) - optional["filters"] = filters_json + if job_parent_resource_name is not None: + optional["filters"] = json_dumps( + {"job_parent_resource_name": job_parent_resource_name} + ) with service_exception_handler( service_name="Webserver", @@ -360,11 +360,13 @@ async def get_project(self, *, project_id: UUID) -> ProjectGet: async def get_projects_w_solver_page( self, *, solver_name: str, limit: int, offset: int ) -> Page[ProjectGet]: + assert not solver_name.endswith("/") # nosec + return await self._page_projects( limit=limit, offset=offset, show_hidden=True, - search_by_project_name=solver_name, + job_parent_resource_name=solver_name, ) async def get_projects_page(self, *, limit: int, offset: int): diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index f20360d1cc0..407ae184905 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -158,6 +158,7 @@ async def list_projects(request: web.Request): workspace_id=query_params.workspace_id, search_by_multi_columns=query_params.search, search_by_project_name=query_params.filters.search_by_project_name, + # TODO: query_params.filters.job_parent_resource_name offset=query_params.offset, limit=query_params.limit, order_by=OrderBy.model_construct(**query_params.order_by.model_dump()), @@ -198,6 +199,7 @@ async def list_projects_full_search(request: web.Request): tag_ids_list=tag_ids_list, search_by_multi_columns=query_params.text, search_by_project_name=query_params.filters.search_by_project_name, + # TODO: query_params.filters.job_parent_resource_name offset=query_params.offset, limit=query_params.limit, order_by=OrderBy.model_construct(**query_params.order_by.model_dump()), diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest_schemas.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest_schemas.py index 55834b7b658..08de1844520 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest_schemas.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest_schemas.py @@ -37,26 +37,30 @@ class ProjectCreateHeaders(BaseModel): - - simcore_user_agent: str = Field( # type: ignore[literal-required] - default=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, - description="Optional simcore user agent", - alias=X_SIMCORE_USER_AGENT, - ) - - parent_project_uuid: ProjectID | None = Field( # type: ignore[literal-required] - default=None, - description="Optional parent project UUID", - alias=X_SIMCORE_PARENT_PROJECT_UUID, - ) - parent_node_id: NodeID | None = Field( # type: ignore[literal-required] - default=None, - description="Optional parent node ID", - alias=X_SIMCORE_PARENT_NODE_ID, - ) + simcore_user_agent: Annotated[ + str, + Field( + description="Optional simcore user agent", + alias=X_SIMCORE_USER_AGENT, + ), + ] = UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + parent_project_uuid: Annotated[ + ProjectID | None, + Field( + description="Optional parent project UUID", + alias=X_SIMCORE_PARENT_PROJECT_UUID, + ), + ] = None + parent_node_id: Annotated[ + NodeID | None, + Field( + description="Optional parent node ID", + alias=X_SIMCORE_PARENT_NODE_ID, + ), + ] = None @model_validator(mode="after") - def check_parent_valid(self) -> Self: + def _check_parent_valid(self) -> Self: if (self.parent_project_uuid is None and self.parent_node_id is not None) or ( self.parent_project_uuid is not None and self.parent_node_id is None ): @@ -68,34 +72,52 @@ def check_parent_valid(self) -> Self: class ProjectCreateQueryParams(BaseModel): - from_study: ProjectID | None = Field( - None, - description="Option to create a project from existing template or study: from_study={study_uuid}", - ) - as_template: bool = Field( - default=False, - description="Option to create a template from existing project: as_template=true", - ) - copy_data: bool = Field( - default=True, - description="Option to copy data when creating from an existing template or as a template, defaults to True", - ) - hidden: bool = Field( - default=False, - description="Enables/disables hidden flag. Hidden projects are by default unlisted", - ) + from_study: Annotated[ + ProjectID | None, + Field( + description="Option to create a project from existing template or study: from_study={study_uuid}", + ), + ] = None + as_template: Annotated[ + bool, + Field( + description="Option to create a template from existing project: as_template=true", + ), + ] = False + copy_data: Annotated[ + bool, + Field( + description="Option to copy data when creating from an existing template or as a template, defaults to True", + ), + ] = True + hidden: Annotated[ + bool, + Field( + description="Enables/disables hidden flag. Hidden projects are by default unlisted", + ), + ] = False model_config = ConfigDict(extra="forbid") class ProjectFilters(Filters): - trashed: bool | None = Field( - default=False, - description="Set to true to list trashed, false to list non-trashed (default), None to list all", - ) - search_by_project_name: str | None = Field( - default=None, - description="A search query to filter projects by their name. This field performs a case-insensitive partial match against the project name field.", - ) + trashed: Annotated[ + bool | None, + Field( + description="Set to true to list trashed, false to list non-trashed (default), None to list all", + ), + ] = False + search_by_project_name: Annotated[ + str | None, + Field( + description="A search query to filter projects by their name. This field performs a case-insensitive partial match against the project name field.", + ), + ] = None + job_parent_resource_name: Annotated[ + str | None, + Field( + description="A search query to filter projects with associated job_parent_resource_name", + ), + ] = None ProjectsListOrderParams = create_ordering_query_model_class( @@ -113,28 +135,34 @@ class ProjectFilters(Filters): class ProjectsListExtraQueryParams(RequestParameters): - project_type: ProjectTypeAPI = Field(default=ProjectTypeAPI.all, alias="type") - show_hidden: bool = Field( - default=False, description="includes projects marked as hidden in the listing" - ) - search: str | None = Field( - default=None, - description="Multi column full text search", - max_length=100, - examples=["My Project"], - ) - folder_id: FolderID | None = Field( - default=None, - description="Filter projects in specific folder. Default filtering is a root directory.", - ) - workspace_id: WorkspaceID | None = Field( - default=None, - description="Filter projects in specific workspace. Default filtering is a private workspace.", - ) + project_type: Annotated[ProjectTypeAPI, Field(alias="type")] = ProjectTypeAPI.all + show_hidden: Annotated[ + bool, Field(description="includes projects marked as hidden in the listing") + ] = False + search: Annotated[ + str | None, + Field( + description="Multi column full text search", + max_length=100, + examples=["My Project"], + ), + ] = None + folder_id: Annotated[ + FolderID | None, + Field( + description="Filter projects in specific folder. Default filtering is a root directory.", + ), + ] = None + workspace_id: Annotated[ + WorkspaceID | None, + Field( + description="Filter projects in specific workspace. Default filtering is a private workspace.", + ), + ] = None @field_validator("search", mode="before") @classmethod - def search_check_empty_string(cls, v): + def _search_check_empty_string(cls, v): if not v: return None return v @@ -157,27 +185,28 @@ class ProjectsListQueryParams( class ProjectActiveQueryParams(BaseModel): - client_session_id: str + client_session_id: Annotated[str, Field()] class ProjectSearchExtraQueryParams( PageQueryParameters, FiltersQueryParameters[ProjectFilters], ): - text: str | None = Field( - default=None, - description="Multi column full text search, across all folders and workspaces", - max_length=100, - examples=["My Project"], - ) + text: Annotated[ + str | None, + Field( + description="Multi column full text search, across all folders and workspaces", + max_length=100, + examples=["My Project"], + ), + ] = None tag_ids: Annotated[ str | None, Field( - default=None, description="Search by tag ID (multiple tag IDs may be provided separated by column)", examples=["1,3"], ), - ] + ] = None _empty_is_none = field_validator("text", mode="before")( empty_str_to_none_pre_validator From f8a881e682f12d23801168a8d89dffbf0337192f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 11:22:40 +0100 Subject: [PATCH 14/39] start interface --- .../api/routes/solvers_jobs.py | 10 ++++++++++ .../api/routes/studies_jobs.py | 12 +++++++++--- .../services_rpc/wb_api_server.py | 13 +++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index abefb98bd4e..76cf0ae57a2 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -12,6 +12,10 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from pydantic.types import PositiveInt +from simcore_service_api_server.api.dependencies.webserver_rpc import ( + get_wb_api_rpc_client, +) +from simcore_service_api_server.services_rpc.wb_api_server import WbApiRpcClient from ...exceptions.backend_errors import ProjectAlreadyStartedError from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES @@ -95,6 +99,7 @@ async def create_job( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], hidden: Annotated[bool, Query()] = True, @@ -126,6 +131,7 @@ async def create_job( parent_project_uuid=x_simcore_parent_project_uuid, parent_node_id=x_simcore_parent_node_id, ) + assert new_project # nosec assert new_project.uuid == pre_job.id # nosec @@ -140,6 +146,10 @@ async def create_job( assert job.name == pre_job.name # nosec assert job.name == _compose_job_resource_name(solver_key, version, job.id) # nosec + await wb_api_rpc.mark_project_as_job( + project_uuid=new_project.uuid, job_parent_resource_name=job.runner_name + ) + return job diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index 101aff22488..48b548b4a7f 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -15,6 +15,10 @@ from models_library.projects_nodes_io import NodeID from pydantic import PositiveInt from servicelib.logging_utils import log_context +from simcore_service_api_server.api.dependencies.webserver_rpc import ( + get_wb_api_rpc_client, +) +from simcore_service_api_server.services_rpc.wb_api_server import WbApiRpcClient from ...api.dependencies.authentication import get_current_user_id from ...api.dependencies.services import get_api_client @@ -86,6 +90,7 @@ async def create_study_job( study_id: StudyID, job_inputs: JobInputs, webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], hidden: Annotated[bool, Query()] = True, x_simcore_parent_project_uuid: ProjectID | None = Header(default=None), @@ -94,14 +99,11 @@ async def create_study_job( """ hidden -- if True (default) hides project from UI """ - study_name = Study.compose_resource_name(f"{study_id}") - project = await webserver_api.clone_project( project_id=study_id, hidden=hidden, parent_project_uuid=x_simcore_parent_project_uuid, parent_node_id=x_simcore_parent_node_id, - job_parent_resource_name=study_name, ) job = create_job_from_study( study_key=study_id, project=project, job_inputs=job_inputs @@ -123,6 +125,10 @@ async def create_study_job( patch_params=ProjectPatch(name=job.name), # type: ignore[arg-type] ) + await wb_api_rpc.mark_project_as_job( + project_uuid=job.id, job_parent_resource_name=job.runner_name + ) + project_inputs = await webserver_api.get_project_inputs(project_id=project.uuid) file_param_nodes = {} diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 3781de9e4b7..479aeb876a8 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -6,6 +6,7 @@ from fastapi_pagination import create_page from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.licenses import LicensedItemID +from models_library.projects import ProjectID from models_library.resource_tracker_licensed_items_checkouts import ( LicensedItemCheckoutID, ) @@ -41,6 +42,7 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.licenses.licensed_items import ( release_licensed_item_for_wallet as _release_licensed_item_for_wallet, ) +from simcore_service_api_server.models.api_resources import RelativeResourceName from ..exceptions.backend_errors import ( CanNotCheckoutServiceIsNotRunningError, @@ -200,6 +202,17 @@ async def release_licensed_item_for_wallet( async def ping(self) -> str: return await _ping(self._client) + async def mark_project_as_job( + self, project_uuid: ProjectID, job_parent_resource_name: RelativeResourceName + ): + assert not job_parent_resource_name.startswith("/") # nosec + assert "/" in job_parent_resource_name # nosec + assert not job_parent_resource_name.endswith("/") # nosec + + assert project_uuid + + raise NotImplementedError + def setup(app: FastAPI, rabbitmq_rmp_client: RabbitMQRPCClient): wb_api_rpc_client = WbApiRpcClient(_client=rabbitmq_rmp_client) From 9c135c1a296abe16b0ee6b97baf0964b99a99307 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 15:33:55 +0100 Subject: [PATCH 15/39] save --- .../helpers/webserver_rpc_server.py | 29 +++++++++++ .../rpc_interfaces/webserver/projects.py | 27 ++++++++++ .../models/schemas/files.py | 50 ++++++++++--------- .../services_rpc/wb_api_server.py | 7 ++- .../tests/unit/_with_db/test_api_user.py | 2 + .../tests/unit/api_solvers/conftest.py | 22 ++++++++ .../test_api_routers_solvers_jobs.py | 2 + .../test_api_routers_solvers_jobs_delete.py | 2 + .../test_api_routers_solvers_jobs_metadata.py | 1 + 9 files changed, 117 insertions(+), 25 deletions(-) create mode 100644 packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py new file mode 100644 index 00000000000..8516eb74f13 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py @@ -0,0 +1,29 @@ +# pylint: disable=not-context-manager +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +from models_library.projects import ProjectID +from pydantic import TypeAdapter +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient + + +class WebserverRpcSideEffects: + # pylint: disable=no-self-use + + async def mark_project_as_job( + self, + rpc_client: RabbitMQRPCClient, + *, + project_uuid: ProjectID, + job_parent_resource_name: str, + ) -> None: + assert rpc_client + + assert not job_parent_resource_name.startswith("/") # nosec + assert "/" in job_parent_resource_name # nosec + assert not job_parent_resource_name.endswith("/") # nosec + + TypeAdapter(ProjectID).validate_python(project_uuid) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py new file mode 100644 index 00000000000..de25df47b2d --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py @@ -0,0 +1,27 @@ +import logging + +from models_library.projects import ProjectID +from pydantic import TypeAdapter +from servicelib.logging_utils import log_decorator +from servicelib.rabbitmq import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def mark_project_as_job( + rpc_client: RabbitMQRPCClient, + *, + project_uuid: ProjectID, + job_parent_resource_name: str, +) -> None: + + assert rpc_client + + assert not job_parent_resource_name.startswith("/") # nosec + assert "/" in job_parent_resource_name # nosec + assert not job_parent_resource_name.endswith("/") # nosec + + TypeAdapter(ProjectID).validate_python(project_uuid) + + raise NotImplementedError diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/files.py b/services/api-server/src/simcore_service_api_server/models/schemas/files.py index 29cc9aacf0a..47ea68a4e0c 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/files.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/files.py @@ -37,28 +37,28 @@ class ClientFile(BaseModel): filename: FileName = Field(..., description="File name") filesize: NonNegativeInt = Field(..., description="File size in bytes") sha256_checksum: SHA256Str = Field(..., description="SHA256 checksum") + destination: Annotated[ + ProgramJobFilePath | None, + Field(..., description="Destination within a program job"), + ] class File(BaseModel): """Represents a file stored on the server side i.e. a unique reference to a file in the cloud.""" - # WARNING: from pydantic import File as FileParam - # NOTE: see https://ant.apache.org/manual/Tasks/checksum.html - - id: UUID = Field(..., description="Resource identifier") - - filename: str = Field(..., description="Name of the file with extension") - content_type: str | None = Field( - default=None, - description="Guess of type content [EXPERIMENTAL]", - validate_default=True, - ) - sha256_checksum: SHA256Str | None = Field( - default=None, - description="SHA256 hash of the file's content", - alias="checksum", # alias for backwards compatibility - ) - e_tag: ETag | None = Field(default=None, description="S3 entity tag") + id: Annotated[UUID, Field(description="Resource identifier")] + filename: Annotated[str, Field(description="Name of the file with extension")] + content_type: Annotated[ + str | None, + Field( + description="Guess of type content [EXPERIMENTAL]", validate_default=True + ), + ] = None + sha256_checksum: Annotated[ + SHA256Str | None, + Field(description="SHA256 hash of the file's content", alias="checksum"), + ] = None + e_tag: Annotated[ETag | None, Field(description="S3 entity tag")] = None model_config = ConfigDict( populate_by_name=True, @@ -163,16 +163,18 @@ def quoted_storage_file_id(self) -> str: class UploadLinks(BaseModel): - abort_upload: str - complete_upload: str + abort_upload: Annotated[str, Field()] + complete_upload: Annotated[str, Field()] class FileUploadData(BaseModel): - chunk_size: NonNegativeInt - urls: list[Annotated[AnyHttpUrl, UriSchema()]] - links: UploadLinks + chunk_size: Annotated[int, Field(description="Chunk size in bytes")] + urls: Annotated[list[Annotated[AnyHttpUrl, UriSchema()]], Field()] + links: Annotated[UploadLinks, Field()] class ClientFileUploadData(BaseModel): - file_id: UUID = Field(..., description="The file resource id") - upload_schema: FileUploadData = Field(..., description="Schema for uploading file") + file_id: Annotated[UUID, Field(description="The file resource id")] + upload_schema: Annotated[ + FileUploadData, Field(description="Schema for uploading file") + ] diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 479aeb876a8..6e45e33bb5c 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -27,6 +27,7 @@ from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( NotEnoughAvailableSeatsError, ) +from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions import ( ping as _ping, ) @@ -211,7 +212,11 @@ async def mark_project_as_job( assert project_uuid - raise NotImplementedError + await projects_rpc.mark_project_as_job( + rpc_client=self._client, + project_uuid=project_uuid, + job_parent_resource_name=job_parent_resource_name, + ) def setup(app: FastAPI, rabbitmq_rmp_client: RabbitMQRPCClient): diff --git a/services/api-server/tests/unit/_with_db/test_api_user.py b/services/api-server/tests/unit/_with_db/test_api_user.py index 93a3bdf8f68..eca2ea3e70b 100644 --- a/services/api-server/tests/unit/_with_db/test_api_user.py +++ b/services/api-server/tests/unit/_with_db/test_api_user.py @@ -54,6 +54,7 @@ async def test_get_profile( client: httpx.AsyncClient, auth: httpx.BasicAuth, mocked_webserver_service_api: MockRouter, + mocked_rpc_webserver_service_api: dict[str, MockType], ): # needs no auth resp = await client.get(f"/{API_VTAG}/meta") @@ -77,6 +78,7 @@ async def test_update_profile( client: httpx.AsyncClient, auth: httpx.BasicAuth, mocked_webserver_service_api: MockRouter, + mocked_rpc_webserver_service_api: dict[str, MockType], ): # needs auth resp = await client.put( diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index 82c125139b6..2f06f902695 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -13,7 +13,9 @@ from fastapi import FastAPI, status from fastapi.encoders import jsonable_encoder from models_library.projects_state import RunningState +from pytest_mock import MockerFixture, MockType from pytest_simcore.helpers import faker_catalog +from pytest_simcore.helpers.webserver_rpc_server import WebserverRpcSideEffects from respx import MockRouter from simcore_service_api_server.core.settings import ApplicationSettings from simcore_service_api_server.services_http.director_v2 import ComputationTaskGet @@ -43,6 +45,26 @@ def mocked_webserver_service_api( return mocked_webserver_service_api_base +@pytest.fixture +def mocked_rpc_webserver_service_api( + app: FastAPI, mocker: MockerFixture +) -> dict[str, MockType]: + from simcore_service_api_server.services_rpc.wb_api_server import projects_rpc + + settings: ApplicationSettings = app.state.settings + assert settings.API_SERVER_WEBSERVER + + side_effects = WebserverRpcSideEffects() + + return { + "mark_project_as_job": mocker.patch.object( + projects_rpc, + "mark_project_as_job", + side_effects.mark_project_as_job, + ), + } + + @pytest.fixture def mocked_catalog_service_api( app: FastAPI, diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py index e3f97def12d..2d6c10bb13f 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py @@ -18,6 +18,7 @@ from models_library.services import ServiceMetaDataPublished from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import AnyUrl, HttpUrl, TypeAdapter +from pytest_mock import MockType from respx import MockRouter from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.core.settings import ApplicationSettings @@ -206,6 +207,7 @@ async def test_run_solver_job( mocked_catalog_service_api: MockRouter, mocked_directorv2_service_api: MockRouter, mocked_webserver_service_api: MockRouter, + mocked_rpc_webserver_service_api: dict[str, MockType], auth: httpx.BasicAuth, project_id: str, solver_key: str, diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py index bdb3886ebec..2b264d771a6 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py @@ -33,6 +33,7 @@ class MockedBackendApiDict(TypedDict): @pytest.fixture def mocked_backend_services_apis_for_delete_non_existing_project( mocked_webserver_service_api: MockRouter, + mocked_rpc_webserver_service_api: dict[str, MockType], project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "delete_project_not_found.json" @@ -85,6 +86,7 @@ async def test_delete_non_existing_solver_job( @pytest.fixture def mocked_backend_services_apis_for_create_and_delete_solver_job( mocked_webserver_service_api: MockRouter, + mocked_rpc_webserver_service_api: dict[str, MockType], mocked_catalog_service_api: MockRouter, project_tests_dir: Path, ) -> MockedBackendApiDict: diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py index 6b62c89b6b8..bc8495389e9 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py @@ -38,6 +38,7 @@ def _as_path_regex(initial_path: str): @pytest.fixture def mocked_backend( mocked_webserver_service_api: MockRouter, + mocked_rpc_webserver_service_api: dict[str, MockType], mocked_catalog_service_api: MockRouter, project_tests_dir: Path, ) -> MockedBackendApiDict: From d29ac43a83ca513751ea3d5aae0f5a1e984abb23 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 16:27:13 +0100 Subject: [PATCH 16/39] rm header on creation --- packages/service-library/src/servicelib/common_headers.py | 3 --- .../simcore_service_api_server/api/routes/solvers_jobs.py | 1 - .../src/simcore_service_api_server/api/routes/studies.py | 1 - .../simcore_service_api_server/services_http/webserver.py | 5 ----- services/api-server/tests/unit/api_solvers/conftest.py | 3 ++- 5 files changed, 2 insertions(+), 11 deletions(-) diff --git a/packages/service-library/src/servicelib/common_headers.py b/packages/service-library/src/servicelib/common_headers.py index 0dc279b8956..430823fa776 100644 --- a/packages/service-library/src/servicelib/common_headers.py +++ b/packages/service-library/src/servicelib/common_headers.py @@ -4,9 +4,6 @@ X_DYNAMIC_SIDECAR_REQUEST_DNS: Final[str] = "X-Dynamic-Sidecar-Request-DNS" X_DYNAMIC_SIDECAR_REQUEST_SCHEME: Final[str] = "X-Dynamic-Sidecar-Request-Scheme" X_FORWARDED_PROTO: Final[str] = "X-Forwarded-Proto" -X_SIMCORE_API_JOB_PARENT_RESOURCE_NAME: Final[str] = ( - "X-Simcore-Api-Job-Parent-Resource-Name" -) X_SIMCORE_PARENT_NODE_ID: Final[str] = "X-Simcore-Parent-Node-Id" X_SIMCORE_PARENT_PROJECT_UUID: Final[str] = "X-Simcore-Parent-Project-Uuid" X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent" diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 76cf0ae57a2..7be222edd58 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -127,7 +127,6 @@ async def create_job( new_project: ProjectGet = await webserver_api.create_project( project_in, is_hidden=hidden, - job_parent_resource_name=pre_job.runner_name, parent_project_uuid=x_simcore_parent_project_uuid, parent_node_id=x_simcore_parent_node_id, ) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies.py b/services/api-server/src/simcore_service_api_server/api/routes/studies.py index 53fb3a818f5..d5f3e2c821e 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies.py @@ -90,7 +90,6 @@ async def clone_study( hidden=False, parent_project_uuid=x_simcore_parent_project_uuid, parent_node_id=x_simcore_parent_node_id, - job_parent_resource_name=None, # this resource is NOT a job ) return _create_study_from_project(project) diff --git a/services/api-server/src/simcore_service_api_server/services_http/webserver.py b/services/api-server/src/simcore_service_api_server/services_http/webserver.py index 10fda838463..a522ff9ebdd 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services_http/webserver.py @@ -43,7 +43,6 @@ from pydantic import PositiveInt from servicelib.aiohttp.long_running_tasks.server import TaskStatus from servicelib.common_headers import ( - X_SIMCORE_API_JOB_PARENT_RESOURCE_NAME, X_SIMCORE_PARENT_NODE_ID, X_SIMCORE_PARENT_PROJECT_UUID, ) @@ -295,7 +294,6 @@ async def create_project( project: ProjectCreateNew, *, is_hidden: bool, - job_parent_resource_name: str, parent_project_uuid: ProjectID | None, parent_node_id: NodeID | None, ) -> ProjectGet: @@ -304,7 +302,6 @@ async def create_project( headers = { X_SIMCORE_PARENT_PROJECT_UUID: parent_project_uuid, X_SIMCORE_PARENT_NODE_ID: parent_node_id, - X_SIMCORE_API_JOB_PARENT_RESOURCE_NAME: job_parent_resource_name, } response = await self.client.post( @@ -326,14 +323,12 @@ async def clone_project( hidden: bool, parent_project_uuid: ProjectID | None, parent_node_id: NodeID | None, - job_parent_resource_name: str | None = None, ) -> ProjectGet: # POST /projects --> 202 Accepted query_params = {"from_study": project_id, "hidden": hidden} _headers = { X_SIMCORE_PARENT_PROJECT_UUID: parent_project_uuid, X_SIMCORE_PARENT_NODE_ID: parent_node_id, - X_SIMCORE_API_JOB_PARENT_RESOURCE_NAME: job_parent_resource_name, } response = await self.client.post( diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index 2f06f902695..d27dc47fc84 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -49,7 +49,8 @@ def mocked_webserver_service_api( def mocked_rpc_webserver_service_api( app: FastAPI, mocker: MockerFixture ) -> dict[str, MockType]: - from simcore_service_api_server.services_rpc.wb_api_server import projects_rpc + # from simcore_service_api_server.services_rpc.wb_api_server import projects_rpc + from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc settings: ApplicationSettings = app.state.settings assert settings.API_SERVER_WEBSERVER From f4ec4984e6533507e6c886741906d3f3f2dba3d3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 17:16:47 +0100 Subject: [PATCH 17/39] server-client rpc interface --- .../rpc_interfaces/webserver/projects.py | 23 +++- .../projects/_controller/projects_rpc.py | 33 +++++ .../projects/plugin.py | 11 +- .../unit/with_dbs/02/test_projects_rpc.py | 120 ++++++++++++++++++ 4 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py create mode 100644 services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py index de25df47b2d..bdb4da2f57c 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py @@ -1,7 +1,11 @@ import logging +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.products import ProductName from models_library.projects import ProjectID -from pydantic import TypeAdapter +from models_library.rabbitmq_basic_types import RPCMethodName +from models_library.users import UserID +from pydantic import TypeAdapter, validate_call from servicelib.logging_utils import log_decorator from servicelib.rabbitmq import RabbitMQRPCClient @@ -9,19 +13,26 @@ @log_decorator(_logger, level=logging.DEBUG) +@validate_call(config={"arbitrary_types_allowed": True}) async def mark_project_as_job( rpc_client: RabbitMQRPCClient, *, + product_name: ProductName, + user_id: UserID, project_uuid: ProjectID, job_parent_resource_name: str, ) -> None: - assert rpc_client - assert not job_parent_resource_name.startswith("/") # nosec assert "/" in job_parent_resource_name # nosec assert not job_parent_resource_name.endswith("/") # nosec - TypeAdapter(ProjectID).validate_python(project_uuid) - - raise NotImplementedError + result = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("mark_project_as_job"), + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name=job_parent_resource_name, + ) + assert result is None diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py new file mode 100644 index 00000000000..70a62846e4a --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py @@ -0,0 +1,33 @@ +from aiohttp import web +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.products import ProductName +from models_library.projects import ProjectID +from models_library.users import UserID +from pydantic import validate_call +from servicelib.rabbitmq import RPCRouter + +from ...rabbitmq import get_rabbitmq_rpc_server + +router = RPCRouter() + + +@router.expose() +@validate_call(config={"arbitrary_types_allowed": True}) +async def mark_project_as_job( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + project_uuid: ProjectID, + job_parent_resource_name: str, +) -> None: + msg = ( + f"You have reached {__name__} but this feature is still not implemented. " + f"Inputs: {app=}, {product_name=}, {user_id=}, {project_uuid=}, {job_parent_resource_name=}" + ) + raise NotImplementedError(msg) + + +async def register_rpc_routes_on_startup(app: web.Application): + rpc_server = get_rabbitmq_rpc_server(app) + await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/projects/plugin.py b/services/web/server/src/simcore_service_webserver/projects/plugin.py index f968908c797..06f7a930808 100644 --- a/services/web/server/src/simcore_service_webserver/projects/plugin.py +++ b/services/web/server/src/simcore_service_webserver/projects/plugin.py @@ -19,13 +19,14 @@ nodes_rest, ports_rest, projects_rest, + projects_rpc, + projects_slot, projects_states_rest, tags_rest, trash_rest, wallets_rest, workspaces_rest, ) -from ._controller.projects_slot import setup_project_observer_events from ._projects_repository_legacy import setup_projects_db from ._security_service import setup_projects_access @@ -48,9 +49,13 @@ def setup_projects(app: web.Application) -> bool: # database API setup_projects_db(app) - # registers event handlers (e.g. on_user_disconnect) - setup_project_observer_events(app) + # setup SLOT-controllers + projects_slot.setup_project_observer_events(app) + # setup RPC-controllers + app.on_startup.append(projects_rpc.register_rpc_routes_on_startup) + + # setup REST-controllers app.router.add_routes(projects_states_rest.routes) app.router.add_routes(projects_rest.routes) app.router.add_routes(comments_rest.routes) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py new file mode 100644 index 00000000000..ccdb679b408 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py @@ -0,0 +1,120 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + + +from collections.abc import AsyncIterator, Awaitable, Callable +from uuid import UUID + +import pytest +from aiohttp.test_utils import TestClient +from common_library.users_enums import UserRole +from models_library.products import ProductName +from models_library.projects import ProjectID +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.webserver_login import NewUser, UserInfoDict +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc +from settings_library.rabbit import RabbitSettings +from simcore_service_webserver.application_settings import ApplicationSettings +from simcore_service_webserver.projects.models import ProjectDict + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.fixture +def user_role() -> UserRole: + # for logged_user + return UserRole.USER + + +@pytest.fixture +def app_environment( + rabbit_service: RabbitSettings, + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, +): + new_envs = setenvs_from_dict( + monkeypatch, + { + **app_environment, + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + }, + ) + + settings = ApplicationSettings.create_from_envs() + assert settings.WEBSERVER_RABBITMQ + + return new_envs + + +@pytest.fixture +async def rpc_client( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], +) -> RabbitMQRPCClient: + return await rabbitmq_rpc_client("client") + + +@pytest.fixture +async def other_user( + client: TestClient, + logged_user: UserInfoDict, +) -> AsyncIterator[UserInfoDict]: + + async with NewUser( + user_data={ + "name": "other-user", + "email": "other-user" + logged_user["email"], + }, + app=client.app, + ) as other_user_info: + + assert other_user_info["name"] != logged_user["name"] + yield other_user_info + + +async def test_rpc_client_mark_project_as_job( + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + logged_user: UserInfoDict, + other_user: UserInfoDict, + user_project: ProjectDict, +): + # `logged_user` OWNS the `user_project` but not `other_user` + project_uuid: ProjectID = UUID(user_project["uuid"]) + user_id = logged_user["id"] + other_user_id = other_user["id"] + + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name="solvers/solver123/version/1.2.3", + ) + + with pytest.raises(Exception, match="no access"): + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=other_user_id, # <-- no access + project_uuid=project_uuid, + job_parent_resource_name="solvers/solver123/version/1.2.3", + ) + + with pytest.raises(Exception, match="not found"): + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=logged_user["id"], + project_uuid=UUID("00000000-0000-0000-0000-000000000000"), # <-- wont find + job_parent_resource_name="solvers/solver123/version/1.2.3", + ) From ee54419e890591a498ab58d40205e6046b2abc54 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 17:47:17 +0100 Subject: [PATCH 18/39] jobs service and repo layers --- .../projects/_controller/projects_rpc.py | 12 +++-- .../projects/_jobs_repository.py | 39 +++++++++++++++ .../projects/_jobs_service.py | 48 +++++++++++++++++++ 3 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py create mode 100644 services/web/server/src/simcore_service_webserver/projects/_jobs_service.py diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py index 70a62846e4a..699c617bdd6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py @@ -7,6 +7,7 @@ from servicelib.rabbitmq import RPCRouter from ...rabbitmq import get_rabbitmq_rpc_server +from .. import _jobs_service router = RPCRouter() @@ -21,11 +22,14 @@ async def mark_project_as_job( project_uuid: ProjectID, job_parent_resource_name: str, ) -> None: - msg = ( - f"You have reached {__name__} but this feature is still not implemented. " - f"Inputs: {app=}, {product_name=}, {user_id=}, {project_uuid=}, {job_parent_resource_name=}" + + await _jobs_service.set_project_as_job( + app, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name=job_parent_resource_name, ) - raise NotImplementedError(msg) async def register_rpc_routes_on_startup(app: web.Application): diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py new file mode 100644 index 00000000000..123aeabbb39 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -0,0 +1,39 @@ +import logging + +from models_library.projects import ProjectID +from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs +from simcore_postgres_database.utils_repos import transaction_context +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.base_repository import BaseRepository + +_logger = logging.getLogger(__name__) + + +class ProjectJobsRepository(BaseRepository): + + async def set_project_as_job( + self, + connection: AsyncConnection | None = None, + *, + project_uuid: ProjectID, + job_parent_resource_name: str, + ) -> int: + async with transaction_context(self.engine, connection) as conn: + stmt = ( + pg_insert(projects_to_jobs) + .values( + project_uuid=project_uuid, + job_parent_resource_name=job_parent_resource_name, + ) + .on_conflict_do_update( + index_elements=["project_uuid", "job_parent_resource_name"], + set_={"job_parent_resource_name": job_parent_resource_name}, + ) + .returning(projects_to_jobs.c.id) + ) + + result = await conn.execute(stmt) + row = result.one() + return row.id diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py new file mode 100644 index 00000000000..cca37cc756c --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py @@ -0,0 +1,48 @@ +import logging +from typing import Annotated + +from aiohttp import web +from models_library.products import ProductName +from models_library.projects import ProjectID +from models_library.users import UserID +from pydantic import AfterValidator, validate_call + +from ._access_rights_service import check_user_project_permission +from ._jobs_repository import ProjectJobsRepository + +_logger = logging.getLogger(__name__) + + +def _validate_job_parent_resource_name(value: str) -> str: + if value and not value.startswith("/") and not value.endswith("/") and "/" in value: + return value + msg = "Invalid format: must contain '/' but cannot start or end with '/'" + raise ValueError(msg) + + +@validate_call(config={"arbitrary_types_allowed": True}) +async def set_project_as_job( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + project_uuid: ProjectID, + job_parent_resource_name: Annotated[ + str, AfterValidator(_validate_job_parent_resource_name) + ], +) -> None: + + await check_user_project_permission( + app, + project_id=project_uuid, + user_id=user_id, + product_name=product_name, + permission="write", + ) + + repo = ProjectJobsRepository.create_from_app(app) + + projects_to_jobs_id = await repo.set_project_as_job( + project_uuid=project_uuid, job_parent_resource_name=job_parent_resource_name + ) + assert projects_to_jobs_id # nosec From 7099475a704b6d81623ab1e8af5c3da085f9c37e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 18:45:39 +0100 Subject: [PATCH 19/39] new error factory --- .../src/common_library/errors_classes.py | 27 ++++++++ .../tests/test_errors_classes.py | 63 +++++++++++++++---- 2 files changed, 79 insertions(+), 11 deletions(-) diff --git a/packages/common-library/src/common_library/errors_classes.py b/packages/common-library/src/common_library/errors_classes.py index dfee557d38c..5e27f6fad44 100644 --- a/packages/common-library/src/common_library/errors_classes.py +++ b/packages/common-library/src/common_library/errors_classes.py @@ -52,3 +52,30 @@ def error_context(self) -> dict[str, Any]: def error_code(self) -> str: assert isinstance(self, Exception), "subclass must be exception" # nosec return create_error_code(self) + + +class NotFoundError(OsparcErrorMixin): + msg_template = "{resource} not found: id='{resource_id}'" + + +class ForbiddenError(OsparcErrorMixin): + msg_template = "Access to {resource} is forbidden: id='{resource_id}'" + + +def make_resource_error( + resource: str, + error_cls: type[OsparcErrorMixin], + base_exception: type[Exception] = Exception, +) -> type[OsparcErrorMixin]: + class _ResourceError(error_cls, base_exception): + def __init__(self, **ctx: Any): + ctx.setdefault("resource", resource) + # guesses identifer e.g. project_id, user_id + if resource_id := ctx.get(f"{resource.lower()}_id"): + ctx.setdefault("resource_id", resource_id) + + super().__init__(**ctx) + + resource_class_name = "".join(word.capitalize() for word in resource.split("_")) + _ResourceError.__name__ = f"{resource_class_name}{error_cls.__name__}" + return _ResourceError diff --git a/packages/common-library/tests/test_errors_classes.py b/packages/common-library/tests/test_errors_classes.py index efe4c44b86e..ebc928e607f 100644 --- a/packages/common-library/tests/test_errors_classes.py +++ b/packages/common-library/tests/test_errors_classes.py @@ -9,24 +9,24 @@ from typing import Any import pytest -from common_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import ( + ForbiddenError, + NotFoundError, + OsparcErrorMixin, + make_resource_error, +) def test_get_full_class_name(): - class A(OsparcErrorMixin): - ... + class A(OsparcErrorMixin): ... - class B1(A): - ... + class B1(A): ... - class B2(A): - ... + class B2(A): ... - class C(B2): - ... + class C(B2): ... - class B12(B1, ValueError): - ... + class B12(B1, ValueError): ... assert B1._get_full_class_name() == "A.B1" assert C._get_full_class_name() == "A.B2.C" @@ -159,3 +159,44 @@ class MyError(OsparcErrorMixin, ValueError): "message": "42 and 'missing=?'", "value": 42, } + + +def test_resource_error_factory(): + ProjectNotFoundError = make_resource_error("project", NotFoundError) + + error_1 = ProjectNotFoundError(resource_id="abc123") + assert "resource_id" in error_1.error_context() + assert error_1.resource_id in error_1.message # type: ignore + + +def test_resource_error_factory_auto_detect_resource_id(): + ProjectForbiddenError = make_resource_error("project", ForbiddenError) + error_2 = ProjectForbiddenError(project_id="abc123", other_id="foo") + assert ( + error_2.resource_id == error_2.project_id # type: ignore + ), "auto-detects project ids as resourceid" + assert error_2.other_id # type: ignore + assert error_2.code == "ForbiddenError.ProjectForbiddenError" + + assert error_2.error_context() == { + "project_id": "abc123", + "other_id": "foo", + "resource": "project", + "resource_id": "abc123", + "message": "Access to project is forbidden: id='abc123'", + "code": "ForbiddenError.ProjectForbiddenError", + } + + +def test_resource_error_factory_different_base_exception(): + + class MyBaseError(Exception): ... + + OtherProjectForbiddenError = make_resource_error( + "other_project", ForbiddenError, MyBaseError + ) + + assert issubclass(OtherProjectForbiddenError, MyBaseError) + + error_3 = OtherProjectForbiddenError(project_id="abc123") + assert error_3.code == "MyBaseError.ForbiddenError.OtherProjectForbiddenError" From e7013d4c0cf17c60f2d08f905be10b0c3a630edf Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 19:28:20 +0100 Subject: [PATCH 20/39] error handling --- .../src/common_library/errors_classes.py | 2 +- .../rpc_interfaces/webserver/errors.py | 17 ++++++++++ .../rpc_interfaces/webserver/projects.py | 1 + .../projects/_controller/projects_rpc.py | 33 ++++++++++++++----- .../projects/_jobs_repository.py | 2 +- .../unit/with_dbs/02/test_projects_rpc.py | 5 ++- 6 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py diff --git a/packages/common-library/src/common_library/errors_classes.py b/packages/common-library/src/common_library/errors_classes.py index 5e27f6fad44..d05c64acfd5 100644 --- a/packages/common-library/src/common_library/errors_classes.py +++ b/packages/common-library/src/common_library/errors_classes.py @@ -66,7 +66,7 @@ def make_resource_error( resource: str, error_cls: type[OsparcErrorMixin], base_exception: type[Exception] = Exception, -) -> type[OsparcErrorMixin]: +) -> type[Exception]: class _ResourceError(error_cls, base_exception): def __init__(self, **ctx: Any): ctx.setdefault("resource", resource) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py new file mode 100644 index 00000000000..803e5ab19f0 --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py @@ -0,0 +1,17 @@ +from common_library.errors_classes import ( + OsparcErrorMixin, +) + + +class WebServerRpcError(OsparcErrorMixin, Exception): + msg_template = "{details}" + + @classmethod + def from_domain_error(cls, err: OsparcErrorMixin): + return cls(details=f"{err} [{err.__class__.__name__}]", **err.error_context()) + + +class ProjectNotFoundRpcError(WebServerRpcError): ... + + +class ProjectForbiddenRpcError(WebServerRpcError): ... diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py index bdb4da2f57c..f06ce436abc 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py @@ -34,5 +34,6 @@ async def mark_project_as_job( user_id=user_id, project_uuid=project_uuid, job_parent_resource_name=job_parent_resource_name, + timeout_s=None, # TODO: remove after testing ) assert result is None diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py index 699c617bdd6..8dd014e4ab8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py @@ -3,16 +3,27 @@ from models_library.products import ProductName from models_library.projects import ProjectID from models_library.users import UserID -from pydantic import validate_call +from pydantic import ValidationError, validate_call from servicelib.rabbitmq import RPCRouter +from servicelib.rabbitmq.rpc_interfaces.webserver.errors import ( + ProjectForbiddenRpcError, + ProjectNotFoundRpcError, +) from ...rabbitmq import get_rabbitmq_rpc_server from .. import _jobs_service +from ..exceptions import ProjectInvalidRightsError router = RPCRouter() -@router.expose() +@router.expose( + reraise_if_error_type=( + ProjectForbiddenRpcError, + ProjectNotFoundRpcError, + ValidationError, + ) +) @validate_call(config={"arbitrary_types_allowed": True}) async def mark_project_as_job( app: web.Application, @@ -23,13 +34,17 @@ async def mark_project_as_job( job_parent_resource_name: str, ) -> None: - await _jobs_service.set_project_as_job( - app, - product_name=product_name, - user_id=user_id, - project_uuid=project_uuid, - job_parent_resource_name=job_parent_resource_name, - ) + try: + + await _jobs_service.set_project_as_job( + app, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name=job_parent_resource_name, + ) + except ProjectInvalidRightsError as err: + raise ProjectForbiddenRpcError.from_domain_error(err) from err async def register_rpc_routes_on_startup(app: web.Application): diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index 123aeabbb39..d9ec89dbca5 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -24,7 +24,7 @@ async def set_project_as_job( stmt = ( pg_insert(projects_to_jobs) .values( - project_uuid=project_uuid, + project_uuid=f"{project_uuid}", job_parent_resource_name=job_parent_resource_name, ) .on_conflict_do_update( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py index ccdb679b408..574e8261864 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py @@ -17,6 +17,7 @@ from pytest_simcore.helpers.webserver_login import NewUser, UserInfoDict from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc +from servicelib.rabbitmq.rpc_interfaces.webserver.errors import ProjectForbiddenRpcError from settings_library.rabbit import RabbitSettings from simcore_service_webserver.application_settings import ApplicationSettings from simcore_service_webserver.projects.models import ProjectDict @@ -101,7 +102,7 @@ async def test_rpc_client_mark_project_as_job( job_parent_resource_name="solvers/solver123/version/1.2.3", ) - with pytest.raises(Exception, match="no access"): + with pytest.raises(ProjectForbiddenRpcError) as err_info: await projects_rpc.mark_project_as_job( rpc_client=rpc_client, product_name=product_name, @@ -110,6 +111,8 @@ async def test_rpc_client_mark_project_as_job( job_parent_resource_name="solvers/solver123/version/1.2.3", ) + assert err_info.value.error_context()["project_uuid"] == project_uuid + with pytest.raises(Exception, match="not found"): await projects_rpc.mark_project_as_job( rpc_client=rpc_client, From 0a1b066a739700fac8c0ba095f3de65f848fe80f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 19:38:39 +0100 Subject: [PATCH 21/39] test cleanup --- .../rpc_interfaces/webserver/projects.py | 5 -- .../projects/_controller/projects_rpc.py | 5 +- .../unit/with_dbs/02/test_projects_rpc.py | 54 ++++++++++++++----- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py index f06ce436abc..f1cafcb1618 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py @@ -23,10 +23,6 @@ async def mark_project_as_job( job_parent_resource_name: str, ) -> None: - assert not job_parent_resource_name.startswith("/") # nosec - assert "/" in job_parent_resource_name # nosec - assert not job_parent_resource_name.endswith("/") # nosec - result = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("mark_project_as_job"), @@ -34,6 +30,5 @@ async def mark_project_as_job( user_id=user_id, project_uuid=project_uuid, job_parent_resource_name=job_parent_resource_name, - timeout_s=None, # TODO: remove after testing ) assert result is None diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py index 8dd014e4ab8..9c37f87ae56 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py @@ -12,7 +12,7 @@ from ...rabbitmq import get_rabbitmq_rpc_server from .. import _jobs_service -from ..exceptions import ProjectInvalidRightsError +from ..exceptions import ProjectInvalidRightsError, ProjectNotFoundError router = RPCRouter() @@ -46,6 +46,9 @@ async def mark_project_as_job( except ProjectInvalidRightsError as err: raise ProjectForbiddenRpcError.from_domain_error(err) from err + except ProjectNotFoundError as err: + raise ProjectNotFoundRpcError.from_domain_error(err) from err + async def register_rpc_routes_on_startup(app: web.Application): rpc_server = get_rabbitmq_rpc_server(app) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py index 574e8261864..3cc8dde0e0b 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py @@ -12,12 +12,16 @@ from common_library.users_enums import UserRole from models_library.products import ProductName from models_library.projects import ProjectID +from pydantic import ValidationError from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import NewUser, UserInfoDict from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc -from servicelib.rabbitmq.rpc_interfaces.webserver.errors import ProjectForbiddenRpcError +from servicelib.rabbitmq.rpc_interfaces.webserver.errors import ( + ProjectForbiddenRpcError, + ProjectNotFoundRpcError, +) from settings_library.rabbit import RabbitSettings from simcore_service_webserver.application_settings import ApplicationSettings from simcore_service_webserver.projects.models import ProjectDict @@ -64,6 +68,25 @@ async def rpc_client( return await rabbitmq_rpc_client("client") +async def test_rpc_client_mark_project_as_job( + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + logged_user: UserInfoDict, + user_project: ProjectDict, +): + # `logged_user` OWNS the `user_project` but not `other_user` + project_uuid: ProjectID = UUID(user_project["uuid"]) + user_id = logged_user["id"] + + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name="solvers/solver123/version/1.2.3", + ) + + @pytest.fixture async def other_user( client: TestClient, @@ -82,7 +105,7 @@ async def other_user( yield other_user_info -async def test_rpc_client_mark_project_as_job( +async def test_errors_on_rpc_client_mark_project_as_job( rpc_client: RabbitMQRPCClient, product_name: ProductName, logged_user: UserInfoDict, @@ -94,15 +117,7 @@ async def test_rpc_client_mark_project_as_job( user_id = logged_user["id"] other_user_id = other_user["id"] - await projects_rpc.mark_project_as_job( - rpc_client=rpc_client, - product_name=product_name, - user_id=user_id, - project_uuid=project_uuid, - job_parent_resource_name="solvers/solver123/version/1.2.3", - ) - - with pytest.raises(ProjectForbiddenRpcError) as err_info: + with pytest.raises(ProjectForbiddenRpcError) as exc_info: await projects_rpc.mark_project_as_job( rpc_client=rpc_client, product_name=product_name, @@ -111,9 +126,9 @@ async def test_rpc_client_mark_project_as_job( job_parent_resource_name="solvers/solver123/version/1.2.3", ) - assert err_info.value.error_context()["project_uuid"] == project_uuid + assert exc_info.value.error_context()["project_uuid"] == project_uuid - with pytest.raises(Exception, match="not found"): + with pytest.raises(ProjectNotFoundRpcError, match="not found"): await projects_rpc.mark_project_as_job( rpc_client=rpc_client, product_name=product_name, @@ -121,3 +136,16 @@ async def test_rpc_client_mark_project_as_job( project_uuid=UUID("00000000-0000-0000-0000-000000000000"), # <-- wont find job_parent_resource_name="solvers/solver123/version/1.2.3", ) + + with pytest.raises(ValidationError, match="job_parent_resource_name") as exc_info: + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name="This is not a resource", # <-- wrong format + ) + + assert exc_info.value.error_count() == 1 + assert exc_info.value.errors()[0]["loc"] == ("job_parent_resource_name",) + assert exc_info.value.errors()[0]["type"] == "value_error" From 5d76961c140b38a6e4aa6f738278f4fcc348469e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 22:32:51 +0100 Subject: [PATCH 22/39] fixes mypy --- services/api-server/tests/unit/_with_db/test_api_user.py | 1 + .../api_solvers/test_api_routers_solvers_jobs_delete.py | 1 + .../api_solvers/test_api_routers_solvers_jobs_metadata.py | 1 + .../simcore_service_webserver/projects/_jobs_repository.py | 6 ++++-- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/services/api-server/tests/unit/_with_db/test_api_user.py b/services/api-server/tests/unit/_with_db/test_api_user.py index eca2ea3e70b..72f8fadffda 100644 --- a/services/api-server/tests/unit/_with_db/test_api_user.py +++ b/services/api-server/tests/unit/_with_db/test_api_user.py @@ -10,6 +10,7 @@ import respx from fastapi import FastAPI from models_library.api_schemas_webserver.users import MyProfileGet as WebProfileGet +from pytest_mock import MockType from respx import MockRouter from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.core.settings import ApplicationSettings diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py index 2b264d771a6..314890ba8bc 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py @@ -12,6 +12,7 @@ from faker import Faker from models_library.basic_regex import UUID_RE_BASE from pydantic import TypeAdapter +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from servicelib.common_headers import ( diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py index bc8495389e9..11e9717bf7a 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py @@ -11,6 +11,7 @@ from faker import Faker from models_library.basic_regex import UUID_RE_BASE from pydantic import TypeAdapter +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from simcore_service_api_server._meta import API_VTAG diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index d9ec89dbca5..96eeaa4ee20 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -1,6 +1,7 @@ import logging from models_library.projects import ProjectID +from pydantic import PositiveInt from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs from simcore_postgres_database.utils_repos import transaction_context from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -19,7 +20,7 @@ async def set_project_as_job( *, project_uuid: ProjectID, job_parent_resource_name: str, - ) -> int: + ) -> PositiveInt: async with transaction_context(self.engine, connection) as conn: stmt = ( pg_insert(projects_to_jobs) @@ -36,4 +37,5 @@ async def set_project_as_job( result = await conn.execute(stmt) row = result.one() - return row.id + projects_to_jobs_id: PositiveInt = row.int + return projects_to_jobs_id From 16769e3e0d89757ca93a77d1f551edf20b4f83a7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 22:37:02 +0100 Subject: [PATCH 23/39] cleanup imports --- .../api/routes/solvers_jobs.py | 8 ++++---- .../api/routes/studies_jobs.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 7be222edd58..b7c4ade2cd9 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -12,10 +12,6 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from pydantic.types import PositiveInt -from simcore_service_api_server.api.dependencies.webserver_rpc import ( - get_wb_api_rpc_client, -) -from simcore_service_api_server.services_rpc.wb_api_server import WbApiRpcClient from ...exceptions.backend_errors import ProjectAlreadyStartedError from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES @@ -38,10 +34,14 @@ create_jobstatus_from_task, create_new_project_for_job, ) +from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.application import get_reverse_url_mapper from ..dependencies.authentication import get_current_user_id, get_product_name from ..dependencies.services import get_api_client from ..dependencies.webserver_http import AuthSession, get_webserver_session +from ..dependencies.webserver_rpc import ( + get_wb_api_rpc_client, +) from ._constants import ( FMSG_CHANGELOG_ADDED_IN_VERSION, FMSG_CHANGELOG_CHANGED_IN_VERSION, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index 48b548b4a7f..18a4c77122c 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -15,13 +15,7 @@ from models_library.projects_nodes_io import NodeID from pydantic import PositiveInt from servicelib.logging_utils import log_context -from simcore_service_api_server.api.dependencies.webserver_rpc import ( - get_wb_api_rpc_client, -) -from simcore_service_api_server.services_rpc.wb_api_server import WbApiRpcClient -from ...api.dependencies.authentication import get_current_user_id -from ...api.dependencies.services import get_api_client from ...exceptions.backend_errors import ProjectAlreadyStartedError from ...models.pagination import Page, PaginationParams from ...models.schemas.errors import ErrorGet @@ -50,8 +44,14 @@ get_project_and_file_inputs_from_job_inputs, ) from ...services_http.webserver import AuthSession +from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.application import get_reverse_url_mapper +from ..dependencies.authentication import get_current_user_id +from ..dependencies.services import get_api_client from ..dependencies.webserver_http import get_webserver_session +from ..dependencies.webserver_rpc import ( + get_wb_api_rpc_client, +) from ._common import API_SERVER_DEV_FEATURES_ENABLED from ._constants import FMSG_CHANGELOG_CHANGED_IN_VERSION, FMSG_CHANGELOG_NEW_IN_VERSION from .solvers_jobs import JOBS_STATUS_CODES From e51cee5f204291c090a507e9bdd3d3078672dad3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 22:37:48 +0100 Subject: [PATCH 24/39] undo wrong files --- .../models/schemas/files.py | 50 +++++++++---------- .../services_http/webserver.py | 8 +-- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/files.py b/services/api-server/src/simcore_service_api_server/models/schemas/files.py index 47ea68a4e0c..29cc9aacf0a 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/files.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/files.py @@ -37,28 +37,28 @@ class ClientFile(BaseModel): filename: FileName = Field(..., description="File name") filesize: NonNegativeInt = Field(..., description="File size in bytes") sha256_checksum: SHA256Str = Field(..., description="SHA256 checksum") - destination: Annotated[ - ProgramJobFilePath | None, - Field(..., description="Destination within a program job"), - ] class File(BaseModel): """Represents a file stored on the server side i.e. a unique reference to a file in the cloud.""" - id: Annotated[UUID, Field(description="Resource identifier")] - filename: Annotated[str, Field(description="Name of the file with extension")] - content_type: Annotated[ - str | None, - Field( - description="Guess of type content [EXPERIMENTAL]", validate_default=True - ), - ] = None - sha256_checksum: Annotated[ - SHA256Str | None, - Field(description="SHA256 hash of the file's content", alias="checksum"), - ] = None - e_tag: Annotated[ETag | None, Field(description="S3 entity tag")] = None + # WARNING: from pydantic import File as FileParam + # NOTE: see https://ant.apache.org/manual/Tasks/checksum.html + + id: UUID = Field(..., description="Resource identifier") + + filename: str = Field(..., description="Name of the file with extension") + content_type: str | None = Field( + default=None, + description="Guess of type content [EXPERIMENTAL]", + validate_default=True, + ) + sha256_checksum: SHA256Str | None = Field( + default=None, + description="SHA256 hash of the file's content", + alias="checksum", # alias for backwards compatibility + ) + e_tag: ETag | None = Field(default=None, description="S3 entity tag") model_config = ConfigDict( populate_by_name=True, @@ -163,18 +163,16 @@ def quoted_storage_file_id(self) -> str: class UploadLinks(BaseModel): - abort_upload: Annotated[str, Field()] - complete_upload: Annotated[str, Field()] + abort_upload: str + complete_upload: str class FileUploadData(BaseModel): - chunk_size: Annotated[int, Field(description="Chunk size in bytes")] - urls: Annotated[list[Annotated[AnyHttpUrl, UriSchema()]], Field()] - links: Annotated[UploadLinks, Field()] + chunk_size: NonNegativeInt + urls: list[Annotated[AnyHttpUrl, UriSchema()]] + links: UploadLinks class ClientFileUploadData(BaseModel): - file_id: Annotated[UUID, Field(description="The file resource id")] - upload_schema: Annotated[ - FileUploadData, Field(description="Schema for uploading file") - ] + file_id: UUID = Field(..., description="The file resource id") + upload_schema: FileUploadData = Field(..., description="Schema for uploading file") diff --git a/services/api-server/src/simcore_service_api_server/services_http/webserver.py b/services/api-server/src/simcore_service_api_server/services_http/webserver.py index a522ff9ebdd..8b84ba1af6c 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services_http/webserver.py @@ -186,15 +186,15 @@ async def _page_projects( limit: int, offset: int, show_hidden: bool, - job_parent_resource_name: str | None = None, + search_by_project_name: str | None = None, ) -> Page[ProjectGet]: assert 1 <= limit <= MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE # nosec assert offset >= 0 # nosec optional: dict[str, Any] = {} - if job_parent_resource_name is not None: + if search_by_project_name is not None: optional["filters"] = json_dumps( - {"job_parent_resource_name": job_parent_resource_name} + {"search_by_project_name": search_by_project_name} ) with service_exception_handler( @@ -361,7 +361,7 @@ async def get_projects_w_solver_page( limit=limit, offset=offset, show_hidden=True, - job_parent_resource_name=solver_name, + search_by_project_name=solver_name, ) async def get_projects_page(self, *, limit: int, offset: int): From e5dfe6f1f779bdece34af655d3e568645921a57c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 22:50:20 +0100 Subject: [PATCH 25/39] updates signature --- .../helpers/webserver_rpc_server.py | 10 +++++++- .../api/routes/solvers_jobs.py | 5 +++- .../api/routes/studies_jobs.py | 9 ++++++-- .../services_rpc/wb_api_server.py | 23 ++++++++++--------- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py index 8516eb74f13..8da0eaf66cb 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py @@ -5,18 +5,23 @@ # pylint: disable=unused-variable +from models_library.products import ProductName from models_library.projects import ProjectID -from pydantic import TypeAdapter +from models_library.users import UserID +from pydantic import TypeAdapter, validate_call from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient class WebserverRpcSideEffects: # pylint: disable=no-self-use + @validate_call(config=dict(arbitrary_types_allowed=True)) async def mark_project_as_job( self, rpc_client: RabbitMQRPCClient, *, + product_name: ProductName, + user_id: UserID, project_uuid: ProjectID, job_parent_resource_name: str, ) -> None: @@ -26,4 +31,7 @@ async def mark_project_as_job( assert "/" in job_parent_resource_name # nosec assert not job_parent_resource_name.endswith("/") # nosec + assert product_name + assert user_id + TypeAdapter(ProjectID).validate_python(project_uuid) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index b7c4ade2cd9..b0737356e7f 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -146,7 +146,10 @@ async def create_job( assert job.name == _compose_job_resource_name(solver_key, version, job.id) # nosec await wb_api_rpc.mark_project_as_job( - project_uuid=new_project.uuid, job_parent_resource_name=job.runner_name + product_name=product_name, + user_id=user_id, + project_uuid=new_project.uuid, + job_parent_resource_name=job.runner_name, ) return job diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index 18a4c77122c..0043b5daa70 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -46,7 +46,7 @@ from ...services_http.webserver import AuthSession from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.application import get_reverse_url_mapper -from ..dependencies.authentication import get_current_user_id +from ..dependencies.authentication import get_current_user_id, get_product_name from ..dependencies.services import get_api_client from ..dependencies.webserver_http import get_webserver_session from ..dependencies.webserver_rpc import ( @@ -92,6 +92,8 @@ async def create_study_job( webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + product_name: Annotated[str, Depends(get_product_name)], hidden: Annotated[bool, Query()] = True, x_simcore_parent_project_uuid: ProjectID | None = Header(default=None), x_simcore_parent_node_id: NodeID | None = Header(default=None), @@ -126,7 +128,10 @@ async def create_study_job( ) await wb_api_rpc.mark_project_as_job( - project_uuid=job.id, job_parent_resource_name=job.runner_name + product_name=product_name, + user_id=user_id, + project_uuid=job.id, + job_parent_resource_name=job.runner_name, ) project_inputs = await webserver_api.get_project_inputs(project_id=project.uuid) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 6e45e33bb5c..ca771d913b1 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -6,6 +6,7 @@ from fastapi_pagination import create_page from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.licenses import LicensedItemID +from models_library.products import ProductName from models_library.projects import ProjectID from models_library.resource_tracker_licensed_items_checkouts import ( LicensedItemCheckoutID, @@ -96,7 +97,7 @@ class WbApiRpcClient(SingletonInAppStateMixin): @_exception_mapper(rpc_exception_map={}) async def get_licensed_items( - self, *, product_name: str, page_params: PaginationParams + self, *, product_name: ProductName, page_params: PaginationParams ) -> Page[LicensedItemGet]: licensed_items_page = await _get_licensed_items( rabbitmq_rpc_client=self._client, @@ -112,7 +113,7 @@ async def get_licensed_items( async def get_available_licensed_items_for_wallet( self, *, - product_name: str, + product_name: ProductName, wallet_id: WalletID, user_id: UserID, page_params: PaginationParams, @@ -140,7 +141,7 @@ async def get_available_licensed_items_for_wallet( async def checkout_licensed_item_for_wallet( self, *, - product_name: str, + product_name: ProductName, user_id: UserID, wallet_id: WalletID, licensed_item_id: LicensedItemID, @@ -177,7 +178,7 @@ async def checkout_licensed_item_for_wallet( async def release_licensed_item_for_wallet( self, *, - product_name: str, + product_name: ProductName, user_id: UserID, licensed_item_checkout_id: LicensedItemCheckoutID, ) -> LicensedItemCheckoutGet: @@ -204,16 +205,16 @@ async def ping(self) -> str: return await _ping(self._client) async def mark_project_as_job( - self, project_uuid: ProjectID, job_parent_resource_name: RelativeResourceName + self, + product_name: ProductName, + user_id: UserID, + project_uuid: ProjectID, + job_parent_resource_name: RelativeResourceName, ): - assert not job_parent_resource_name.startswith("/") # nosec - assert "/" in job_parent_resource_name # nosec - assert not job_parent_resource_name.endswith("/") # nosec - - assert project_uuid - await projects_rpc.mark_project_as_job( rpc_client=self._client, + product_name=product_name, + user_id=user_id, project_uuid=project_uuid, job_parent_resource_name=job_parent_resource_name, ) From 5b08bd4b0abcfa287fde1f080769fd4dba9ad142 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 23:17:50 +0100 Subject: [PATCH 26/39] reverted --- .../src/common_library/errors_classes.py | 27 ----------- .../tests/test_errors_classes.py | 48 +------------------ 2 files changed, 1 insertion(+), 74 deletions(-) diff --git a/packages/common-library/src/common_library/errors_classes.py b/packages/common-library/src/common_library/errors_classes.py index d05c64acfd5..dfee557d38c 100644 --- a/packages/common-library/src/common_library/errors_classes.py +++ b/packages/common-library/src/common_library/errors_classes.py @@ -52,30 +52,3 @@ def error_context(self) -> dict[str, Any]: def error_code(self) -> str: assert isinstance(self, Exception), "subclass must be exception" # nosec return create_error_code(self) - - -class NotFoundError(OsparcErrorMixin): - msg_template = "{resource} not found: id='{resource_id}'" - - -class ForbiddenError(OsparcErrorMixin): - msg_template = "Access to {resource} is forbidden: id='{resource_id}'" - - -def make_resource_error( - resource: str, - error_cls: type[OsparcErrorMixin], - base_exception: type[Exception] = Exception, -) -> type[Exception]: - class _ResourceError(error_cls, base_exception): - def __init__(self, **ctx: Any): - ctx.setdefault("resource", resource) - # guesses identifer e.g. project_id, user_id - if resource_id := ctx.get(f"{resource.lower()}_id"): - ctx.setdefault("resource_id", resource_id) - - super().__init__(**ctx) - - resource_class_name = "".join(word.capitalize() for word in resource.split("_")) - _ResourceError.__name__ = f"{resource_class_name}{error_cls.__name__}" - return _ResourceError diff --git a/packages/common-library/tests/test_errors_classes.py b/packages/common-library/tests/test_errors_classes.py index ebc928e607f..808ed09c40d 100644 --- a/packages/common-library/tests/test_errors_classes.py +++ b/packages/common-library/tests/test_errors_classes.py @@ -9,12 +9,7 @@ from typing import Any import pytest -from common_library.errors_classes import ( - ForbiddenError, - NotFoundError, - OsparcErrorMixin, - make_resource_error, -) +from common_library.errors_classes import OsparcErrorMixin def test_get_full_class_name(): @@ -159,44 +154,3 @@ class MyError(OsparcErrorMixin, ValueError): "message": "42 and 'missing=?'", "value": 42, } - - -def test_resource_error_factory(): - ProjectNotFoundError = make_resource_error("project", NotFoundError) - - error_1 = ProjectNotFoundError(resource_id="abc123") - assert "resource_id" in error_1.error_context() - assert error_1.resource_id in error_1.message # type: ignore - - -def test_resource_error_factory_auto_detect_resource_id(): - ProjectForbiddenError = make_resource_error("project", ForbiddenError) - error_2 = ProjectForbiddenError(project_id="abc123", other_id="foo") - assert ( - error_2.resource_id == error_2.project_id # type: ignore - ), "auto-detects project ids as resourceid" - assert error_2.other_id # type: ignore - assert error_2.code == "ForbiddenError.ProjectForbiddenError" - - assert error_2.error_context() == { - "project_id": "abc123", - "other_id": "foo", - "resource": "project", - "resource_id": "abc123", - "message": "Access to project is forbidden: id='abc123'", - "code": "ForbiddenError.ProjectForbiddenError", - } - - -def test_resource_error_factory_different_base_exception(): - - class MyBaseError(Exception): ... - - OtherProjectForbiddenError = make_resource_error( - "other_project", ForbiddenError, MyBaseError - ) - - assert issubclass(OtherProjectForbiddenError, MyBaseError) - - error_3 = OtherProjectForbiddenError(project_id="abc123") - assert error_3.code == "MyBaseError.ForbiddenError.OtherProjectForbiddenError" From e9f4a0cc69589d4129e66fe081e187a9d89e6ba0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 28 Mar 2025 23:23:55 +0100 Subject: [PATCH 27/39] errors --- .../rpc_interfaces/webserver/errors.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py index 803e5ab19f0..dfa2beb7f15 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py @@ -1,14 +1,20 @@ -from common_library.errors_classes import ( - OsparcErrorMixin, -) +from common_library.errors_classes import OsparcErrorMixin +from ..._errors import RPCServerError -class WebServerRpcError(OsparcErrorMixin, Exception): - msg_template = "{details}" + +class WebServerRpcError(RPCServerError): + msg_template = "{domain_error_nessage} [{domain_error_type_name}]" @classmethod def from_domain_error(cls, err: OsparcErrorMixin): - return cls(details=f"{err} [{err.__class__.__name__}]", **err.error_context()) + return cls( + # composes a message + domain_error_nessage=err.message, + domain_error_type_name=f"{err.__class__.__name__}", + # copies context + **err.error_context(), + ) class ProjectNotFoundRpcError(WebServerRpcError): ... From 5722147d02dfc6169f499023bd8bd33725e8b1ba Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Sat, 29 Mar 2025 00:07:42 +0100 Subject: [PATCH 28/39] rpc errors --- .../src/servicelib/rabbitmq/__init__.py | 12 ++--- .../src/servicelib/rabbitmq/_errors.py | 27 ++++++++++-- .../rpc_interfaces/webserver/errors.py | 22 ++-------- .../rabbitmq/test_rabbitmq_rpc_router.py | 44 ++++++++++++++----- 4 files changed, 65 insertions(+), 40 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/__init__.py b/packages/service-library/src/servicelib/rabbitmq/__init__.py index e2a1416d3e3..b2e8b6d0b34 100644 --- a/packages/service-library/src/servicelib/rabbitmq/__init__.py +++ b/packages/service-library/src/servicelib/rabbitmq/__init__.py @@ -5,6 +5,7 @@ from ._constants import BIND_TO_ALL_TOPICS, RPC_REQUEST_DEFAULT_TIMEOUT_S from ._errors import ( RemoteMethodNotRegisteredError, + RPCInterfaceError, RPCNotInitializedError, RPCServerError, ) @@ -14,18 +15,19 @@ __all__: tuple[str, ...] = ( "BIND_TO_ALL_TOPICS", + "RPC_REQUEST_DEFAULT_TIMEOUT_S", "ConsumerTag", "ExchangeName", - "is_rabbitmq_responsive", "QueueName", - "RabbitMQClient", - "RabbitMQRPCClient", - "RemoteMethodNotRegisteredError", - "RPC_REQUEST_DEFAULT_TIMEOUT_S", + "RPCInterfaceError", "RPCNamespace", "RPCNotInitializedError", "RPCRouter", "RPCServerError", + "RabbitMQClient", + "RabbitMQRPCClient", + "RemoteMethodNotRegisteredError", + "is_rabbitmq_responsive", "wait_till_rabbitmq_responsive", ) diff --git a/packages/service-library/src/servicelib/rabbitmq/_errors.py b/packages/service-library/src/servicelib/rabbitmq/_errors.py index c105c2b8ff3..ce58b62fd5c 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/_errors.py @@ -5,17 +5,16 @@ _ERROR_PREFIX: Final[str] = "rabbitmq_error" -class BaseRPCError(OsparcErrorMixin, RuntimeError): - ... +class BaseRPCError(OsparcErrorMixin, RuntimeError): ... class RPCNotInitializedError(BaseRPCError): - code = f"{_ERROR_PREFIX}.not_started" # type: ignore[assignment] + code = f"{_ERROR_PREFIX}.not_started" # type: ignore[assignment] msg_template = "Please check that the RabbitMQ RPC backend was initialized!" class RemoteMethodNotRegisteredError(BaseRPCError): - code = f"{_ERROR_PREFIX}.remote_not_registered" # type: ignore[assignment] + code = f"{_ERROR_PREFIX}.remote_not_registered" # type: ignore[assignment] msg_template = ( "Could not find a remote method named: '{method_name}'. " "Message from remote server was returned: {incoming_message}. " @@ -27,3 +26,23 @@ class RPCServerError(BaseRPCError): "While running method '{method_name}' raised " "'{exc_type}': '{exc_message}'\n{traceback}" ) + + +class RPCInterfaceError(RPCServerError): + """ + Base class for RPC interface exceptions. + + Avoid using domain exceptions directly; if a one-to-one mapping is required, + prefer using the `from_domain_error` transformation function. + """ + + msg_template = "{domain_error_message} [{domain_error_code}]" + + @classmethod + def from_domain_error(cls, err: OsparcErrorMixin): + domain_err_ctx = err.error_context() + return cls( + domain_error_message=domain_err_ctx.pop("message"), + domain_error_code=domain_err_ctx.pop("code"), + **domain_err_ctx, # same context as domain + ) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py index dfa2beb7f15..e0c3fc2419a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py @@ -1,23 +1,7 @@ -from common_library.errors_classes import OsparcErrorMixin +from ..._errors import RPCInterfaceError -from ..._errors import RPCServerError +class ProjectNotFoundRpcError(RPCInterfaceError): ... -class WebServerRpcError(RPCServerError): - msg_template = "{domain_error_nessage} [{domain_error_type_name}]" - @classmethod - def from_domain_error(cls, err: OsparcErrorMixin): - return cls( - # composes a message - domain_error_nessage=err.message, - domain_error_type_name=f"{err.__class__.__name__}", - # copies context - **err.error_context(), - ) - - -class ProjectNotFoundRpcError(WebServerRpcError): ... - - -class ProjectForbiddenRpcError(WebServerRpcError): ... +class ProjectForbiddenRpcError(RPCInterfaceError): ... diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc_router.py b/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc_router.py index 4d0c99939ea..913d5854cd9 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc_router.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc_router.py @@ -2,12 +2,15 @@ # pylint:disable=unused-argument from collections.abc import Awaitable, Callable +from typing import cast import pytest +from common_library.errors_classes import OsparcErrorMixin from faker import Faker from models_library.rabbitmq_basic_types import RPCMethodName from servicelib.rabbitmq import ( RabbitMQRPCClient, + RPCInterfaceError, RPCNamespace, RPCRouter, RPCServerError, @@ -18,15 +21,21 @@ ] -router = RPCRouter() +class MyServiceError(OsparcErrorMixin, Exception): ... -class MyBaseError(Exception): - ... +class MyDomainError(MyServiceError): + msg_template = "This could happen" -class MyExpectedError(MyBaseError): - ... +def raise_my_expected_error(): + raise MyDomainError(user_id=33, project_id=3) + + +router = RPCRouter() # Server-side + + +class MyExpectedRpcError(RPCInterfaceError): ... @router.expose() @@ -41,10 +50,13 @@ async def an_int_method(a_global_arg: str, *, a_global_kwarg: str) -> int: return 34 -@router.expose(reraise_if_error_type=(MyBaseError,)) -async def raising_expected_error(a_global_arg: str, *, a_global_kwarg: str) -> int: - msg = "This could happen" - raise MyExpectedError(msg) +@router.expose(reraise_if_error_type=(MyExpectedRpcError,)) +async def raising_expected_error(a_global_arg: str, *, a_global_kwarg: str): + try: + raise_my_expected_error() + except MyDomainError as exc: + # NOTE how it is adapted from a domain exception to an interface exception + raise MyExpectedRpcError.from_domain_error(exc) from exc @router.expose() @@ -55,7 +67,7 @@ async def raising_unexpected_error(a_global_arg: str, *, a_global_kwarg: str) -> @pytest.fixture def router_namespace(faker: Faker) -> RPCNamespace: - return faker.pystr() + return cast(RPCNamespace, faker.pystr()) async def test_exposed_methods( @@ -100,10 +112,18 @@ async def test_exposed_methods( assert "builtins.ValueError" in f"{exc_info.value}" # This error was classified int he interface - with pytest.raises(MyBaseError) as exc_info: + with pytest.raises(RPCInterfaceError) as exc_info: await rpc_client.request( router_namespace, RPCMethodName(raising_expected_error.__name__), ) - assert isinstance(exc_info.value, MyExpectedError) + assert isinstance(exc_info.value, MyExpectedRpcError) + assert exc_info.value.error_context() == { + "message": "This could happen [MyServiceError.MyDomainError]", + "code": "RuntimeError.BaseRPCError.RPCServerError", + "domain_error_message": "This could happen", + "domain_error_code": "MyServiceError.MyDomainError", + "user_id": 33, + "project_id": 3, + } From 22713b373aaf618f8818bad2e6d507f745e10962 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Sat, 29 Mar 2025 13:28:01 +0100 Subject: [PATCH 29/39] id --- .../src/simcore_service_webserver/projects/_jobs_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index 96eeaa4ee20..a2cb38d547a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -37,5 +37,5 @@ async def set_project_as_job( result = await conn.execute(stmt) row = result.one() - projects_to_jobs_id: PositiveInt = row.int + projects_to_jobs_id: PositiveInt = row.id return projects_to_jobs_id From 0627562d5cd519174757f2b67291c5242789d577 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Mar 2025 10:21:26 +0200 Subject: [PATCH 30/39] cleanup --- .../src/simcore_service_api_server/models/api_resources.py | 2 +- .../projects/_controller/projects_rest.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/models/api_resources.py b/services/api-server/src/simcore_service_api_server/models/api_resources.py index aac72872a45..4f9b90c19a6 100644 --- a/services/api-server/src/simcore_service_api_server/models/api_resources.py +++ b/services/api-server/src/simcore_service_api_server/models/api_resources.py @@ -5,7 +5,7 @@ from pydantic import Field, TypeAdapter from pydantic.types import StringConstraints -# RESOURCE NAMES https://cloud.google.com/apis/design/resource_names +# RESOURCE NAMES https://google.aip.dev/122 # # # API Service Name Collection ID Resource ID Collection ID Resource ID diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index 407ae184905..f20360d1cc0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -158,7 +158,6 @@ async def list_projects(request: web.Request): workspace_id=query_params.workspace_id, search_by_multi_columns=query_params.search, search_by_project_name=query_params.filters.search_by_project_name, - # TODO: query_params.filters.job_parent_resource_name offset=query_params.offset, limit=query_params.limit, order_by=OrderBy.model_construct(**query_params.order_by.model_dump()), @@ -199,7 +198,6 @@ async def list_projects_full_search(request: web.Request): tag_ids_list=tag_ids_list, search_by_multi_columns=query_params.text, search_by_project_name=query_params.filters.search_by_project_name, - # TODO: query_params.filters.job_parent_resource_name offset=query_params.offset, limit=query_params.limit, order_by=OrderBy.model_construct(**query_params.order_by.model_dump()), From eaa9f6b882034b6e8aedadcf63ace2f51650028a Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Mar 2025 10:32:54 +0200 Subject: [PATCH 31/39] api-server tests fixture --- .../helpers/webserver_rpc_server.py | 2 +- .../tests/unit/api_solvers/conftest.py | 16 +++++++++----- .../api_solvers/test_api_routers_solvers.py | 4 ++-- .../test_api_routers_solvers_jobs.py | 16 +++++++------- .../test_api_routers_solvers_jobs_delete.py | 22 +++++++++---------- .../test_api_routers_solvers_jobs_logs.py | 2 +- .../test_api_routers_solvers_jobs_metadata.py | 12 +++++----- 7 files changed, 40 insertions(+), 34 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py index 8da0eaf66cb..bf21eefef1b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py @@ -15,7 +15,7 @@ class WebserverRpcSideEffects: # pylint: disable=no-self-use - @validate_call(config=dict(arbitrary_types_allowed=True)) + @validate_call(config={"arbitrary_types_allowed": True}) async def mark_project_as_job( self, rpc_client: RabbitMQRPCClient, diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index d27dc47fc84..994cfe7b6b0 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -32,7 +32,7 @@ def solver_version() -> str: @pytest.fixture -def mocked_webserver_service_api( +def mocked_webserver_rest_api( app: FastAPI, mocked_webserver_service_api_base: MockRouter, patch_webserver_long_running_project_tasks: Callable[[MockRouter], MockRouter], @@ -46,11 +46,17 @@ def mocked_webserver_service_api( @pytest.fixture -def mocked_rpc_webserver_service_api( +def mocked_webserver_rpc_api( app: FastAPI, mocker: MockerFixture ) -> dict[str, MockType]: - # from simcore_service_api_server.services_rpc.wb_api_server import projects_rpc from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc + from simcore_service_api_server.services_rpc import wb_api_server + + # NOTE: mock_missing_plugins patches `setup_rabbitmq` + try: + wb_api_server.WbApiRpcClient.get_from_app_state(app) + except AttributeError: + wb_api_server.setup(app, mocker.MagicMock()) settings: ApplicationSettings = app.state.settings assert settings.API_SERVER_WEBSERVER @@ -67,7 +73,7 @@ def mocked_rpc_webserver_service_api( @pytest.fixture -def mocked_catalog_service_api( +def mocked_catalog_rest_api( app: FastAPI, mocked_catalog_service_api_base: MockRouter, catalog_service_openapi_specs: dict[str, Any], @@ -113,7 +119,7 @@ def mocked_catalog_service_api( @pytest.fixture -async def mocked_directorv2_service( +async def mocked_directorv2_rest_api( mocked_directorv2_service_api_base, ) -> AsyncIterable[MockRouter]: stop_time: Final[datetime] = datetime.now() + timedelta(seconds=5) diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py index d4a6cf80a76..c07c725fd76 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py @@ -18,7 +18,7 @@ @pytest.mark.skip(reason="Still under development. Currently using fake implementation") async def test_list_solvers( client: httpx.AsyncClient, - mocked_catalog_service_api: MockRouter, + mocked_catalog_rest_api: MockRouter, mocker: MockFixture, ): warn = mocker.patch.object( @@ -63,7 +63,7 @@ async def test_list_solvers( async def test_list_solver_ports( - mocked_catalog_service_api: MockRouter, + mocked_catalog_rest_api: MockRouter, client: httpx.AsyncClient, auth: httpx.BasicAuth, ): diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py index 2d6c10bb13f..cc95649cae7 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py @@ -204,10 +204,10 @@ async def test_run_solver_job( client: httpx.AsyncClient, directorv2_service_openapi_specs: dict[str, Any], catalog_service_openapi_specs: dict[str, Any], - mocked_catalog_service_api: MockRouter, + mocked_catalog_rest_api: MockRouter, mocked_directorv2_service_api: MockRouter, - mocked_webserver_service_api: MockRouter, - mocked_rpc_webserver_service_api: dict[str, MockType], + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], auth: httpx.BasicAuth, project_id: str, solver_key: str, @@ -280,7 +280,7 @@ async def test_run_solver_job( ), ) - mocked_webserver_service_api.post( + mocked_webserver_rest_api.post( path__regex=r"^/v0/computations/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-(3|4|5)[0-9a-fA-F]{3}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}:start$", name="webserver_start_job", ).respond( @@ -319,7 +319,7 @@ async def test_run_solver_job( if "boot-options" in e ) - mocked_catalog_service_api.get( + mocked_catalog_rest_api.get( # path__regex=r"/services/(?P[\w-]+)/(?P[0-9\.]+)", path=f"/v0/services/{solver_key}/{solver_version}", name="get_service_v0_services__service_key___service_version__get", @@ -355,9 +355,9 @@ async def test_run_solver_job( ) assert resp.status_code == status.HTTP_201_CREATED - assert mocked_webserver_service_api["create_projects"].called - assert mocked_webserver_service_api["get_task_status"].called - assert mocked_webserver_service_api["get_task_result"].called + assert mocked_webserver_rest_api["create_projects"].called + assert mocked_webserver_rest_api["get_task_status"].called + assert mocked_webserver_rest_api["get_task_result"].called job = Job.model_validate(resp.json()) diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py index 314890ba8bc..c5fad3f6d2a 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py @@ -33,8 +33,8 @@ class MockedBackendApiDict(TypedDict): @pytest.fixture def mocked_backend_services_apis_for_delete_non_existing_project( - mocked_webserver_service_api: MockRouter, - mocked_rpc_webserver_service_api: dict[str, MockType], + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "delete_project_not_found.json" @@ -51,12 +51,12 @@ def _response(request: httpx.Request, project_id: str): status_code=capture.status_code, json=capture.response_body ) - mocked_webserver_service_api.delete( + mocked_webserver_rest_api.delete( path__regex=rf"/projects/(?P{UUID_RE_BASE})$", name="delete_project", ).mock(side_effect=_response) - return MockedBackendApiDict(webserver=mocked_webserver_service_api, catalog=None) + return MockedBackendApiDict(webserver=mocked_webserver_rest_api, catalog=None) @pytest.mark.acceptance_test( @@ -86,9 +86,9 @@ async def test_delete_non_existing_solver_job( @pytest.fixture def mocked_backend_services_apis_for_create_and_delete_solver_job( - mocked_webserver_service_api: MockRouter, - mocked_rpc_webserver_service_api: dict[str, MockType], - mocked_catalog_service_api: MockRouter, + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_catalog_rest_api: MockRouter, project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "on_create_job.json" @@ -101,7 +101,7 @@ def mocked_backend_services_apis_for_create_and_delete_solver_job( capture = captures[0] assert capture.host == "catalog" assert capture.method == "GET" - mocked_catalog_service_api.request( + mocked_catalog_rest_api.request( method=capture.method, path=capture.path, name="get_service" # GET service ).respond(status_code=capture.status_code, json=capture.response_body) @@ -109,14 +109,14 @@ def mocked_backend_services_apis_for_create_and_delete_solver_job( assert capture.host == "webserver" assert capture.method == "DELETE" - mocked_webserver_service_api.delete( + mocked_webserver_rest_api.delete( path__regex=rf"/projects/(?P{UUID_RE_BASE})$", name="delete_project", ).respond(status_code=capture.status_code, json=capture.response_body) return MockedBackendApiDict( - catalog=mocked_catalog_service_api, - webserver=mocked_webserver_service_api, + catalog=mocked_catalog_rest_api, + webserver=mocked_webserver_rest_api, ) diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py index 80a7176ca85..ca0c433e4e9 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py @@ -98,7 +98,7 @@ async def test_log_streaming( solver_version: str, fake_log_distributor, fake_project_for_streaming: ProjectGet, - mocked_directorv2_service: MockRouter, + mocked_directorv2_rest_api: MockRouter, disconnect: bool, ): diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py index 11e9717bf7a..1693e579fa8 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py @@ -38,9 +38,9 @@ def _as_path_regex(initial_path: str): @pytest.fixture def mocked_backend( - mocked_webserver_service_api: MockRouter, - mocked_rpc_webserver_service_api: dict[str, MockType], - mocked_catalog_service_api: MockRouter, + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_catalog_rest_api: MockRouter, project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "for_test_get_and_update_job_metadata.json" @@ -54,7 +54,7 @@ def mocked_backend( capture = captures["get_service"] assert capture.host == "catalog" - mocked_catalog_service_api.request( + mocked_catalog_rest_api.request( method=capture.method, path=capture.path, name=capture.name, @@ -68,7 +68,7 @@ def mocked_backend( assert capture.host == "webserver" capture_path_regex = _as_path_regex(capture.path.removeprefix("/v0")) - route = mocked_webserver_service_api.request( + route = mocked_webserver_rest_api.request( method=capture.method, path__regex=capture_path_regex, name=capture.name, @@ -88,7 +88,7 @@ def mocked_backend( ) return MockedBackendApiDict( - webserver=mocked_webserver_service_api, catalog=mocked_catalog_service_api + webserver=mocked_webserver_rest_api, catalog=mocked_catalog_rest_api ) From 908c7acac67811b1092e93ec2dd3ef52214ba87f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Mar 2025 10:39:52 +0200 Subject: [PATCH 32/39] fixes api-server tests --- services/api-server/tests/unit/api_solvers/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index 994cfe7b6b0..ab071ae752a 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -17,6 +17,7 @@ from pytest_simcore.helpers import faker_catalog from pytest_simcore.helpers.webserver_rpc_server import WebserverRpcSideEffects from respx import MockRouter +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from simcore_service_api_server.core.settings import ApplicationSettings from simcore_service_api_server.services_http.director_v2 import ComputationTaskGet @@ -56,7 +57,9 @@ def mocked_webserver_rpc_api( try: wb_api_server.WbApiRpcClient.get_from_app_state(app) except AttributeError: - wb_api_server.setup(app, mocker.MagicMock()) + wb_api_server.setup( + app, RabbitMQRPCClient("fake_rpc_client", settings=mocker.MagicMock()) + ) settings: ApplicationSettings = app.state.settings assert settings.API_SERVER_WEBSERVER From b5ac4e250ef3e0116f936b3e91b8751674066ebf Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Mar 2025 11:53:04 +0200 Subject: [PATCH 33/39] mocks --- .../tests/unit/_with_db/test_product.py | 8 ++-- .../tests/unit/api_solvers/conftest.py | 16 +++---- .../test_api_routers_solvers_jobs.py | 4 +- .../test_api_routers_solvers_jobs_logs.py | 6 +-- .../test_api_routers_solvers_jobs_read.py | 12 ++--- .../test_api_routers_studies_jobs_metadata.py | 10 +++-- .../api_studies/test_api_routes_studies.py | 32 +++++++------- .../test_api_routes_studies_jobs.py | 38 ++++++++-------- .../api_studies/test_api_studies_mocks.py | 4 +- services/api-server/tests/unit/conftest.py | 8 ++-- .../tests/unit/test_api__study_workflows.py | 18 ++++---- .../api-server/tests/unit/test_api_files.py | 18 ++++---- .../api-server/tests/unit/test_api_health.py | 8 ++-- .../tests/unit/test_api_solver_jobs.py | 44 +++++++++---------- .../api-server/tests/unit/test_api_solvers.py | 4 +- .../api-server/tests/unit/test_api_wallets.py | 8 ++-- .../api-server/tests/unit/test_credits.py | 4 +- .../tests/unit/test_services_directorv2.py | 4 +- .../tests/unit/test_services_rabbitmq.py | 4 +- 19 files changed, 127 insertions(+), 123 deletions(-) diff --git a/services/api-server/tests/unit/_with_db/test_product.py b/services/api-server/tests/unit/_with_db/test_product.py index 274869d094a..acf20949618 100644 --- a/services/api-server/tests/unit/_with_db/test_product.py +++ b/services/api-server/tests/unit/_with_db/test_product.py @@ -26,7 +26,7 @@ async def test_product_webserver( client: httpx.AsyncClient, - mocked_webserver_service_api_base: respx.MockRouter, + mocked_webserver_rest_api_base: respx.MockRouter, create_fake_api_keys: Callable[[PositiveInt], AsyncGenerator[ApiKeyInDB, None]], faker: Faker, ) -> None: @@ -64,7 +64,7 @@ def _check_key_product_compatibility(request: httpx.Request, **kwargs): ), ) - wallet_get_mock = mocked_webserver_service_api_base.get( + wallet_get_mock = mocked_webserver_rest_api_base.get( path__regex=r"/wallets/(?P[-+]?\d+)" ).mock(side_effect=_check_key_product_compatibility) @@ -80,7 +80,7 @@ def _check_key_product_compatibility(request: httpx.Request, **kwargs): async def test_product_catalog( client: httpx.AsyncClient, - mocked_catalog_service_api_base: respx.MockRouter, + mocked_catalog_rest_api_base: respx.MockRouter, create_fake_api_keys: Callable[[PositiveInt], AsyncGenerator[ApiKeyInDB, None]], ) -> None: assert client @@ -99,7 +99,7 @@ def _get_service_side_effect(request: httpx.Request, **kwargs): assert key.product_name == received_product return httpx.Response(status_code=status.HTTP_200_OK) - respx_mock = mocked_catalog_service_api_base.get( + respx_mock = mocked_catalog_rest_api_base.get( r"/v0/services/simcore%2Fservices%2Fcomp%2Fisolve/2.0.24" ).mock(side_effect=_get_service_side_effect) diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index ab071ae752a..782f2cf87e7 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -35,15 +35,15 @@ def solver_version() -> str: @pytest.fixture def mocked_webserver_rest_api( app: FastAPI, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, patch_webserver_long_running_project_tasks: Callable[[MockRouter], MockRouter], ) -> MockRouter: settings: ApplicationSettings = app.state.settings assert settings.API_SERVER_WEBSERVER - patch_webserver_long_running_project_tasks(mocked_webserver_service_api_base) + patch_webserver_long_running_project_tasks(mocked_webserver_rest_api_base) - return mocked_webserver_service_api_base + return mocked_webserver_rest_api_base @pytest.fixture @@ -78,10 +78,10 @@ def mocked_webserver_rpc_api( @pytest.fixture def mocked_catalog_rest_api( app: FastAPI, - mocked_catalog_service_api_base: MockRouter, + mocked_catalog_rest_api_base: MockRouter, catalog_service_openapi_specs: dict[str, Any], ) -> MockRouter: - respx_mock = mocked_catalog_service_api_base + respx_mock = mocked_catalog_rest_api_base openapi = deepcopy(catalog_service_openapi_specs) schemas = openapi["components"]["schemas"] @@ -123,7 +123,7 @@ def mocked_catalog_rest_api( @pytest.fixture async def mocked_directorv2_rest_api( - mocked_directorv2_service_api_base, + mocked_directorv2_rest_api_base, ) -> AsyncIterable[MockRouter]: stop_time: Final[datetime] = datetime.now() + timedelta(seconds=5) @@ -138,7 +138,7 @@ def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: status_code=status.HTTP_200_OK, json=jsonable_encoder(task) ) - mocked_directorv2_service_api_base.get( + mocked_directorv2_rest_api_base.get( path__regex=r"/v2/computations/(?P[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" ).mock(side_effect=_get_computation) - return mocked_directorv2_service_api_base + return mocked_directorv2_rest_api_base diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py index cc95649cae7..a167f2b4603 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py @@ -86,7 +86,7 @@ def presigned_download_link( def mocked_directorv2_service_api( app: FastAPI, presigned_download_link: AnyUrl, - mocked_directorv2_service_api_base: MockRouter, + mocked_directorv2_rest_api_base: MockRouter, directorv2_service_openapi_specs: dict[str, Any], ): settings: ApplicationSettings = app.state.settings @@ -94,7 +94,7 @@ def mocked_directorv2_service_api( oas = directorv2_service_openapi_specs # pylint: disable=not-context-manager - respx_mock = mocked_directorv2_service_api_base + respx_mock = mocked_directorv2_rest_api_base # check that what we emulate, actually still exists path = "/v2/computations/{project_id}/tasks/-/logfile" assert path in oas["paths"] diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py index ca0c433e4e9..901eea6277c 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py @@ -127,15 +127,15 @@ async def test_log_streaming( @pytest.fixture async def mock_job_not_found( - mocked_directorv2_service_api_base: MockRouter, + mocked_directorv2_rest_api_base: MockRouter, ) -> MockRouter: def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: return httpx.Response(status_code=status.HTTP_404_NOT_FOUND) - mocked_directorv2_service_api_base.get( + mocked_directorv2_rest_api_base.get( path__regex=r"/v2/computations/(?P[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" ).mock(side_effect=_get_computation) - return mocked_directorv2_service_api_base + return mocked_directorv2_rest_api_base async def test_logstreaming_job_not_found_exception( diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py index b51c580eb82..09fc53194f6 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py @@ -23,8 +23,8 @@ class MockBackendRouters(NamedTuple): @pytest.fixture def mocked_backend( - mocked_webserver_service_api_base: MockRouter, - mocked_catalog_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, + mocked_catalog_rest_api_base: MockRouter, project_tests_dir: Path, ) -> MockBackendRouters: mock_name = "on_list_jobs.json" @@ -35,7 +35,7 @@ def mocked_backend( capture = captures[0] assert capture.host == "catalog" assert capture.name == "get_service" - mocked_catalog_service_api_base.request( + mocked_catalog_rest_api_base.request( method=capture.method, path=capture.path, name=capture.name, @@ -47,7 +47,7 @@ def mocked_backend( capture = captures[1] assert capture.host == "webserver" assert capture.name == "list_projects" - mocked_webserver_service_api_base.request( + mocked_webserver_rest_api_base.request( method=capture.method, name=capture.name, path=capture.path, @@ -57,8 +57,8 @@ def mocked_backend( ) return MockBackendRouters( - catalog=mocked_catalog_service_api_base, - webserver=mocked_webserver_service_api_base, + catalog=mocked_catalog_rest_api_base, + webserver=mocked_webserver_rest_api_base, ) diff --git a/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py b/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py index d1fae307589..b1cc4c80073 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py +++ b/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py @@ -13,6 +13,7 @@ import pytest from fastapi.encoders import jsonable_encoder from pydantic import TypeAdapter +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from pytest_simcore.helpers.httpx_calls_capture_parameters import PathDescription from respx import MockRouter @@ -33,7 +34,8 @@ class MockedBackendApiDict(TypedDict): @pytest.fixture def mocked_backend( project_tests_dir: Path, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], ) -> MockedBackendApiDict | None: # load captures = { @@ -85,7 +87,7 @@ def mocked_backend( if group: # mock this entrypoint using https://lundberg.github.io/respx/guide/#iterable cc = [c] + [captures[_] for _ in group] - mocked_webserver_service_api_base.request( + mocked_webserver_rest_api_base.request( method=c.method.upper(), url=None, path__regex=f"^{c.path.to_path_regex()}$", @@ -94,7 +96,7 @@ def mocked_backend( side_effect=[_.as_response() for _ in cc], ) else: - mocked_webserver_service_api_base.request( + mocked_webserver_rest_api_base.request( method=c.method.upper(), url=None, path__regex=f"^{c.path.to_path_regex()}$", @@ -102,7 +104,7 @@ def mocked_backend( ).mock(return_value=c.as_response()) return MockedBackendApiDict( - webserver=mocked_webserver_service_api_base, + webserver=mocked_webserver_rest_api_base, ) diff --git a/services/api-server/tests/unit/api_studies/test_api_routes_studies.py b/services/api-server/tests/unit/api_studies/test_api_routes_studies.py index d5369bb0314..6c289763d7b 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routes_studies.py +++ b/services/api-server/tests/unit/api_studies/test_api_routes_studies.py @@ -13,6 +13,7 @@ from faker import Faker from fastapi import status from pydantic import TypeAdapter +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from servicelib.common_headers import ( @@ -33,7 +34,8 @@ class MockedBackendApiDict(TypedDict): @pytest.fixture def mocked_backend( - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "for_test_api_routes_studies.json" @@ -56,7 +58,7 @@ def mocked_backend( capture = captures[name] assert capture.host == "webserver" - route = mocked_webserver_service_api_base.request( + route = mocked_webserver_rest_api_base.request( method=capture.method, path__regex=capture.path.removeprefix("/v0") + "$", name=capture.name, @@ -65,9 +67,7 @@ def mocked_backend( json=capture.response_body, ) print(route) - return MockedBackendApiDict( - webserver=mocked_webserver_service_api_base, catalog=None - ) + return MockedBackendApiDict(webserver=mocked_webserver_rest_api_base, catalog=None) @pytest.mark.acceptance_test( @@ -124,13 +124,13 @@ async def test_studies_read_workflow( async def test_list_study_ports( client: httpx.AsyncClient, auth: httpx.BasicAuth, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, fake_study_ports: list[dict[str, Any]], study_id: StudyID, ): # Mocks /projects/{*}/metadata/ports - mocked_webserver_service_api_base.get( + mocked_webserver_rest_api_base.get( path__regex=r"/projects/(?P[\w-]+)/metadata/ports$", name="list_project_metadata_ports", ).respond( @@ -155,15 +155,15 @@ async def test_clone_study( client: httpx.AsyncClient, auth: httpx.BasicAuth, study_id: StudyID, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, patch_webserver_long_running_project_tasks: Callable[[MockRouter], MockRouter], parent_project_id: UUID | None, parent_node_id: UUID | None, ): # Mocks /projects - patch_webserver_long_running_project_tasks(mocked_webserver_service_api_base) + patch_webserver_long_running_project_tasks(mocked_webserver_rest_api_base) - callback = mocked_webserver_service_api_base["create_projects"].side_effect + callback = mocked_webserver_rest_api_base["create_projects"].side_effect assert callback is not None def clone_project_side_effect(request: httpx.Request): @@ -179,9 +179,9 @@ def clone_project_side_effect(request: httpx.Request): assert _parent_node_id == f"{parent_node_id}" return callback(request) - mocked_webserver_service_api_base[ - "create_projects" - ].side_effect = clone_project_side_effect + mocked_webserver_rest_api_base["create_projects"].side_effect = ( + clone_project_side_effect + ) _headers = {} if parent_project_id is not None: @@ -192,7 +192,7 @@ def clone_project_side_effect(request: httpx.Request): f"/{API_VTAG}/studies/{study_id}:clone", headers=_headers, auth=auth ) - assert mocked_webserver_service_api_base["create_projects"].called + assert mocked_webserver_rest_api_base["create_projects"].called assert resp.status_code == status.HTTP_201_CREATED @@ -201,11 +201,11 @@ async def test_clone_study_not_found( client: httpx.AsyncClient, auth: httpx.BasicAuth, faker: Faker, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, patch_webserver_long_running_project_tasks: Callable[[MockRouter], MockRouter], ): # Mocks /projects - mocked_webserver_service_api_base.post( + mocked_webserver_rest_api_base.post( path__regex=r"/projects", name="project_clone", ).respond( diff --git a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py index 811818a8939..6a3d996809d 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py +++ b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py @@ -14,6 +14,7 @@ import respx from faker import Faker from fastapi import status +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import ( CreateRespxMockCallback, HttpApiCallCaptureModel, @@ -37,7 +38,8 @@ async def test_studies_jobs_workflow( client: httpx.AsyncClient, auth: httpx.BasicAuth, - mocked_webserver_service_api_base: respx.MockRouter, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], study_id: StudyID, ): # get_study @@ -121,8 +123,8 @@ async def test_studies_jobs_workflow( async def test_start_stop_delete_study_job( client: httpx.AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -154,8 +156,8 @@ def _side_effect_with_project_id( create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, ], capture_path=capture_file, side_effects_callbacks=[_side_effect_no_project_id] @@ -200,8 +202,8 @@ def _check_response(response: httpx.Response, status_code: int): @pytest.mark.parametrize("hidden", [True, False]) async def test_create_study_job( client: httpx.AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -251,8 +253,8 @@ def _default_side_effect( create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, ], capture_path=_capture_file, side_effects_callbacks=[_default_side_effect] * 5, @@ -279,7 +281,7 @@ async def test_get_study_job_outputs( client: httpx.AsyncClient, fake_study_id: UUID, auth: httpx.BasicAuth, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, ): job_id = "cfe9a77a-f71e-11ee-8fca-0242ac140008" @@ -316,7 +318,7 @@ async def test_get_study_job_outputs( "status_code": 200, } - mocked_webserver_service_api_base.get( + mocked_webserver_rest_api_base.get( path=capture["path"]["path"].format(project_id=job_id) ).respond( status_code=capture["status_code"], @@ -336,8 +338,8 @@ async def test_get_study_job_outputs( async def test_get_job_logs( client: httpx.AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -347,7 +349,7 @@ async def test_get_job_logs( create_respx_mock_from_capture( respx_mocks=[ - mocked_directorv2_service_api_base, + mocked_directorv2_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "get_study_job_logs.json", side_effects_callbacks=[], @@ -363,8 +365,8 @@ async def test_get_job_logs( async def test_get_study_outputs( client: httpx.AsyncClient, create_respx_mock_from_capture: CreateRespxMockCallback, - mocked_directorv2_service_api_base, - mocked_webserver_service_api_base, + mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base, auth: httpx.BasicAuth, project_tests_dir: Path, ): @@ -373,8 +375,8 @@ async def test_get_study_outputs( create_respx_mock_from_capture( respx_mocks=[ - mocked_directorv2_service_api_base, - mocked_webserver_service_api_base, + mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "get_job_outputs.json", side_effects_callbacks=[], diff --git a/services/api-server/tests/unit/api_studies/test_api_studies_mocks.py b/services/api-server/tests/unit/api_studies/test_api_studies_mocks.py index df7f10dc3ce..4727b0f87f1 100644 --- a/services/api-server/tests/unit/api_studies/test_api_studies_mocks.py +++ b/services/api-server/tests/unit/api_studies/test_api_studies_mocks.py @@ -12,7 +12,7 @@ def test_mocked_webserver_service_api( app: FastAPI, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, services_mocks_enabled: bool, ): if not services_mocks_enabled: @@ -34,4 +34,4 @@ def test_mocked_webserver_service_api( assert resp.status_code == status.HTTP_200_OK assert resp.json() - mocked_webserver_service_api_base.assert_all_called() + mocked_webserver_rest_api_base.assert_all_called() diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 83db84a5d2c..58cc3aa083e 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -256,7 +256,7 @@ def catalog_service_openapi_specs(osparc_simcore_services_dir: Path) -> dict[str @pytest.fixture -def mocked_directorv2_service_api_base( +def mocked_directorv2_rest_api_base( app: FastAPI, directorv2_service_openapi_specs: dict[str, Any], services_mocks_enabled: bool, @@ -291,7 +291,7 @@ def mocked_directorv2_service_api_base( @pytest.fixture -def mocked_webserver_service_api_base( +def mocked_webserver_rest_api_base( app: FastAPI, webserver_service_openapi_specs: dict[str, Any], services_mocks_enabled: bool, @@ -333,7 +333,7 @@ def mocked_webserver_service_api_base( @pytest.fixture -def mocked_storage_service_api_base( +def mocked_storage_rest_api_base( app: FastAPI, storage_service_openapi_specs: dict[str, Any], faker: Faker, @@ -393,7 +393,7 @@ def mocked_storage_service_api_base( @pytest.fixture -def mocked_catalog_service_api_base( +def mocked_catalog_rest_api_base( app: FastAPI, catalog_service_openapi_specs: dict[str, Any], services_mocks_enabled: bool, diff --git a/services/api-server/tests/unit/test_api__study_workflows.py b/services/api-server/tests/unit/test_api__study_workflows.py index dc6abbdadaf..07653159d63 100644 --- a/services/api-server/tests/unit/test_api__study_workflows.py +++ b/services/api-server/tests/unit/test_api__study_workflows.py @@ -204,9 +204,9 @@ class MockedBackendApiDict(TypedDict): @pytest.fixture def mocked_backend( project_tests_dir: Path, - mocked_webserver_service_api_base: MockRouter, - mocked_storage_service_api_base: MockRouter, - mocked_directorv2_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, + mocked_storage_rest_api_base: MockRouter, + mocked_directorv2_rest_api_base: MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, mocker: MockerFixture, ) -> MockedBackendApiDict: @@ -218,17 +218,17 @@ def mocked_backend( create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_storage_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_storage_rest_api_base, + mocked_directorv2_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "run_study_workflow.json", side_effects_callbacks=[], ) return MockedBackendApiDict( - webserver=mocked_webserver_service_api_base, - storage=mocked_storage_service_api_base, - director_v2=mocked_directorv2_service_api_base, + webserver=mocked_webserver_rest_api_base, + storage=mocked_storage_rest_api_base, + director_v2=mocked_directorv2_rest_api_base, ) diff --git a/services/api-server/tests/unit/test_api_files.py b/services/api-server/tests/unit/test_api_files.py index 323332f1f5b..8f44f00371e 100644 --- a/services/api-server/tests/unit/test_api_files.py +++ b/services/api-server/tests/unit/test_api_files.py @@ -96,7 +96,7 @@ def checksum(cls) -> SHA256Str: @pytest.mark.xfail(reason="Under dev") async def test_list_files_legacy( - client: AsyncClient, mocked_storage_service_api_base: MockRouter + client: AsyncClient, mocked_storage_rest_api_base: MockRouter ): response = await client.get(f"{API_VTAG}/files") @@ -117,7 +117,7 @@ async def test_list_files_legacy( @pytest.mark.xfail(reason="Under dev") async def test_list_files_with_pagination( client: AsyncClient, - mocked_storage_service_api_base: MockRouter, + mocked_storage_rest_api_base: MockRouter, ): response = await client.get(f"{API_VTAG}/files/page") @@ -146,7 +146,7 @@ async def test_list_files_with_pagination( @pytest.mark.xfail(reason="Under dev") async def test_upload_content( - client: AsyncClient, mocked_storage_service_api_base: MockRouter, tmp_path: Path + client: AsyncClient, mocked_storage_rest_api_base: MockRouter, tmp_path: Path ): upload_path = tmp_path / "test_upload_content.txt" upload_path.write_text("test_upload_content") @@ -166,7 +166,7 @@ async def test_upload_content( @pytest.mark.xfail(reason="Under dev") async def test_get_file( - client: AsyncClient, mocked_storage_service_api_base: MockRouter, tmp_path: Path + client: AsyncClient, mocked_storage_rest_api_base: MockRouter, tmp_path: Path ): response = await client.get( f"{API_VTAG}/files/3fa85f64-5717-4562-b3fc-2c963f66afa6" @@ -183,7 +183,7 @@ async def test_get_file( async def test_delete_file( client: AsyncClient, - mocked_storage_service_api_base: respx.MockRouter, + mocked_storage_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -205,7 +205,7 @@ def delete_side_effect( return capture.response_body create_respx_mock_from_capture( - respx_mocks=[mocked_storage_service_api_base], + respx_mocks=[mocked_storage_rest_api_base], capture_path=project_tests_dir / "mocks" / "delete_file.json", side_effects_callbacks=[search_side_effect, delete_side_effect], ) @@ -218,7 +218,7 @@ def delete_side_effect( @pytest.mark.xfail(reason="Under dev") async def test_download_content( - client: AsyncClient, mocked_storage_service_api_base: MockRouter, tmp_path: Path + client: AsyncClient, mocked_storage_rest_api_base: MockRouter, tmp_path: Path ): response = await client.get( f"{API_VTAG}/files/3fa85f64-5717-4562-b3fc-2c963f66afa6/content" @@ -297,7 +297,7 @@ async def test_get_upload_links( async def test_search_file( query: dict[str, str], client: AsyncClient, - mocked_storage_service_api_base: respx.MockRouter, + mocked_storage_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -325,7 +325,7 @@ def side_effect_callback( return response create_respx_mock_from_capture( - respx_mocks=[mocked_storage_service_api_base], + respx_mocks=[mocked_storage_rest_api_base], capture_path=project_tests_dir / "mocks" / "search_file_checksum.json", side_effects_callbacks=[side_effect_callback], ) diff --git a/services/api-server/tests/unit/test_api_health.py b/services/api-server/tests/unit/test_api_health.py index d421703b0f2..a5d92343962 100644 --- a/services/api-server/tests/unit/test_api_health.py +++ b/services/api-server/tests/unit/test_api_health.py @@ -30,10 +30,10 @@ def healthy(self) -> bool: async def test_get_service_state( client: AsyncClient, - mocked_catalog_service_api_base: MockRouter, - mocked_directorv2_service_api_base: MockRouter, - mocked_storage_service_api_base: MockRouter, - mocked_webserver_service_api_base: MockRouter, + mocked_catalog_rest_api_base: MockRouter, + mocked_directorv2_rest_api_base: MockRouter, + mocked_storage_rest_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, ): response = await client.get(f"{API_VTAG}/state") assert response.status_code == status.HTTP_200_OK diff --git a/services/api-server/tests/unit/test_api_solver_jobs.py b/services/api-server/tests/unit/test_api_solver_jobs.py index 0f0e91f126f..70171ce2736 100644 --- a/services/api-server/tests/unit/test_api_solver_jobs.py +++ b/services/api-server/tests/unit/test_api_solver_jobs.py @@ -59,7 +59,7 @@ def _inspect_job_side_effect( ) async def test_get_solver_job_wallet( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -94,7 +94,7 @@ def _get_wallet_side_effect( return response create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / capture, side_effects_callbacks=[_get_job_wallet_side_effect, _get_wallet_side_effect], ) @@ -131,7 +131,7 @@ def _get_wallet_side_effect( ) async def test_get_solver_job_pricing_unit( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -169,7 +169,7 @@ def _get_pricing_unit_side_effect( return capture.response_body create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / capture_file, side_effects_callbacks=( [_get_job_side_effect, _get_pricing_unit_side_effect] @@ -202,8 +202,8 @@ def _get_pricing_unit_side_effect( ) async def test_start_solver_job_pricing_unit_with_payment( client: AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, mocked_groups_extra_properties, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, @@ -256,8 +256,8 @@ def _put_pricing_plan_and_unit_side_effect( _put_pricing_plan_and_unit_side_effect.was_called = False create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, ], capture_path=project_tests_dir / "mocks" / capture_name, side_effects_callbacks=callbacks, @@ -279,8 +279,8 @@ def _put_pricing_plan_and_unit_side_effect( async def test_get_solver_job_pricing_unit_no_payment( client: AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, mocked_groups_extra_properties, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, @@ -293,8 +293,8 @@ async def test_get_solver_job_pricing_unit_no_payment( create_respx_mock_from_capture( respx_mocks=[ - mocked_directorv2_service_api_base, - mocked_webserver_service_api_base, + mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "start_job_no_payment.json", side_effects_callbacks=[ @@ -314,8 +314,8 @@ async def test_get_solver_job_pricing_unit_no_payment( async def test_start_solver_job_conflict( client: AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, mocked_groups_extra_properties, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, @@ -328,8 +328,8 @@ async def test_start_solver_job_conflict( create_respx_mock_from_capture( respx_mocks=[ - mocked_directorv2_service_api_base, - mocked_webserver_service_api_base, + mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "start_solver_job.json", side_effects_callbacks=[ @@ -350,7 +350,7 @@ async def test_start_solver_job_conflict( async def test_stop_job( client: AsyncClient, - mocked_directorv2_service_api_base, + mocked_directorv2_rest_api_base, mocked_groups_extra_properties, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, @@ -372,7 +372,7 @@ def _stop_job_side_effect( return jsonable_encoder(task) create_respx_mock_from_capture( - respx_mocks=[mocked_directorv2_service_api_base], + respx_mocks=[mocked_directorv2_rest_api_base], capture_path=project_tests_dir / "mocks" / "stop_job.json", side_effects_callbacks=[ _stop_job_side_effect, @@ -396,8 +396,8 @@ def _stop_job_side_effect( ) async def test_get_solver_job_outputs( client: AsyncClient, - mocked_webserver_service_api_base, - mocked_storage_service_api_base, + mocked_webserver_rest_api_base, + mocked_storage_rest_api_base, mocked_groups_extra_properties, mocked_solver_job_outputs, create_respx_mock_from_capture: CreateRespxMockCallback, @@ -433,8 +433,8 @@ def _wallet_side_effect( create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_storage_service_api_base, + mocked_webserver_rest_api_base, + mocked_storage_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "get_solver_outputs.json", side_effects_callbacks=[_sf, _sf, _sf, _wallet_side_effect, _sf], diff --git a/services/api-server/tests/unit/test_api_solvers.py b/services/api-server/tests/unit/test_api_solvers.py index 31e8ccb7f59..d35b648629e 100644 --- a/services/api-server/tests/unit/test_api_solvers.py +++ b/services/api-server/tests/unit/test_api_solvers.py @@ -26,7 +26,7 @@ ) async def test_get_solver_pricing_plan( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -35,7 +35,7 @@ async def test_get_solver_pricing_plan( ): respx_mock = create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / capture, side_effects_callbacks=[], ) diff --git a/services/api-server/tests/unit/test_api_wallets.py b/services/api-server/tests/unit/test_api_wallets.py index bf6f7167f23..5249d94565a 100644 --- a/services/api-server/tests/unit/test_api_wallets.py +++ b/services/api-server/tests/unit/test_api_wallets.py @@ -25,7 +25,7 @@ ) async def test_get_wallet( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -45,7 +45,7 @@ def _get_wallet_side_effect( return response create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / capture, side_effects_callbacks=[_get_wallet_side_effect], ) @@ -65,14 +65,14 @@ def _get_wallet_side_effect( async def test_get_default_wallet( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, ): create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / "get_default_wallet.json", side_effects_callbacks=[], ) diff --git a/services/api-server/tests/unit/test_credits.py b/services/api-server/tests/unit/test_credits.py index f9548949b81..c78165a78f0 100644 --- a/services/api-server/tests/unit/test_credits.py +++ b/services/api-server/tests/unit/test_credits.py @@ -10,13 +10,13 @@ async def test_get_credits_price( client: AsyncClient, auth: BasicAuth, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, project_tests_dir: Path, ): create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / "get_credits_price.json", side_effects_callbacks=[], ) diff --git a/services/api-server/tests/unit/test_services_directorv2.py b/services/api-server/tests/unit/test_services_directorv2.py index bd5e6d9a7ae..e30187c1707 100644 --- a/services/api-server/tests/unit/test_services_directorv2.py +++ b/services/api-server/tests/unit/test_services_directorv2.py @@ -28,7 +28,7 @@ def api() -> DirectorV2Api: async def test_oec_139646582688800_missing_ctx_values_for_msg_template( - mocked_directorv2_service_api_base: MockRouter, + mocked_directorv2_rest_api_base: MockRouter, project_id: ProjectID, user_id: UserID, api: DirectorV2Api, @@ -44,7 +44,7 @@ async def test_oec_139646582688800_missing_ctx_values_for_msg_template( # httpx.HTTPStatusError: Client error '404 Not Found' for url '/v2/computations/c7ad07d3-513f-4368-bcf0-354143b6a048?user_id=94' for method in ("GET", "POST", "DELETE"): - mocked_directorv2_service_api_base.request( + mocked_directorv2_rest_api_base.request( method, path__regex=r"/v2/computations/", ).respond(status_code=status.HTTP_404_NOT_FOUND) diff --git a/services/api-server/tests/unit/test_services_rabbitmq.py b/services/api-server/tests/unit/test_services_rabbitmq.py index aa644e81500..ab7652a8252 100644 --- a/services/api-server/tests/unit/test_services_rabbitmq.py +++ b/services/api-server/tests/unit/test_services_rabbitmq.py @@ -327,7 +327,7 @@ async def log_streamer_with_distributor( app: FastAPI, project_id: ProjectID, user_id: UserID, - mocked_directorv2_service_api_base: respx.MockRouter, + mocked_directorv2_rest_api_base: respx.MockRouter, computation_done: Callable[[], bool], log_distributor: LogDistributor, ) -> AsyncIterable[LogStreamer]: @@ -342,7 +342,7 @@ def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: status_code=status.HTTP_200_OK, json=jsonable_encoder(task) ) - mocked_directorv2_service_api_base.get(f"/v2/computations/{project_id}").mock( + mocked_directorv2_rest_api_base.get(f"/v2/computations/{project_id}").mock( side_effect=_get_computation ) From 709ee334976d0d2bcbfb37594380a09593c4f2b9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Mar 2025 13:36:36 +0200 Subject: [PATCH 34/39] fixes tests --- .../server/src/simcore_service_webserver/projects/plugin.py | 5 ++++- .../studies_dispatcher/test_studies_dispatcher_handlers.py | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/plugin.py b/services/web/server/src/simcore_service_webserver/projects/plugin.py index 06f7a930808..0500af2c3dd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/plugin.py +++ b/services/web/server/src/simcore_service_webserver/projects/plugin.py @@ -10,6 +10,7 @@ from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from ..constants import APP_SETTINGS_KEY +from ..rabbitmq import setup_rabbitmq from ._controller import ( comments_rest, folders_rest, @@ -53,7 +54,9 @@ def setup_projects(app: web.Application) -> bool: projects_slot.setup_project_observer_events(app) # setup RPC-controllers - app.on_startup.append(projects_rpc.register_rpc_routes_on_startup) + setup_rabbitmq(app) + if app[APP_SETTINGS_KEY].WEBSERVER_RABBITMQ: + app.on_startup.append(projects_rpc.register_rpc_routes_on_startup) # setup REST-controllers app.router.add_routes(projects_states_rest.routes) diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py index 3498fd2abcb..b2a88545160 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py @@ -7,7 +7,6 @@ import asyncio import re import urllib.parse -from collections.abc import AsyncIterator from typing import Any import pytest @@ -89,7 +88,7 @@ def web_server(redis_service: RedisSettings, web_server: TestServer) -> TestServ @pytest.fixture(autouse=True) async def director_v2_automock( director_v2_service_mock: aioresponses, -) -> AsyncIterator[aioresponses]: +) -> aioresponses: return director_v2_service_mock From 3cc787444083209b3b5a63940f95ae898775f3c3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Mar 2025 15:15:20 +0200 Subject: [PATCH 35/39] @GitHK review: doc --- .../src/simcore_service_api_server/models/api_resources.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/api-server/src/simcore_service_api_server/models/api_resources.py b/services/api-server/src/simcore_service_api_server/models/api_resources.py index 4f9b90c19a6..1f3e4e71e38 100644 --- a/services/api-server/src/simcore_service_api_server/models/api_resources.py +++ b/services/api-server/src/simcore_service_api_server/models/api_resources.py @@ -46,7 +46,6 @@ def parse_last_resource_id(resource_name: RelativeResourceName) -> str: def compose_resource_name(*collection_or_resource_ids) -> RelativeResourceName: - # NOTE: collection_or_resource_ids expected de quoted_parts = [ urllib.parse.quote_plus(f"{_id}".lstrip("/")) for _id in collection_or_resource_ids From 74dff1da13176667ae8cb332b8abb77c0d9509d7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Mar 2025 15:29:48 +0200 Subject: [PATCH 36/39] @GitHK review: drop id col --- .../versions/48604dfdc5f4_new_projects_to_job_map.py | 1 - .../simcore_postgres_database/models/projects_to_jobs.py | 8 +------- .../projects/_jobs_repository.py | 9 ++------- .../simcore_service_webserver/projects/_jobs_service.py | 3 +-- 4 files changed, 4 insertions(+), 17 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py index bbec001bd22..285e20f9ff0 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py @@ -20,7 +20,6 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table( "projects_to_jobs", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), sa.Column("project_uuid", sa.String(), nullable=False), sa.Column( "job_parent_resource_name", diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py index 85a7ea14c0e..4f3859fb36e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py @@ -8,13 +8,6 @@ # Maps projects used as jobs in the public-api "projects_to_jobs", metadata, - sa.Column( - "id", - sa.BigInteger, - primary_key=True, - autoincrement=True, - doc="Identifier index", - ), sa.Column( "project_uuid", sa.String, @@ -35,6 +28,7 @@ "the relative resource name is shelves/shelf1/jobs/job2, " "the parent resource name is shelves/shelf1.", ), + # Composite key (project_uuid, job_parent_resource_name) uniquely identifies very row sa.UniqueConstraint( "project_uuid", "job_parent_resource_name", diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index a2cb38d547a..385f0fb9a2c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -1,7 +1,6 @@ import logging from models_library.projects import ProjectID -from pydantic import PositiveInt from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs from simcore_postgres_database.utils_repos import transaction_context from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -20,7 +19,7 @@ async def set_project_as_job( *, project_uuid: ProjectID, job_parent_resource_name: str, - ) -> PositiveInt: + ) -> None: async with transaction_context(self.engine, connection) as conn: stmt = ( pg_insert(projects_to_jobs) @@ -32,10 +31,6 @@ async def set_project_as_job( index_elements=["project_uuid", "job_parent_resource_name"], set_={"job_parent_resource_name": job_parent_resource_name}, ) - .returning(projects_to_jobs.c.id) ) - result = await conn.execute(stmt) - row = result.one() - projects_to_jobs_id: PositiveInt = row.id - return projects_to_jobs_id + await conn.execute(stmt) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py index cca37cc756c..508a7eb232e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py @@ -42,7 +42,6 @@ async def set_project_as_job( repo = ProjectJobsRepository.create_from_app(app) - projects_to_jobs_id = await repo.set_project_as_job( + await repo.set_project_as_job( project_uuid=project_uuid, job_parent_resource_name=job_parent_resource_name ) - assert projects_to_jobs_id # nosec From 89a09812b8495015a8a0e37bf5ee455c0e828dd6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 1 Apr 2025 13:26:05 +0200 Subject: [PATCH 37/39] fixes migration --- .../migration/versions/48604dfdc5f4_new_projects_to_job_map.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py index 285e20f9ff0..5f742464ddd 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py @@ -34,7 +34,6 @@ def upgrade(): onupdate="CASCADE", ondelete="CASCADE", ), - sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint( "project_uuid", "job_parent_resource_name", From c764776097690e94047b85d8568f880367f476fd Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 1 Apr 2025 14:21:15 +0200 Subject: [PATCH 38/39] fixing tests by moving fixture --- .../tests/unit/api_solvers/conftest.py | 32 ------------------ .../test_api_routes_studies_jobs.py | 24 ++++++++------ services/api-server/tests/unit/conftest.py | 33 ++++++++++++++++++- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index 782f2cf87e7..7c91f7a13a8 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -13,11 +13,8 @@ from fastapi import FastAPI, status from fastapi.encoders import jsonable_encoder from models_library.projects_state import RunningState -from pytest_mock import MockerFixture, MockType from pytest_simcore.helpers import faker_catalog -from pytest_simcore.helpers.webserver_rpc_server import WebserverRpcSideEffects from respx import MockRouter -from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from simcore_service_api_server.core.settings import ApplicationSettings from simcore_service_api_server.services_http.director_v2 import ComputationTaskGet @@ -46,35 +43,6 @@ def mocked_webserver_rest_api( return mocked_webserver_rest_api_base -@pytest.fixture -def mocked_webserver_rpc_api( - app: FastAPI, mocker: MockerFixture -) -> dict[str, MockType]: - from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc - from simcore_service_api_server.services_rpc import wb_api_server - - # NOTE: mock_missing_plugins patches `setup_rabbitmq` - try: - wb_api_server.WbApiRpcClient.get_from_app_state(app) - except AttributeError: - wb_api_server.setup( - app, RabbitMQRPCClient("fake_rpc_client", settings=mocker.MagicMock()) - ) - - settings: ApplicationSettings = app.state.settings - assert settings.API_SERVER_WEBSERVER - - side_effects = WebserverRpcSideEffects() - - return { - "mark_project_as_job": mocker.patch.object( - projects_rpc, - "mark_project_as_job", - side_effects.mark_project_as_job, - ), - } - - @pytest.fixture def mocked_catalog_rest_api( app: FastAPI, diff --git a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py index 6a3d996809d..84b300cf07a 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py +++ b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py @@ -19,7 +19,6 @@ CreateRespxMockCallback, HttpApiCallCaptureModel, ) -from respx import MockRouter from servicelib.common_headers import ( X_SIMCORE_PARENT_NODE_ID, X_SIMCORE_PARENT_PROJECT_UUID, @@ -123,8 +122,9 @@ async def test_studies_jobs_workflow( async def test_start_stop_delete_study_job( client: httpx.AsyncClient, - mocked_webserver_rest_api_base, - mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_directorv2_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -202,8 +202,9 @@ def _check_response(response: httpx.Response, status_code: int): @pytest.mark.parametrize("hidden", [True, False]) async def test_create_study_job( client: httpx.AsyncClient, - mocked_webserver_rest_api_base, - mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_directorv2_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -281,7 +282,8 @@ async def test_get_study_job_outputs( client: httpx.AsyncClient, fake_study_id: UUID, auth: httpx.BasicAuth, - mocked_webserver_rest_api_base: MockRouter, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], ): job_id = "cfe9a77a-f71e-11ee-8fca-0242ac140008" @@ -338,8 +340,9 @@ async def test_get_study_job_outputs( async def test_get_job_logs( client: httpx.AsyncClient, - mocked_webserver_rest_api_base, - mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_directorv2_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -365,8 +368,9 @@ async def test_get_job_logs( async def test_get_study_outputs( client: httpx.AsyncClient, create_respx_mock_from_capture: CreateRespxMockCallback, - mocked_directorv2_rest_api_base, - mocked_webserver_rest_api_base, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_directorv2_rest_api_base: respx.MockRouter, auth: httpx.BasicAuth, project_tests_dir: Path, ): diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 58cc3aa083e..8fb9fb2a445 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -37,12 +37,14 @@ from moto.server import ThreadedMotoServer from packaging.version import Version from pydantic import EmailStr, HttpUrl, TypeAdapter -from pytest_mock import MockerFixture +from pytest_mock import MockerFixture, MockType from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.webserver_rpc_server import WebserverRpcSideEffects from pytest_simcore.simcore_webserver_projects_rest_api import GET_PROJECT from requests.auth import HTTPBasicAuth from respx import MockRouter +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from simcore_service_api_server.core.application import init_app from simcore_service_api_server.core.settings import ApplicationSettings from simcore_service_api_server.db.repositories.api_keys import UserAndProductTuple @@ -332,6 +334,35 @@ def mocked_webserver_rest_api_base( yield respx_mock +@pytest.fixture +def mocked_webserver_rpc_api( + app: FastAPI, mocker: MockerFixture +) -> dict[str, MockType]: + from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc + from simcore_service_api_server.services_rpc import wb_api_server + + # NOTE: mock_missing_plugins patches `setup_rabbitmq` + try: + wb_api_server.WbApiRpcClient.get_from_app_state(app) + except AttributeError: + wb_api_server.setup( + app, RabbitMQRPCClient("fake_rpc_client", settings=mocker.MagicMock()) + ) + + settings: ApplicationSettings = app.state.settings + assert settings.API_SERVER_WEBSERVER + + side_effects = WebserverRpcSideEffects() + + return { + "mark_project_as_job": mocker.patch.object( + projects_rpc, + "mark_project_as_job", + side_effects.mark_project_as_job, + ), + } + + @pytest.fixture def mocked_storage_rest_api_base( app: FastAPI, From 14eb2c8bc712d677eed9c7a02ec5c6f5fbd0f1fa Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 1 Apr 2025 16:58:53 +0200 Subject: [PATCH 39/39] fixes fixtures --- .../tests/unit/_with_db/test_api_user.py | 14 +++++++------- .../tests/unit/test_api__study_workflows.py | 15 ++++++++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/services/api-server/tests/unit/_with_db/test_api_user.py b/services/api-server/tests/unit/_with_db/test_api_user.py index 72f8fadffda..5b29f72ef15 100644 --- a/services/api-server/tests/unit/_with_db/test_api_user.py +++ b/services/api-server/tests/unit/_with_db/test_api_user.py @@ -19,7 +19,7 @@ @pytest.fixture -def mocked_webserver_service_api(app: FastAPI): +def mocked_webserver_rest_api(app: FastAPI): """Mocks some responses of web-server service""" settings: ApplicationSettings = app.state.settings @@ -54,8 +54,8 @@ def _update_me(request: httpx.Request): async def test_get_profile( client: httpx.AsyncClient, auth: httpx.BasicAuth, - mocked_webserver_service_api: MockRouter, - mocked_rpc_webserver_service_api: dict[str, MockType], + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], ): # needs no auth resp = await client.get(f"/{API_VTAG}/meta") @@ -64,11 +64,11 @@ async def test_get_profile( # needs auth resp = await client.get(f"/{API_VTAG}/me") assert resp.status_code == status.HTTP_401_UNAUTHORIZED - assert not mocked_webserver_service_api["get_me"].called + assert not mocked_webserver_rest_api["get_me"].called resp = await client.get(f"/{API_VTAG}/me", auth=auth) assert resp.status_code == status.HTTP_200_OK - assert mocked_webserver_service_api["get_me"].called + assert mocked_webserver_rest_api["get_me"].called profile = Profile(**resp.json()) assert profile.first_name == "James" @@ -78,8 +78,8 @@ async def test_get_profile( async def test_update_profile( client: httpx.AsyncClient, auth: httpx.BasicAuth, - mocked_webserver_service_api: MockRouter, - mocked_rpc_webserver_service_api: dict[str, MockType], + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], ): # needs auth resp = await client.put( diff --git a/services/api-server/tests/unit/test_api__study_workflows.py b/services/api-server/tests/unit/test_api__study_workflows.py index 07653159d63..0656ad49732 100644 --- a/services/api-server/tests/unit/test_api__study_workflows.py +++ b/services/api-server/tests/unit/test_api__study_workflows.py @@ -14,10 +14,10 @@ import httpx import pytest +import respx from fastapi.encoders import jsonable_encoder from pytest_mock import MockerFixture from pytest_simcore.helpers.httpx_calls_capture_models import CreateRespxMockCallback -from respx import MockRouter from simcore_sdk.node_ports_common.filemanager import UploadedFile from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.models.pagination import OnePage @@ -196,17 +196,18 @@ def test_py_path(tmp_path: Path) -> Path: class MockedBackendApiDict(TypedDict): - webserver: MockRouter | None - storage: MockRouter | None - director_v2: MockRouter | None + webserver: respx.MockRouter | None + storage: respx.MockRouter | None + director_v2: respx.MockRouter | None @pytest.fixture def mocked_backend( project_tests_dir: Path, - mocked_webserver_rest_api_base: MockRouter, - mocked_storage_rest_api_base: MockRouter, - mocked_directorv2_rest_api_base: MockRouter, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: respx.MockRouter, + mocked_storage_rest_api_base: respx.MockRouter, + mocked_directorv2_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, mocker: MockerFixture, ) -> MockedBackendApiDict: