✨ api and web-server: introduce job-project mapping via projects_to_jobs table and RPC integration

packages/common-library/tests/test_errors_classes.py

@@ -13,20 +13,15 @@
 
 
 def test_get_full_class_name():
-    class A(OsparcErrorMixin):
-        ...
+    class A(OsparcErrorMixin): ...
 
-    class B1(A):
-        ...
+    class B1(A): ...
 
-    class B2(A):
-        ...
+    class B2(A): ...
 
-    class C(B2):
-        ...
+    class C(B2): ...
 
-    class B12(B1, ValueError):
-        ...
+    class B12(B1, ValueError): ...
 
     assert B1._get_full_class_name() == "A.B1"
     assert C._get_full_class_name() == "A.B2.C"

packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py

@@ -0,0 +1,62 @@
+"""new projects to job map
+
+Revision ID: 48604dfdc5f4
+Revises: 8403acca8759
+Create Date: 2025-03-26 12:00:14.763439+00:00
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "48604dfdc5f4"
+down_revision = "8403acca8759"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "projects_to_jobs",
+        sa.Column("project_uuid", sa.String(), nullable=False),
+        sa.Column(
+            "job_parent_resource_name",
+            sa.String(),
+            nullable=False,
+            doc="Prefix for the job resource name. For example, if the relative resource name is shelves/shelf1/books/book2, the parent resource name is shelves/shelf1.",
+        ),
+        sa.ForeignKeyConstraint(
+            ["project_uuid"],
+            ["projects.uuid"],
+            name="fk_projects_to_jobs_project_uuid",
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint(
+            "project_uuid",
+            "job_parent_resource_name",
+            name="uq_projects_to_jobs_project_uuid_job_parent_resource_name",
+        ),
+    )
+
+    # Populate the new table
+    op.execute(
+        sa.text(
+            r"""
+INSERT INTO projects_to_jobs (project_uuid, job_parent_resource_name)
+SELECT
+    uuid AS project_uuid,
+    regexp_replace(name, '/jobs/.+$', '', 'g') AS job_parent_resource_name -- trim /jobs/.+$
+FROM projects
+WHERE name ~* '^solvers/.+/jobs/.+$' OR name ~* '^studies/.+/jobs/.+$';
+    """
+        )
+    )
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("projects_to_jobs")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py

@@ -0,0 +1,37 @@
+import sqlalchemy as sa
+
+from ._common import RefActions
+from .base import metadata
+from .projects import projects
+
+projects_to_jobs = sa.Table(
+    # Maps projects used as jobs in the public-api
+    "projects_to_jobs",
+    metadata,
+    sa.Column(
+        "project_uuid",
+        sa.String,
+        sa.ForeignKey(
+            projects.c.uuid,
+            onupdate=RefActions.CASCADE,
+            ondelete=RefActions.CASCADE,
+            name="fk_projects_to_jobs_project_uuid",
+        ),
+        nullable=False,
+        doc="Foreign key to projects.uuid",
+    ),
+    sa.Column(
+        "job_parent_resource_name",
+        sa.String,
+        nullable=False,
+        doc="Prefix for the job resource name use in the public-api. For example, if "
+        "the relative resource name is shelves/shelf1/jobs/job2, "
+        "the parent resource name is shelves/shelf1.",
+    ),
+    # Composite key (project_uuid, job_parent_resource_name) uniquely identifies very row
+    sa.UniqueConstraint(
+        "project_uuid",
+        "job_parent_resource_name",
+        name="uq_projects_to_jobs_project_uuid_job_parent_resource_name",
+    ),
+)

packages/postgres-database/tests/test_models_projects_to_jobs.py

@@ -0,0 +1,152 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+# pylint: disable=too-many-arguments
+
+from collections.abc import Iterator
+
+import pytest
+import simcore_postgres_database.cli
+import sqlalchemy as sa
+import sqlalchemy.engine
+import sqlalchemy.exc
+from faker import Faker
+from pytest_simcore.helpers import postgres_tools
+from pytest_simcore.helpers.faker_factories import random_project, random_user
+from simcore_postgres_database.models.projects import projects
+from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs
+from simcore_postgres_database.models.users import users
+
+
+@pytest.fixture
+def sync_engine(
+    sync_engine: sqlalchemy.engine.Engine, db_metadata: sa.MetaData
+) -> Iterator[sqlalchemy.engine.Engine]:
+    # EXTENDS sync_engine fixture to include cleanup and parare migration
+
+    # cleanup tables
+    db_metadata.drop_all(sync_engine)
+
+    # prepare migration upgrade
+    assert simcore_postgres_database.cli.discover.callback
+    assert simcore_postgres_database.cli.upgrade.callback
+
+    dsn = sync_engine.url
+    simcore_postgres_database.cli.discover.callback(
+        user=dsn.username,
+        password=dsn.password,
+        host=dsn.host,
+        database=dsn.database,
+        port=dsn.port,
+    )
+
+    yield sync_engine
+
+    # cleanup tables
+    postgres_tools.force_drop_all_tables(sync_engine)
+
+
+def test_populate_projects_to_jobs_during_migration(
+    sync_engine: sqlalchemy.engine.Engine, faker: Faker
+):
+    assert simcore_postgres_database.cli.discover.callback
+    assert simcore_postgres_database.cli.upgrade.callback
+
+    # UPGRADE just one before 48604dfdc5f4_new_projects_to_job_map.py
+    simcore_postgres_database.cli.upgrade.callback("8403acca8759")
+
+    with sync_engine.connect() as conn:
+
+        # Ensure the projects_to_jobs table does NOT exist yet
+        with pytest.raises(sqlalchemy.exc.ProgrammingError) as exc_info:
+            conn.execute(
+                sa.select(sa.func.count()).select_from(projects_to_jobs)
+            ).scalar()
+        assert "psycopg2.errors.UndefinedTable" in f"{exc_info.value}"
+
+        # INSERT data (emulates data in-place)
+        user_data = random_user(
+            faker, name="test_populate_projects_to_jobs_during_migration"
+        )
+        stmt = users.insert().values(**user_data).returning(users.c.id)
+        result = conn.execute(stmt)
+        user_id = result.scalar()
+
+        SPACES = " " * 3
+        projects_data = [
+            random_project(
+                faker,
+                uuid="cd03450c-4c17-4c2c-85fd-0d951d7dcd5a",
+                name="solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a",
+                description=(
+                    "Study associated to solver job:"
+                    """{
+                    "id": "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a",
+                    "name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c2c-85fd-0d951d7dcd5a",
+                    "inputs_checksum": "015ba4cd5cf00c511a8217deb65c242e3b15dc6ae4b1ecf94982d693887d9e8a",
+                    "created_at": "2025-01-27T13:12:58.676564Z"
+                    }
+                    """
+                ),
+                prj_owner=user_id,
+            ),
+            random_project(
+                faker,
+                uuid="bf204942-007b-11ef-befd-0242ac114f07",
+                name=f"studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07{SPACES}",
+                description="Valid project 2",
+                prj_owner=user_id,
+            ),
+            random_project(
+                faker,
+                uuid="33333333-3333-3333-3333-333333333333",
+                name="invalid/project/name",
+                description="Invalid project",
+                prj_owner=user_id,
+            ),
+        ]
+        for prj in projects_data:
+            conn.execute(sa.insert(projects).values(prj))
+
+    # MIGRATE UPGRADE: this should populate
+    simcore_postgres_database.cli.upgrade.callback("head")
+
+    with sync_engine.connect() as conn:
+        # Query the projects_to_jobs table
+        result = conn.execute(
+            sa.select(
+                projects_to_jobs.c.project_uuid,
+                projects_to_jobs.c.job_parent_resource_name,
+            )
+        ).fetchall()
+
+        # Assert only valid projects are added
+        assert len(result) == 2
+        assert (
+            "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a",
+            "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1",
+        ) in result
+        assert (
+            "bf204942-007b-11ef-befd-0242ac114f07",
+            "studies/4b7a704a-007a-11ef-befd-0242ac114f07",
+        ) in result
+
+        # Query project name and description for projects also in projects_to_jobs
+        result = conn.execute(
+            sa.select(
+                projects.c.name,
+                projects.c.uuid,
+                projects_to_jobs.c.job_parent_resource_name,
+            ).select_from(
+                projects.join(
+                    projects_to_jobs, projects.c.uuid == projects_to_jobs.c.project_uuid
+                )
+            )
+        ).fetchall()
+
+        # Print or assert the result as needed
+        for project_name, project_uuid, job_parent_resource_name in result:
+            assert (
+                f"{job_parent_resource_name}/jobs/{project_uuid}"
+                == project_name.strip()
+            )

packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py

@@ -0,0 +1,37 @@
+# pylint: disable=not-context-manager
+# pylint: disable=protected-access
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+
+from models_library.products import ProductName
+from models_library.projects import ProjectID
+from models_library.users import UserID
+from pydantic import TypeAdapter, validate_call
+from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient
+
+
+class WebserverRpcSideEffects:
+    # pylint: disable=no-self-use
+
+    @validate_call(config={"arbitrary_types_allowed": True})
+    async def mark_project_as_job(
+        self,
+        rpc_client: RabbitMQRPCClient,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        project_uuid: ProjectID,
+        job_parent_resource_name: str,
+    ) -> None:
+        assert rpc_client
+
+        assert not job_parent_resource_name.startswith("/")  # nosec
+        assert "/" in job_parent_resource_name  # nosec
+        assert not job_parent_resource_name.endswith("/")  # nosec
+
+        assert product_name
+        assert user_id
+
+        TypeAdapter(ProjectID).validate_python(project_uuid)

packages/service-library/src/servicelib/common_headers.py

@@ -1,9 +1,9 @@
 from typing import Final
 
+UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE: Final[str] = "undefined"
 X_DYNAMIC_SIDECAR_REQUEST_DNS: Final[str] = "X-Dynamic-Sidecar-Request-DNS"
 X_DYNAMIC_SIDECAR_REQUEST_SCHEME: Final[str] = "X-Dynamic-Sidecar-Request-Scheme"
 X_FORWARDED_PROTO: Final[str] = "X-Forwarded-Proto"
-X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent"
-X_SIMCORE_PARENT_PROJECT_UUID: Final[str] = "X-Simcore-Parent-Project-Uuid"
 X_SIMCORE_PARENT_NODE_ID: Final[str] = "X-Simcore-Parent-Node-Id"
-UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE: Final[str] = "undefined"
+X_SIMCORE_PARENT_PROJECT_UUID: Final[str] = "X-Simcore-Parent-Project-Uuid"
+X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent"

packages/service-library/src/servicelib/rabbitmq/__init__.py

@@ -5,6 +5,7 @@
 from ._constants import BIND_TO_ALL_TOPICS, RPC_REQUEST_DEFAULT_TIMEOUT_S
 from ._errors import (
     RemoteMethodNotRegisteredError,
+    RPCInterfaceError,
     RPCNotInitializedError,
     RPCServerError,
 )
@@ -14,18 +15,19 @@
 
 __all__: tuple[str, ...] = (
     "BIND_TO_ALL_TOPICS",
+    "RPC_REQUEST_DEFAULT_TIMEOUT_S",
     "ConsumerTag",
     "ExchangeName",
-    "is_rabbitmq_responsive",
     "QueueName",
-    "RabbitMQClient",
-    "RabbitMQRPCClient",
-    "RemoteMethodNotRegisteredError",
-    "RPC_REQUEST_DEFAULT_TIMEOUT_S",
+    "RPCInterfaceError",
     "RPCNamespace",
     "RPCNotInitializedError",
     "RPCRouter",
     "RPCServerError",
+    "RabbitMQClient",
+    "RabbitMQRPCClient",
+    "RemoteMethodNotRegisteredError",
+    "is_rabbitmq_responsive",
     "wait_till_rabbitmq_responsive",
 )