Skip to content

Commit 020e2df

Browse files
pcrespovpcrespov
authored
✨ api and web-server: introduce job-project mapping via projects_to_jobs table and RPC integration (#7435)
1 parent c0e0037 commit 020e2df

File tree

47 files changed

+1116
-286
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1116
-286
lines changed

‎packages/common-library/tests/test_errors_classes.py

+5-10
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,15 @@
1313

1414

1515
def test_get_full_class_name():
16-
class A(OsparcErrorMixin):
17-
...
16+
class A(OsparcErrorMixin): ...
1817

19-
class B1(A):
20-
...
18+
class B1(A): ...
2119

22-
class B2(A):
23-
...
20+
class B2(A): ...
2421

25-
class C(B2):
26-
...
22+
class C(B2): ...
2723

28-
class B12(B1, ValueError):
29-
...
24+
class B12(B1, ValueError): ...
3025

3126
assert B1._get_full_class_name() == "A.B1"
3227
assert C._get_full_class_name() == "A.B2.C"

‎packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
"""new projects to job map
2+
3+
Revision ID: 48604dfdc5f4
4+
Revises: 8403acca8759
5+
Create Date: 2025-03-26 12:00:14.763439+00:00
6+
7+
"""
8+
9+
import sqlalchemy as sa
10+
from alembic import op
11+
12+
# revision identifiers, used by Alembic.
13+
revision = "48604dfdc5f4"
14+
down_revision = "8403acca8759"
15+
branch_labels = None
16+
depends_on = None
17+
18+
19+
def upgrade():
20+
# ### commands auto generated by Alembic - please adjust! ###
21+
op.create_table(
22+
"projects_to_jobs",
23+
sa.Column("project_uuid", sa.String(), nullable=False),
24+
sa.Column(
25+
"job_parent_resource_name",
26+
sa.String(),
27+
nullable=False,
28+
doc="Prefix for the job resource name. For example, if the relative resource name is shelves/shelf1/books/book2, the parent resource name is shelves/shelf1.",
29+
),
30+
sa.ForeignKeyConstraint(
31+
["project_uuid"],
32+
["projects.uuid"],
33+
name="fk_projects_to_jobs_project_uuid",
34+
onupdate="CASCADE",
35+
ondelete="CASCADE",
36+
),
37+
sa.UniqueConstraint(
38+
"project_uuid",
39+
"job_parent_resource_name",
40+
name="uq_projects_to_jobs_project_uuid_job_parent_resource_name",
41+
),
42+
)
43+
44+
# Populate the new table
45+
op.execute(
46+
sa.text(
47+
r"""
48+
INSERT INTO projects_to_jobs (project_uuid, job_parent_resource_name)
49+
SELECT
50+
uuid AS project_uuid,
51+
regexp_replace(name, '/jobs/.+$', '', 'g') AS job_parent_resource_name -- trim /jobs/.+$
52+
FROM projects
53+
WHERE name ~* '^solvers/.+/jobs/.+$' OR name ~* '^studies/.+/jobs/.+$';
54+
"""
55+
)
56+
)
57+
58+
59+
def downgrade():
60+
# ### commands auto generated by Alembic - please adjust! ###
61+
op.drop_table("projects_to_jobs")
62+
# ### end Alembic commands ###

‎packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import sqlalchemy as sa
2+
3+
from ._common import RefActions
4+
from .base import metadata
5+
from .projects import projects
6+
7+
projects_to_jobs = sa.Table(
8+
# Maps projects used as jobs in the public-api
9+
"projects_to_jobs",
10+
metadata,
11+
sa.Column(
12+
"project_uuid",
13+
sa.String,
14+
sa.ForeignKey(
15+
projects.c.uuid,
16+
onupdate=RefActions.CASCADE,
17+
ondelete=RefActions.CASCADE,
18+
name="fk_projects_to_jobs_project_uuid",
19+
),
20+
nullable=False,
21+
doc="Foreign key to projects.uuid",
22+
),
23+
sa.Column(
24+
"job_parent_resource_name",
25+
sa.String,
26+
nullable=False,
27+
doc="Prefix for the job resource name use in the public-api. For example, if "
28+
"the relative resource name is shelves/shelf1/jobs/job2, "
29+
"the parent resource name is shelves/shelf1.",
30+
),
31+
# Composite key (project_uuid, job_parent_resource_name) uniquely identifies very row
32+
sa.UniqueConstraint(
33+
"project_uuid",
34+
"job_parent_resource_name",
35+
name="uq_projects_to_jobs_project_uuid_job_parent_resource_name",
36+
),
37+
)

‎packages/postgres-database/tests/test_models_projects_to_jobs.py

+152
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
# pylint: disable=redefined-outer-name
2+
# pylint: disable=unused-argument
3+
# pylint: disable=unused-variable
4+
# pylint: disable=too-many-arguments
5+
6+
from collections.abc import Iterator
7+
8+
import pytest
9+
import simcore_postgres_database.cli
10+
import sqlalchemy as sa
11+
import sqlalchemy.engine
12+
import sqlalchemy.exc
13+
from faker import Faker
14+
from pytest_simcore.helpers import postgres_tools
15+
from pytest_simcore.helpers.faker_factories import random_project, random_user
16+
from simcore_postgres_database.models.projects import projects
17+
from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs
18+
from simcore_postgres_database.models.users import users
19+
20+
21+
@pytest.fixture
22+
def sync_engine(
23+
sync_engine: sqlalchemy.engine.Engine, db_metadata: sa.MetaData
24+
) -> Iterator[sqlalchemy.engine.Engine]:
25+
# EXTENDS sync_engine fixture to include cleanup and parare migration
26+
27+
# cleanup tables
28+
db_metadata.drop_all(sync_engine)
29+
30+
# prepare migration upgrade
31+
assert simcore_postgres_database.cli.discover.callback
32+
assert simcore_postgres_database.cli.upgrade.callback
33+
34+
dsn = sync_engine.url
35+
simcore_postgres_database.cli.discover.callback(
36+
user=dsn.username,
37+
password=dsn.password,
38+
host=dsn.host,
39+
database=dsn.database,
40+
port=dsn.port,
41+
)
42+
43+
yield sync_engine
44+
45+
# cleanup tables
46+
postgres_tools.force_drop_all_tables(sync_engine)
47+
48+
49+
def test_populate_projects_to_jobs_during_migration(
50+
sync_engine: sqlalchemy.engine.Engine, faker: Faker
51+
):
52+
assert simcore_postgres_database.cli.discover.callback
53+
assert simcore_postgres_database.cli.upgrade.callback
54+
55+
# UPGRADE just one before 48604dfdc5f4_new_projects_to_job_map.py
56+
simcore_postgres_database.cli.upgrade.callback("8403acca8759")
57+
58+
with sync_engine.connect() as conn:
59+
60+
# Ensure the projects_to_jobs table does NOT exist yet
61+
with pytest.raises(sqlalchemy.exc.ProgrammingError) as exc_info:
62+
conn.execute(
63+
sa.select(sa.func.count()).select_from(projects_to_jobs)
64+
).scalar()
65+
assert "psycopg2.errors.UndefinedTable" in f"{exc_info.value}"
66+
67+
# INSERT data (emulates data in-place)
68+
user_data = random_user(
69+
faker, name="test_populate_projects_to_jobs_during_migration"
70+
)
71+
stmt = users.insert().values(**user_data).returning(users.c.id)
72+
result = conn.execute(stmt)
73+
user_id = result.scalar()
74+
75+
SPACES = " " * 3
76+
projects_data = [
77+
random_project(
78+
faker,
79+
uuid="cd03450c-4c17-4c2c-85fd-0d951d7dcd5a",
80+
name="solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a",
81+
description=(
82+
"Study associated to solver job:"
83+
"""{
84+
"id": "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a",
85+
"name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c2c-85fd-0d951d7dcd5a",
86+
"inputs_checksum": "015ba4cd5cf00c511a8217deb65c242e3b15dc6ae4b1ecf94982d693887d9e8a",
87+
"created_at": "2025-01-27T13:12:58.676564Z"
88+
}
89+
"""
90+
),
91+
prj_owner=user_id,
92+
),
93+
random_project(
94+
faker,
95+
uuid="bf204942-007b-11ef-befd-0242ac114f07",
96+
name=f"studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07{SPACES}",
97+
description="Valid project 2",
98+
prj_owner=user_id,
99+
),
100+
random_project(
101+
faker,
102+
uuid="33333333-3333-3333-3333-333333333333",
103+
name="invalid/project/name",
104+
description="Invalid project",
105+
prj_owner=user_id,
106+
),
107+
]
108+
for prj in projects_data:
109+
conn.execute(sa.insert(projects).values(prj))
110+
111+
# MIGRATE UPGRADE: this should populate
112+
simcore_postgres_database.cli.upgrade.callback("head")
113+
114+
with sync_engine.connect() as conn:
115+
# Query the projects_to_jobs table
116+
result = conn.execute(
117+
sa.select(
118+
projects_to_jobs.c.project_uuid,
119+
projects_to_jobs.c.job_parent_resource_name,
120+
)
121+
).fetchall()
122+
123+
# Assert only valid projects are added
124+
assert len(result) == 2
125+
assert (
126+
"cd03450c-4c17-4c2c-85fd-0d951d7dcd5a",
127+
"solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1",
128+
) in result
129+
assert (
130+
"bf204942-007b-11ef-befd-0242ac114f07",
131+
"studies/4b7a704a-007a-11ef-befd-0242ac114f07",
132+
) in result
133+
134+
# Query project name and description for projects also in projects_to_jobs
135+
result = conn.execute(
136+
sa.select(
137+
projects.c.name,
138+
projects.c.uuid,
139+
projects_to_jobs.c.job_parent_resource_name,
140+
).select_from(
141+
projects.join(
142+
projects_to_jobs, projects.c.uuid == projects_to_jobs.c.project_uuid
143+
)
144+
)
145+
).fetchall()
146+
147+
# Print or assert the result as needed
148+
for project_name, project_uuid, job_parent_resource_name in result:
149+
assert (
150+
f"{job_parent_resource_name}/jobs/{project_uuid}"
151+
== project_name.strip()
152+
)

‎packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# pylint: disable=not-context-manager
2+
# pylint: disable=protected-access
3+
# pylint: disable=redefined-outer-name
4+
# pylint: disable=unused-argument
5+
# pylint: disable=unused-variable
6+
7+
8+
from models_library.products import ProductName
9+
from models_library.projects import ProjectID
10+
from models_library.users import UserID
11+
from pydantic import TypeAdapter, validate_call
12+
from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient
13+
14+
15+
class WebserverRpcSideEffects:
16+
# pylint: disable=no-self-use
17+
18+
@validate_call(config={"arbitrary_types_allowed": True})
19+
async def mark_project_as_job(
20+
self,
21+
rpc_client: RabbitMQRPCClient,
22+
*,
23+
product_name: ProductName,
24+
user_id: UserID,
25+
project_uuid: ProjectID,
26+
job_parent_resource_name: str,
27+
) -> None:
28+
assert rpc_client
29+
30+
assert not job_parent_resource_name.startswith("/") # nosec
31+
assert "/" in job_parent_resource_name # nosec
32+
assert not job_parent_resource_name.endswith("/") # nosec
33+
34+
assert product_name
35+
assert user_id
36+
37+
TypeAdapter(ProjectID).validate_python(project_uuid)

‎packages/service-library/src/servicelib/common_headers.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
from typing import Final
22

3+
UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE: Final[str] = "undefined"
34
X_DYNAMIC_SIDECAR_REQUEST_DNS: Final[str] = "X-Dynamic-Sidecar-Request-DNS"
45
X_DYNAMIC_SIDECAR_REQUEST_SCHEME: Final[str] = "X-Dynamic-Sidecar-Request-Scheme"
56
X_FORWARDED_PROTO: Final[str] = "X-Forwarded-Proto"
6-
X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent"
7-
X_SIMCORE_PARENT_PROJECT_UUID: Final[str] = "X-Simcore-Parent-Project-Uuid"
87
X_SIMCORE_PARENT_NODE_ID: Final[str] = "X-Simcore-Parent-Node-Id"
9-
UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE: Final[str] = "undefined"
8+
X_SIMCORE_PARENT_PROJECT_UUID: Final[str] = "X-Simcore-Parent-Project-Uuid"
9+
X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent"

‎packages/service-library/src/servicelib/rabbitmq/__init__.py

+7-5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from ._constants import BIND_TO_ALL_TOPICS, RPC_REQUEST_DEFAULT_TIMEOUT_S
66
from ._errors import (
77
RemoteMethodNotRegisteredError,
8+
RPCInterfaceError,
89
RPCNotInitializedError,
910
RPCServerError,
1011
)
@@ -14,18 +15,19 @@
1415

1516
__all__: tuple[str, ...] = (
1617
"BIND_TO_ALL_TOPICS",
18+
"RPC_REQUEST_DEFAULT_TIMEOUT_S",
1719
"ConsumerTag",
1820
"ExchangeName",
19-
"is_rabbitmq_responsive",
2021
"QueueName",
21-
"RabbitMQClient",
22-
"RabbitMQRPCClient",
23-
"RemoteMethodNotRegisteredError",
24-
"RPC_REQUEST_DEFAULT_TIMEOUT_S",
22+
"RPCInterfaceError",
2523
"RPCNamespace",
2624
"RPCNotInitializedError",
2725
"RPCRouter",
2826
"RPCServerError",
27+
"RabbitMQClient",
28+
"RabbitMQRPCClient",
29+
"RemoteMethodNotRegisteredError",
30+
"is_rabbitmq_responsive",
2931
"wait_till_rabbitmq_responsive",
3032
)
3133

0 commit comments

Comments
 (0)