diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py index 22a56b0da4e..72001f8b550 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py @@ -20,6 +20,7 @@ class ServiceRunGet(BaseModel): user_email: str project_id: ProjectID project_name: str + project_tags: list[str] node_id: NodeID node_name: str root_parent_project_id: ProjectID diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/8e1f83486be7_enhance_projects_tags_for_rut.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/8e1f83486be7_enhance_projects_tags_for_rut.py new file mode 100644 index 00000000000..6c0d6608185 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/8e1f83486be7_enhance_projects_tags_for_rut.py @@ -0,0 +1,90 @@ +"""enhance projects_tags for RUT + +Revision ID: 8e1f83486be7 +Revises: 8bfe65a5e294 +Create Date: 2024-11-15 09:12:57.789183+00:00 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "8e1f83486be7" +down_revision = "8bfe65a5e294" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "projects_tags", sa.Column("project_uuid_for_rut", sa.String(), nullable=True) + ) + + # Migrate + op.execute( + sa.DDL( + """ + UPDATE projects_tags + SET project_uuid_for_rut = projects.uuid + FROM projects + WHERE projects_tags.project_id = projects.id; + """ + ) + ) + + op.alter_column( + "projects_tags", + "project_uuid_for_rut", + existing_type=sa.String(), + nullable=False, + ) + op.alter_column( + "projects_tags", "project_id", existing_type=sa.BIGINT(), nullable=True + ) + op.drop_constraint( + "study_tags_study_id_tag_id_key", "projects_tags", type_="unique" + ) + op.create_unique_constraint( + "project_tags_project_uuid_unique", + "projects_tags", + ["project_uuid_for_rut", "tag_id"], + ) + op.drop_constraint("study_tags_study_id_fkey", "projects_tags", type_="foreignkey") + op.create_foreign_key( + "project_tags_project_id_fkey", + "projects_tags", + "projects", + ["project_id"], + ["id"], + onupdate="CASCADE", + ondelete="SET NULL", + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + "project_tags_project_id_fkey", "projects_tags", type_="foreignkey" + ) + op.create_foreign_key( + "study_tags_study_id_fkey", + "projects_tags", + "projects", + ["project_id"], + ["id"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + op.drop_constraint( + "project_tags_project_uuid_unique", "projects_tags", type_="unique" + ) + op.create_unique_constraint( + "study_tags_study_id_tag_id_key", "projects_tags", ["project_id", "tag_id"] + ) + op.alter_column( + "projects_tags", "project_id", existing_type=sa.BIGINT(), nullable=False + ) + op.drop_column("projects_tags", "project_uuid_for_rut") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_tags.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_tags.py index 4ac88510e2d..223271872b7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_tags.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_tags.py @@ -13,9 +13,14 @@ sa.Column( "project_id", sa.BigInteger, - sa.ForeignKey(projects.c.id, onupdate="CASCADE", ondelete="CASCADE"), - nullable=False, - doc="NOTE that project.c.id != project.c.uuid", + sa.ForeignKey( + projects.c.id, + onupdate="CASCADE", + ondelete="SET NULL", + name="project_tags_project_id_fkey", + ), + nullable=True, # <-- NULL means that project was deleted + doc="NOTE that project.c.id != project.c.uuid. If project is deleted, we do not delete project in this table, we just set this column to NULL. Why? Because the `project_uuid_for_rut` is still used by resource usage tracker", ), sa.Column( "tag_id", @@ -23,5 +28,12 @@ sa.ForeignKey(tags.c.id, onupdate="CASCADE", ondelete="CASCADE"), nullable=False, ), - sa.UniqueConstraint("project_id", "tag_id"), + sa.Column( + "project_uuid_for_rut", + sa.String, + nullable=False, + ), + sa.UniqueConstraint( + "project_uuid_for_rut", "tag_id", name="project_tags_project_uuid_unique" + ), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_tags_sql.py b/packages/postgres-database/src/simcore_postgres_database/utils_tags_sql.py index bd727a0dcc3..072a6bd2d67 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_tags_sql.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_tags_sql.py @@ -1,4 +1,5 @@ import functools +from uuid import UUID import sqlalchemy as sa from simcore_postgres_database.models.groups import user_to_groups @@ -60,7 +61,7 @@ def get_tag_stmt( # aggregation ensures MOST PERMISSIVE policy of access-rights sa.func.bool_or(tags_access_rights.c.read).label("read"), sa.func.bool_or(tags_access_rights.c.write).label("write"), - sa.func.bool_or(tags_access_rights.c.delete).label("delete") + sa.func.bool_or(tags_access_rights.c.delete).label("delete"), ) .select_from( _join_user_to_given_tag( @@ -80,7 +81,7 @@ def list_tags_stmt(*, user_id: int): # aggregation ensures MOST PERMISSIVE policy of access-rights sa.func.bool_or(tags_access_rights.c.read).label("read"), sa.func.bool_or(tags_access_rights.c.write).label("write"), - sa.func.bool_or(tags_access_rights.c.delete).label("delete") + sa.func.bool_or(tags_access_rights.c.delete).label("delete"), ) .select_from( _join_user_to_tags( @@ -104,7 +105,7 @@ def count_groups_with_given_access_rights_stmt( tag_id: int, read: bool | None, write: bool | None, - delete: bool | None + delete: bool | None, ): """ How many groups (from this user_id) are given EXACTLY these access permissions @@ -192,12 +193,15 @@ def get_tags_for_project_stmt(*, project_index: int): ) -def add_tag_to_project_stmt(*, project_index: int, tag_id: int): +def add_tag_to_project_stmt( + *, project_index: int, tag_id: int, project_uuid_for_rut: UUID +): return ( pg_insert(projects_tags) .values( project_id=project_index, tag_id=tag_id, + project_uuid_for_rut=f"{project_uuid_for_rut}", ) .on_conflict_do_nothing() ) diff --git a/packages/postgres-database/tests/test_utils_tags.py b/packages/postgres-database/tests/test_utils_tags.py index 26f9a301f76..1f7f882da0a 100644 --- a/packages/postgres-database/tests/test_utils_tags.py +++ b/packages/postgres-database/tests/test_utils_tags.py @@ -668,6 +668,7 @@ def _check(func_smt, **kwargs): user_id = 425 # 4 tag_id = 4 project_index = 1 + project_uuid = "106f8b4b-ffb6-459a-a27b-981c779e6d3f" service_key = "simcore/services/comp/isolve" service_version = "2.0.85" @@ -726,6 +727,7 @@ def _check(func_smt, **kwargs): add_tag_to_project_stmt, project_index=project_index, tag_id=tag_id, + project_uuid_for_rut=project_uuid, ) _check( diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/service_runs.py index 6bceaab4f8c..3ff9f66f8b6 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/service_runs.py @@ -101,6 +101,7 @@ class Config: class ServiceRunWithCreditsDB(ServiceRunDB): osparc_credits: Decimal | None transaction_status: CreditTransactionStatus | None + project_tags: list[str] class Config: orm_mode = True diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py index 33a3e58d137..2301bf9e99f 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py @@ -28,6 +28,7 @@ from models_library.users import UserID from models_library.wallets import WalletID from pydantic import PositiveInt +from simcore_postgres_database.models.projects_tags import projects_tags from simcore_postgres_database.models.resource_tracker_credit_transactions import ( resource_tracker_credit_transactions, ) @@ -46,6 +47,7 @@ from simcore_postgres_database.models.resource_tracker_service_runs import ( resource_tracker_service_runs, ) +from simcore_postgres_database.models.tags import tags from sqlalchemy.dialects.postgresql import ARRAY, INTEGER from .....exceptions.errors import ( @@ -212,6 +214,15 @@ async def get_service_run_by_id( return None return ServiceRunDB.from_orm(row) + _project_tags_subquery = ( + sa.select( + projects_tags.c.project_uuid_for_rut, + sa.func.array_agg(tags.c.name).label("project_tags"), + ) + .select_from(projects_tags.join(tags, projects_tags.c.tag_id == tags.c.id)) + .group_by(projects_tags.c.project_uuid_for_rut) + ).subquery("project_tags_subquery") + async def list_service_runs_by_product_and_user_and_wallet( self, product_name: ProductName, @@ -260,6 +271,10 @@ async def list_service_runs_by_product_and_user_and_wallet( resource_tracker_service_runs.c.missed_heartbeat_counter, resource_tracker_credit_transactions.c.osparc_credits, resource_tracker_credit_transactions.c.transaction_status, + sa.func.coalesce( + self._project_tags_subquery.c.project_tags, + sa.cast(sa.text("'{}'"), sa.ARRAY(sa.String)), + ).label("project_tags"), ) .select_from( resource_tracker_service_runs.join( @@ -273,6 +288,11 @@ async def list_service_runs_by_product_and_user_and_wallet( == resource_tracker_credit_transactions.c.service_run_id ), isouter=True, + ).join( + self._project_tags_subquery, + resource_tracker_service_runs.c.project_id + == self._project_tags_subquery.c.project_uuid_for_rut, + isouter=True, ) ) .where(resource_tracker_service_runs.c.product_name == product_name) @@ -436,7 +456,9 @@ async def export_service_runs_table_to_s3( resource_tracker_service_runs.c.service_run_id, resource_tracker_service_runs.c.wallet_name, resource_tracker_service_runs.c.user_email, - resource_tracker_service_runs.c.project_name, + resource_tracker_service_runs.c.root_parent_project_name.label( + "project_name" + ), resource_tracker_service_runs.c.node_name, resource_tracker_service_runs.c.service_key, resource_tracker_service_runs.c.service_version, @@ -445,6 +467,10 @@ async def export_service_runs_table_to_s3( resource_tracker_service_runs.c.stopped_at, resource_tracker_credit_transactions.c.osparc_credits, resource_tracker_credit_transactions.c.transaction_status, + sa.func.coalesce( + self._project_tags_subquery.c.project_tags, + sa.cast(sa.text("'{}'"), sa.ARRAY(sa.String)), + ).label("project_tags"), ) .select_from( resource_tracker_service_runs.join( @@ -452,6 +478,11 @@ async def export_service_runs_table_to_s3( resource_tracker_service_runs.c.service_run_id == resource_tracker_credit_transactions.c.service_run_id, isouter=True, + ).join( + self._project_tags_subquery, + resource_tracker_service_runs.c.project_id + == self._project_tags_subquery.c.project_uuid_for_rut, + isouter=True, ) ) .where(resource_tracker_service_runs.c.product_name == product_name) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py index 782b084c789..a963b8340df 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py @@ -121,6 +121,7 @@ async def list_service_runs( user_email=service.user_email, project_id=service.project_id, project_name=service.project_name, + project_tags=service.project_tags, root_parent_project_id=service.root_parent_project_id, root_parent_project_name=service.root_parent_project_name, node_id=service.node_id, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py index 581952e1100..032b64a10fc 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py @@ -51,6 +51,7 @@ def mock_env(monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: "SC_BOOT_MODE": "production", "POSTGRES_CLIENT_NAME": "postgres_test_client", "RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED": "0", + "RESOURCE_USAGE_TRACKER_TRACING": "null", } setenvs_from_dict(monkeypatch, env_vars) return env_vars diff --git a/services/web/server/src/simcore_service_webserver/projects/db.py b/services/web/server/src/simcore_service_webserver/projects/db.py index 2281b807a71..5554bee1d18 100644 --- a/services/web/server/src/simcore_service_webserver/projects/db.py +++ b/services/web/server/src/simcore_service_webserver/projects/db.py @@ -403,7 +403,9 @@ async def list_projects( # pylint: disable=too-many-arguments,too-many-statemen sa.select( projects_tags.c.project_id, sa.func.array_agg(projects_tags.c.tag_id).label("tags"), - ).group_by(projects_tags.c.project_id) + ) + .where(projects_tags.c.project_id.is_not(None)) + .group_by(projects_tags.c.project_id) ).subquery("project_tags_subquery") ### @@ -1218,6 +1220,7 @@ async def add_tag( projects_tags.insert().values( project_id=project["id"], tag_id=tag_id, + project_uuid_for_rut=project["uuid"], ) ) project_tags.append(tag_id) diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py index 33b9d9146f5..9c8a29f2b6c 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py @@ -35,6 +35,7 @@ "user_email": "name@email.testing", "project_id": "5c2110be-441b-11ee-a0e8-02420a000040", "project_name": "osparc", + "project_tags": [], "node_id": "3d2133f4-aba4-4364-9f7a-9377dea1221f", "node_name": "sleeper", "root_parent_project_id": "5c2110be-441b-11ee-a0e8-02420a000040",