🎨 folder deletion (#6324)

Author: matusdrobuliak66

services/web/server/src/simcore_service_webserver/folders/_folders_api.py

@@ -7,14 +7,19 @@
 from models_library.api_schemas_webserver.folders_v2 import FolderGet, FolderGetPage
 from models_library.folders import FolderID
 from models_library.products import ProductName
+from models_library.projects import ProjectID
 from models_library.rest_ordering import OrderBy
 from models_library.users import UserID
 from models_library.workspaces import WorkspaceID
 from pydantic import NonNegativeInt
+from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY
+from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE
+from servicelib.utils import fire_and_forget_task
 from simcore_service_webserver.workspaces._workspaces_api import (
     check_user_workspace_access,
 )
 
+from ..projects.projects_api import submit_delete_project_task
 from ..users.api import get_user
 from ..workspaces.errors import (
     WorkspaceAccessForbiddenError,
@@ -224,7 +229,7 @@ async def update_folder(
         workspace_is_private = False
         user_folder_access_rights = user_workspace_access_rights.my_access_rights
 
-    # Check user has acces to the folder
+    # Check user has access to the folder
     await folders_db.get_for_user_or_workspace(
         app,
         folder_id=folder_id,
@@ -273,7 +278,7 @@ async def delete_folder(
         )
         workspace_is_private = False
 
-    # Check user has acces to the folder
+    # Check user has access to the folder
     await folders_db.get_for_user_or_workspace(
         app,
         folder_id=folder_id,
@@ -282,4 +287,32 @@ async def delete_folder(
         workspace_id=folder_db.workspace_id,
     )
 
-    await folders_db.delete(app, folder_id=folder_id, product_name=product_name)
+    # 1. Delete folder content
+    # 1.1 Delete all child projects that I am an owner
+    project_id_list: list[
+        ProjectID
+    ] = await folders_db.get_projects_recursively_only_if_user_is_owner(
+        app,
+        folder_id=folder_id,
+        private_workspace_user_id_or_none=user_id if workspace_is_private else None,
+        user_id=user_id,
+        product_name=product_name,
+    )
+
+    # fire and forget task for project deletion
+    for project_id in project_id_list:
+        fire_and_forget_task(
+            submit_delete_project_task(
+                app,
+                project_uuid=project_id,
+                user_id=user_id,
+                simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE,
+            ),
+            task_suffix_name=f"delete_project_task_{project_id}",
+            fire_and_forget_tasks_collection=app[APP_FIRE_AND_FORGET_TASKS_KEY],
+        )
+
+    # 1.2 Delete all child folders
+    await folders_db.delete_recursively(
+        app, folder_id=folder_id, product_name=product_name
+    )

services/web/server/src/simcore_service_webserver/folders/_folders_db.py

@@ -10,12 +10,16 @@
 from aiohttp import web
 from models_library.folders import FolderDB, FolderID
 from models_library.products import ProductName
+from models_library.projects import ProjectID
 from models_library.rest_ordering import OrderBy, OrderDirection
 from models_library.users import GroupID, UserID
 from models_library.workspaces import WorkspaceID
 from pydantic import NonNegativeInt
 from simcore_postgres_database.models.folders_v2 import folders_v2
+from simcore_postgres_database.models.projects import projects
+from simcore_postgres_database.models.projects_to_folders import projects_to_folders
 from sqlalchemy import func
+from sqlalchemy.orm import aliased
 from sqlalchemy.sql import asc, desc, select
 
 from ..db.plugin import get_database_engine
@@ -212,16 +216,109 @@ async def update(
         return FolderDB.from_orm(row)
 
 
-async def delete(
+async def delete_recursively(
     app: web.Application,
     *,
     folder_id: FolderID,
     product_name: ProductName,
 ) -> None:
-    async with get_database_engine(app).acquire() as conn:
+    async with get_database_engine(app).acquire() as conn, conn.begin():
+        # Step 1: Define the base case for the recursive CTE
+        base_query = select(
+            folders_v2.c.folder_id, folders_v2.c.parent_folder_id
+        ).where(
+            (folders_v2.c.folder_id == folder_id)  # <-- specified folder id
+            & (folders_v2.c.product_name == product_name)
+        )
+        folder_hierarchy_cte = base_query.cte(name="folder_hierarchy", recursive=True)
+        # Step 2: Define the recursive case
+        folder_alias = aliased(folders_v2)
+        recursive_query = select(
+            folder_alias.c.folder_id, folder_alias.c.parent_folder_id
+        ).select_from(
+            folder_alias.join(
+                folder_hierarchy_cte,
+                folder_alias.c.parent_folder_id == folder_hierarchy_cte.c.folder_id,
+            )
+        )
+        # Step 3: Combine base and recursive cases into a CTE
+        folder_hierarchy_cte = folder_hierarchy_cte.union_all(recursive_query)
+        # Step 4: Execute the query to get all descendants
+        final_query = select(folder_hierarchy_cte)
+        result = await conn.execute(final_query)
+        rows = (  # list of tuples [(folder_id, parent_folder_id), ...] ex. [(1, None), (2, 1)]
+            await result.fetchall() or []
+        )
+
+        # Sort folders so that child folders come first
+        sorted_folders = sorted(
+            rows, key=lambda x: (x[1] is not None, x[1]), reverse=True
+        )
+        folder_ids = [item[0] for item in sorted_folders]
         await conn.execute(
-            folders_v2.delete().where(
-                (folders_v2.c.folder_id == folder_id)
-                & (folders_v2.c.product_name == product_name)
+            folders_v2.delete().where(folders_v2.c.folder_id.in_(folder_ids))
+        )
+
+
+async def get_projects_recursively_only_if_user_is_owner(
+    app: web.Application,
+    *,
+    folder_id: FolderID,
+    private_workspace_user_id_or_none: UserID | None,
+    user_id: UserID,
+    product_name: ProductName,
+) -> list[ProjectID]:
+    """
+    The purpose of this function is to retrieve all projects within the provided folder ID.
+    These projects are subsequently deleted, so we only return projects where the user is the owner.
+    For future improvement, we can return all projects for which the user has delete permissions.
+    This permission check would require using the `workspace_access_rights` table for workspace projects,
+    or the `users_to_groups` table for private workspace projects.
+    """
+
+    async with get_database_engine(app).acquire() as conn, conn.begin():
+        # Step 1: Define the base case for the recursive CTE
+        base_query = select(
+            folders_v2.c.folder_id, folders_v2.c.parent_folder_id
+        ).where(
+            (folders_v2.c.folder_id == folder_id)  # <-- specified folder id
+            & (folders_v2.c.product_name == product_name)
+        )
+        folder_hierarchy_cte = base_query.cte(name="folder_hierarchy", recursive=True)
+        # Step 2: Define the recursive case
+        folder_alias = aliased(folders_v2)
+        recursive_query = select(
+            folder_alias.c.folder_id, folder_alias.c.parent_folder_id
+        ).select_from(
+            folder_alias.join(
+                folder_hierarchy_cte,
+                folder_alias.c.parent_folder_id == folder_hierarchy_cte.c.folder_id,
             )
         )
+        # Step 3: Combine base and recursive cases into a CTE
+        folder_hierarchy_cte = folder_hierarchy_cte.union_all(recursive_query)
+        # Step 4: Execute the query to get all descendants
+        final_query = select(folder_hierarchy_cte)
+        result = await conn.execute(final_query)
+        rows = (  # list of tuples [(folder_id, parent_folder_id), ...] ex. [(1, None), (2, 1)]
+            await result.fetchall() or []
+        )
+
+        folder_ids = [item[0] for item in rows]
+
+        query = (
+            select(projects_to_folders.c.project_uuid)
+            .join(projects)
+            .where(
+                (projects_to_folders.c.folder_id.in_(folder_ids))
+                & (projects_to_folders.c.user_id == private_workspace_user_id_or_none)
+            )
+        )
+        if private_workspace_user_id_or_none is not None:
+            query = query.where(projects.c.prj_owner == user_id)
+
+        result = await conn.execute(query)
+
+        rows = await result.fetchall() or []
+        results = [ProjectID(row[0]) for row in rows]
+        return results

services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py

@@ -320,6 +320,12 @@ async def create_project(  # pylint: disable=too-many-arguments,too-many-branche
                 # As user has access to the project, it has implicitly access to the folder
                 folder_id = prj_to_folder_db.folder_id
 
+            if as_template:
+                # For template we do not care about workspace/folder
+                workspace_id = None
+                new_project["workspaceId"] = workspace_id
+                folder_id = None
+
         if predefined_project:
             # 2. overrides with optional body and re-validate
             new_project, project_nodes = await _compose_project_data(

services/web/server/tests/unit/with_dbs/03/folders/test_folders.py

@@ -1,9 +1,12 @@
+import asyncio
+
 # pylint: disable=redefined-outer-name
 # pylint: disable=unused-argument
 # pylint: disable=unused-variable
 # pylint: disable=too-many-arguments
 # pylint: disable=too-many-statements
 from http import HTTPStatus
+from unittest import mock
 
 import pytest
 from aiohttp.test_utils import TestClient
@@ -16,6 +19,7 @@
     standard_role_response,
 )
 from servicelib.aiohttp import status
+from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY
 from simcore_service_webserver.db.models import UserRole
 from simcore_service_webserver.projects._groups_db import update_or_insert_project_group
 from simcore_service_webserver.projects.models import ProjectDict
@@ -336,3 +340,128 @@ async def test_project_listing_inside_of_private_folder(
         data, _ = await assert_status(resp, status.HTTP_200_OK)
         assert len(data) == 1
         assert data[0]["uuid"] == user_project["uuid"]
+
+
+@pytest.fixture
+def mock_storage_delete_data_folders(mocker: MockerFixture) -> mock.Mock:
+    mocker.patch(
+        "simcore_service_webserver.dynamic_scheduler.api.list_dynamic_services",
+        autospec=True,
+    )
+    mocker.patch(
+        "simcore_service_webserver.projects.projects_api.remove_project_dynamic_services",
+        autospec=True,
+    )
+    mocker.patch(
+        "simcore_service_webserver.projects._crud_api_delete.api.delete_pipeline",
+        autospec=True,
+    )
+    return mocker.patch(
+        "simcore_service_webserver.projects._crud_api_delete.delete_data_folders_of_project",
+        return_value=None,
+    )
+
+
+@pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)])
+async def test_folders_deletion(
+    client: TestClient,
+    logged_user: UserInfoDict,
+    user_project: ProjectDict,
+    expected: HTTPStatus,
+    mock_catalog_api_get_services_for_user_in_product: MockerFixture,
+    mock_storage_delete_data_folders: mock.Mock,
+):
+    assert client.app
+
+    # create a new folder
+    url = client.app.router["create_folder"].url_for()
+    resp = await client.post(url.path, json={"name": "My first folder"})
+    root_folder, _ = await assert_status(resp, status.HTTP_201_CREATED)
+    assert FolderGet.parse_obj(root_folder)
+
+    # create a subfolder folder
+    url = client.app.router["create_folder"].url_for()
+    resp = await client.post(
+        url.path,
+        json={
+            "name": "My subfolder 1",
+            "parentFolderId": root_folder["folderId"],
+        },
+    )
+    subfolder_1, _ = await assert_status(resp, status.HTTP_201_CREATED)
+
+    # create a subfolder folder
+    url = client.app.router["create_folder"].url_for()
+    resp = await client.post(
+        url.path,
+        json={
+            "name": "My subfolder 2",
+            "parentFolderId": root_folder["folderId"],
+        },
+    )
+    subfolder_2, _ = await assert_status(resp, status.HTTP_201_CREATED)
+
+    # add project to the sub folder
+    url = client.app.router["replace_project_folder"].url_for(
+        folder_id=f"{subfolder_2['folderId']}",
+        project_id=f"{user_project['uuid']}",
+    )
+    resp = await client.put(url.path)
+    await assert_status(resp, status.HTTP_204_NO_CONTENT)
+
+    # create a sub sub folder folder
+    url = client.app.router["create_folder"].url_for()
+    resp = await client.post(
+        url.path,
+        json={
+            "name": "My sub sub folder",
+            "parentFolderId": subfolder_1["folderId"],
+        },
+    )
+    await assert_status(resp, status.HTTP_201_CREATED)
+
+    # list user folders
+    url = client.app.router["list_folders"].url_for()
+    resp = await client.get(url.path)
+    data, _ = await assert_status(resp, status.HTTP_200_OK)
+    assert len(data) == 1
+
+    # list subfolder projects
+    base_url = client.app.router["list_projects"].url_for()
+    url = base_url.with_query({"folder_id": f"{subfolder_2['folderId']}"})
+    resp = await client.get(url)
+    data, _ = await assert_status(resp, status.HTTP_200_OK)
+    assert len(data) == 1
+    assert data[0]["uuid"] == user_project["uuid"]
+
+    # list root projects
+    base_url = client.app.router["list_projects"].url_for()
+    resp = await client.get(base_url)
+    data, _ = await assert_status(resp, status.HTTP_200_OK)
+    assert len(data) == 0
+
+    # delete a subfolder
+    url = client.app.router["delete_folder"].url_for(
+        folder_id=f"{subfolder_1['folderId']}"
+    )
+    resp = await client.delete(url.path)
+    await assert_status(resp, status.HTTP_204_NO_CONTENT)
+
+    # delete a root folder
+    url = client.app.router["delete_folder"].url_for(
+        folder_id=f"{root_folder['folderId']}"
+    )
+    resp = await client.delete(url.path)
+    await assert_status(resp, status.HTTP_204_NO_CONTENT)
+
+    fire_and_forget_tasks = client.app[APP_FIRE_AND_FORGET_TASKS_KEY]
+    t: asyncio.Task = list(fire_and_forget_tasks)[0]
+    assert t.get_name().startswith("fire_and_forget_task_delete_project_task_")
+    await t
+    assert len(client.app[APP_FIRE_AND_FORGET_TASKS_KEY]) == 0
+
+    # list root projects (The project should have been deleted)
+    base_url = client.app.router["list_projects"].url_for()
+    resp = await client.get(base_url)
+    data, _ = await assert_status(resp, status.HTTP_200_OK)
+    assert len(data) == 0