Cleanup webserver's project_*

Author: pcrespov

services/web/server/src/simcore_service_webserver/projects/projects_db.py

@@ -6,6 +6,7 @@
 """
 
 import logging
+import textwrap
 import uuid as uuidlib
 from collections import deque
 from datetime import datetime
@@ -170,7 +171,7 @@ async def add_project(
         force_project_uuid=False,
         force_as_template=False,
     ) -> Dict:
-        """ Inserts a new project in the database and, if a user is specified, it assigns ownership
+        """Inserts a new project in the database and, if a user is specified, it assigns ownership
 
         - A valid uuid is automaticaly assigned to the project except if force_project_uuid=False. In the latter case,
         invalid uuid will raise an exception.
@@ -193,7 +194,10 @@ async def add_project(
             # TODO: check best rollback design. see transaction.begin...
             # TODO: check if template, otherwise standard (e.g. template-  prefix in uuid)
             prj.update(
-                {"creationDate": now_str(), "lastChangeDate": now_str(),}
+                {
+                    "creationDate": now_str(),
+                    "lastChangeDate": now_str(),
+                }
             )
             kargs = _convert_to_db_names(prj)
             kargs.update(
@@ -249,13 +253,15 @@ async def load_user_projects(self, user_id: int) -> List[Dict]:
         log.info("Loading projects for user %s", user_id)
         async with self.engine.acquire() as conn:
             user_groups: List[RowProxy] = await self.__load_user_groups(conn, user_id)
-            query = f"""
-    SELECT *
-    FROM projects
-    WHERE projects.type != 'TEMPLATE'
-    AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}])
-    OR prj_owner = {user_id})
-    """
+            query = textwrap.dedent(
+                f"""\
+                SELECT *
+                FROM projects
+                WHERE projects.type != 'TEMPLATE'
+                AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}])
+                OR prj_owner = {user_id})
+                """
+            )
             projects_list = await self.__load_projects(
                 conn, query, user_id, user_groups
             )
@@ -272,16 +278,20 @@ async def load_template_projects(
 
         async with self.engine.acquire() as conn:
             user_groups: List[RowProxy] = await self.__load_user_groups(conn, user_id)
+
             # NOTE: in order to use specific postgresql function jsonb_exists_any we use raw call here
-            query = f"""
-SELECT *
-FROM projects
-WHERE projects.type = 'TEMPLATE'
-{'AND projects.published ' if only_published else ''}
-AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}])
-OR prj_owner = {user_id})
-            """
+            query = textwrap.dedent(
+                f"""\
+                SELECT *
+                FROM projects
+                WHERE projects.type = 'TEMPLATE'
+                {'AND projects.published ' if only_published else ''}
+                AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}])
+                OR prj_owner = {user_id})
+                """
+            )
             db_projects = await self.__load_projects(conn, query, user_id, user_groups)
+
             projects_list.extend(db_projects)
 
         return projects_list
@@ -337,15 +347,17 @@ async def _get_project(
             user_groups: List[RowProxy] = await self.__load_user_groups(conn, user_id)
 
             # NOTE: in order to use specific postgresql function jsonb_exists_any we use raw call here
-            query = f"""
-SELECT *
-FROM projects
-WHERE
-{"" if include_templates else "projects.type != 'TEMPLATE' AND"}
-uuid = '{project_uuid}'
-AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}])
-OR prj_owner = {user_id})
-"""
+            query = textwrap.dedent(
+                f"""\
+                SELECT *
+                FROM projects
+                WHERE
+                {"" if include_templates else "projects.type != 'TEMPLATE' AND"}
+                uuid = '{project_uuid}'
+                AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}])
+                OR prj_owner = {user_id})
+                """
+            )
             result = await conn.execute(query)
             project_row = await result.first()
 
@@ -392,7 +404,7 @@ async def remove_tag(self, user_id: int, project_uuid: str, tag_id: int) -> Dict
                 return _convert_to_schema_names(project, user_email)
 
     async def get_user_project(self, user_id: int, project_uuid: str) -> Dict:
-        """ Returns all projects *owned* by the user
+        """Returns all projects *owned* by the user
 
             - prj_owner
             - Notice that a user can have access to a template but he might not onw it
@@ -448,16 +460,21 @@ async def get_template_project(
         return template_prj
 
     async def update_user_project(
-        self, project_data: Dict, user_id: int, project_uuid: str, include_templates: Optional[bool] = False
+        self,
+        project_data: Dict,
+        user_id: int,
+        project_uuid: str,
+        include_templates: Optional[bool] = False,
     ):
-        """ updates a project from a user
-
-        """
+        """updates a project from a user"""
         log.info("Updating project %s for user %s", project_uuid, user_id)
 
         async with self.engine.acquire() as conn:
             row = await self._get_project(
-                user_id, project_uuid, exclude_foreign=["tags"], include_templates=include_templates
+                user_id,
+                project_uuid,
+                exclude_foreign=["tags"],
+                include_templates=include_templates,
             )
             user_groups: List[RowProxy] = await self.__load_user_groups(conn, user_id)
             _check_project_permissions(row, user_id, user_groups, "write")
@@ -505,7 +522,7 @@ async def delete_user_project(self, user_id: int, project_uuid: str):
             )
 
     async def make_unique_project_uuid(self) -> str:
-        """ Generates a project identifier still not used in database
+        """Generates a project identifier still not used in database
 
         WARNING: this method does not guarantee always unique id due to possible race condition
         (i.e. while client gets this uuid and uses it, another client might have used the same id already)

services/web/server/src/simcore_service_webserver/projects/projects_handlers.py

@@ -143,7 +143,12 @@ async def list_projects(request: web.Request):
             projects_api.validate_project(request.app, project)
             validated_projects.append(project)
         except ValidationError:
-            log.exception("Skipping invalid project from list")
+            log.warning(
+                "Invalid project with id='%s' in database."
+                "Skipping project from listed response."
+                "RECOMMENDED db data diagnose and cleanup",
+                project.get("uuid", "undefined"),
+            )
             continue
 
     return {"data": validated_projects}
@@ -152,9 +157,7 @@ async def list_projects(request: web.Request):
 @login_required
 @permission_required("project.read")
 async def get_project(request: web.Request):
-    """ Returns all projects accessible to a user (not necesarly owned)
-
-    """
+    """Returns all projects accessible to a user (not necesarly owned)"""
     # TODO: temporary hidden until get_handlers_from_namespace refactor to seek marked functions instead!
     user_id = request[RQT_USERID_KEY]
     from .projects_api import get_project_for_user
@@ -180,7 +183,7 @@ async def get_project(request: web.Request):
 @login_required
 @permission_required("services.pipeline.*")  # due to update_pipeline_db
 async def replace_project(request: web.Request):
-    """ Implements PUT /projects
+    """Implements PUT /projects
 
      In a PUT request, the enclosed entity is considered to be a modified version of
      the resource stored on the origin server, and the client is requesting that the
@@ -410,7 +413,10 @@ async def state_project(request: web.Request) -> web.Response:
 
     # check that project exists
     await get_project_for_user(
-        request.app, project_uuid=project_uuid, user_id=user_id, include_templates=True,
+        request.app,
+        project_uuid=project_uuid,
+        user_id=user_id,
+        include_templates=True,
     )
     with managed_resource(user_id, None, request.app) as rt:
         users_of_project = await rt.find_users_of_resource("project_id", project_uuid)