validation now done through pydantic

validation done when loading from DB
filtering of projects done directly after load

Author: sanderegg

services/web/server/src/simcore_service_webserver/projects/projects_db.py

@@ -4,7 +4,7 @@
     - Shall be used as entry point for all the queries to the database regarding projects
 
 """
-
+import asyncio
 import logging
 import textwrap
 import uuid as uuidlib
@@ -22,6 +22,8 @@
 from aiopg.sa.connection import SAConnection
 from aiopg.sa.result import ResultProxy, RowProxy
 from change_case import ChangeCase
+from jsonschema.exceptions import ValidationError
+from models_library.projects import ProjectAtDB
 from servicelib.application_keys import APP_DB_ENGINE_KEY
 from simcore_postgres_database.webserver_models import ProjectType, projects
 from sqlalchemy import literal_column
@@ -37,6 +39,7 @@
     ProjectsException,
 )
 from .projects_fakes import Fake
+from .projects_utils import project_uses_available_services
 
 log = logging.getLogger(__name__)
 
@@ -307,7 +310,11 @@ async def add_project(
                 prj["tags"] = []
             return prj
 
-    async def load_user_projects(self, user_id: int) -> List[Dict]:
+    async def load_user_projects(
+        self,
+        user_id: int,
+        filter_by_services: Optional[List[Dict]] = None,
+    ) -> List[Dict]:
         log.info("Loading projects for user %s", user_id)
         async with self.engine.acquire() as conn:
             user_groups: List[RowProxy] = await self.__load_user_groups(conn, user_id)
@@ -330,13 +337,17 @@ async def load_user_projects(self, user_id: int) -> List[Dict]:
                 """
             )
             projects_list = await self.__load_projects(
-                conn, query, user_id, user_groups
+                conn, query, user_id, user_groups, filter_by_services=filter_by_services
             )
 
         return projects_list
 
     async def load_template_projects(
-        self, user_id: int, *, only_published=False
+        self,
+        user_id: int,
+        *,
+        only_published=False,
+        filter_by_services: Optional[List[Dict]] = None,
     ) -> List[Dict]:
         log.info("Loading public template projects")
 
@@ -358,7 +369,9 @@ async def load_template_projects(
                 """
             )
 
-            db_projects = await self.__load_projects(conn, query, user_id, user_groups)
+            db_projects = await self.__load_projects(
+                conn, query, user_id, user_groups, filter_by_services
+            )
 
             projects_list.extend(db_projects)
 
@@ -378,7 +391,12 @@ async def __load_user_groups(
         return user_groups
 
     async def __load_projects(
-        self, conn: SAConnection, query: str, user_id: int, user_groups: List[RowProxy]
+        self,
+        conn: SAConnection,
+        query: str,
+        user_id: int,
+        user_groups: List[RowProxy],
+        filter_by_services: Optional[List[Dict]] = None,
     ) -> List[Dict]:
         api_projects: List[Dict] = []  # API model-compatible projects
         db_projects: List[Dict] = []  # DB model-compatible projects
@@ -387,7 +405,21 @@ async def __load_projects(
                 _check_project_permissions(row, user_id, user_groups, "read")
             except ProjectInvalidRightsError:
                 continue
+            try:
+                import concurrent.futures
+
+                with concurrent.futures.ThreadPoolExecutor() as pool:
+                    await asyncio.get_event_loop().run_in_executor(
+                        pool, ProjectAtDB.from_orm, row
+                    )
+                # ProjectAtDB.from_orm(row)
+            except ValidationError as exc:
+                log.warning("project failed validation: %s", exc)
+                continue
             prj = dict(row.items())
+            if filter_by_services:
+                if not await project_uses_available_services(prj, filter_by_services):
+                    continue
             db_projects.append(prj)
 
         # NOTE: DO NOT nest _get_tags_by_project in async loop above !!!

services/web/server/src/simcore_service_webserver/projects/projects_handlers.py

@@ -1,7 +1,6 @@
 """ Handlers for CRUD operations on /projects/
 
 """
-import asyncio
 import json
 import logging
 from typing import Any, Dict, List, Optional, Set
@@ -143,14 +142,24 @@ async def set_all_project_states(projects: List[Dict[str, Any]], is_template: bo
             reraise=True,
         )
 
+    user_available_services: List[
+        Dict
+    ] = await catalog.get_services_for_user_in_product(
+        request.app, user_id, product_name, only_key_versions=True
+    )
+
     projects_list = []
     if ptype in ("template", "all"):
-        template_projects = await db.load_template_projects(user_id=user_id)
+        template_projects = await db.load_template_projects(
+            user_id=user_id, filter_by_services=user_available_services
+        )
         await set_all_project_states(template_projects, is_template=True)
         projects_list += template_projects
 
     if ptype in ("user", "all"):  # standard only (notice that templates will only)
-        user_projects = await db.load_user_projects(user_id=user_id)
+        user_projects = await db.load_user_projects(
+            user_id=user_id, filter_by_services=user_available_services
+        )
         await set_all_project_states(user_projects, is_template=False)
         projects_list += user_projects
 
@@ -159,35 +168,7 @@ async def set_all_project_states(projects: List[Dict[str, Any]], is_template: bo
 
     stop = min(start + count, len(projects_list))
     projects_list = projects_list[start:stop]
-    user_available_services: List[
-        Dict
-    ] = await catalog.get_services_for_user_in_product(
-        request.app, user_id, product_name, only_key_versions=True
-    )
-
-    # validate response
-    async def validate_project(prj: Dict[str, Any]) -> Dict[str, Any]:
-        try:
-
-            await asyncio.get_event_loop().run_in_executor(
-                None, projects_api.validate_project, request.app, prj
-            )
-            if await project_uses_available_services(prj, user_available_services):
-                return prj
-        except ValidationError:
-            log.warning(
-                "Invalid project with id='%s' in database."
-                "Skipping project from listed response."
-                "RECOMMENDED db data diagnose and cleanup",
-                prj.get("uuid", "undefined"),
-            )
-
-    validation_tasks = [validate_project(project) for project in projects_list]
-    # FIXME: if some invalid, then it should not reraise but instead
-    results = await logged_gather(*validation_tasks, reraise=True)
-    validated_projects = [r for r in results if r]
-
-    return {"data": validated_projects}
+    return {"data": projects_list}
 
 
 @login_required