[maintenance] black-formatted packages/services and new common.Makefile

.vscode-template/settings.json

@@ -28,7 +28,7 @@
   },
   "python.testing.pyTestEnabled": true,
   "autoDocstring.docstringFormat": "sphinx",
-  "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck",
+  "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck.bash",
   "shellcheck.run": "onSave",
   "shellcheck.enableQuickFix": true
 }

Makefile

@@ -335,9 +335,9 @@ openapi-specs: ## bundles and validates openapi specifications and schemas of AL
 .PHONY: code-analysis
 code-analysis: .codeclimate.yml ## runs code-climate analysis
 	# Validates $<
-	./scripts/code-climate.sh validate-config
+	./scripts/code-climate.bash validate-config
 	# Running analysis
-	./scripts/code-climate.sh analyze
+	./scripts/code-climate.bash analyze
 
 
 .PHONY: info info-images info-swarm  info-tools

scripts/check_requirements.sh → scripts/check_requirements.bash

@@ -2,4 +2,4 @@
 
 # lists all python packages used throughout all the repository that are not tied to a specific version
 
-find . \( -name "requirements.txt" -o -name "common.txt" -o -name "devel.txt" -o -name "prod.txt" \) | xargs -I % grep -v "\-r " % | sort |uniq | awk '$0 !~ /==/'
+find . \( -name "requirements.txt" -o -name "common.txt" -o -name "devel.txt" -o -name "prod.txt" \)  | xargs -I % grep -v "\-r " % | sort |uniq | awk '$0 !~ /==/'

scripts/code-climate.sh → scripts/code-climate.bash

@@ -19,7 +19,8 @@ docker run \
   codeclimate/codeclimate "$@"
 
 
-if [ -z "$@" ];then
+if test -z "$@"
+then
   echo "----"
   echo "Listing other engines (in dockers)"
   docker images codeclimate/*

scripts/upgrade_test_requirements.sh → scripts/upgrade_test_requirements.bash

@@ -5,6 +5,6 @@
 
 for path_to_req_test in $(find ../ -type f -name '_test.txt')
 do
-  rm --verbose $path_to_req_test
-  make --directory $(dirname -- $path_to_req_test)
+  rm --verbose "$path_to_req_test"
+  make --directory "$(dirname -- "$path_to_req_test")"
 done

services/web/server/src/simcore_service_webserver/projects/projects_api.py

@@ -8,7 +8,7 @@
 """
 import logging
 from asyncio import ensure_future, gather
-from pprint import pprint
+from pprint import pformat
 from typing import Dict, Optional
 from uuid import uuid4
 
@@ -20,26 +20,31 @@
 
 from ..computation_api import delete_pipeline_db
 from ..director import director_api
-from ..storage_api import \
-    copy_data_folders_from_project  # mocked in unit-tests
-from ..storage_api import (delete_data_folders_of_project,
-                           delete_data_folders_of_project_node)
+from ..storage_api import copy_data_folders_from_project  # mocked in unit-tests
+from ..storage_api import (
+    delete_data_folders_of_project,
+    delete_data_folders_of_project_node,
+)
 from .config import CONFIG_SECTION_NAME
 from .projects_db import APP_PROJECT_DBAPI
 from .projects_exceptions import NodeNotFoundError, ProjectNotFoundError
 from .projects_utils import clone_project_document
 
 log = logging.getLogger(__name__)
 
+
 def _is_node_dynamic(node_key: str) -> bool:
     return "/dynamic/" in node_key
 
+
 def validate_project(app: web.Application, project: Dict):
     project_schema = app[APP_JSONSCHEMA_SPECS_KEY][CONFIG_SECTION_NAME]
-    validate_instance(project, project_schema) # TODO: handl
+    validate_instance(project, project_schema)  # TODO: handl
 
 
-async def get_project_for_user(app: web.Application, project_uuid, user_id, *, include_templates=False) -> Dict:
+async def get_project_for_user(
+    app: web.Application, project_uuid:str, user_id:int, *, include_templates: bool=False
+) -> Dict:
     """ Returns a project accessible to user
 
     :raises web.HTTPNotFound: if no match found
@@ -66,7 +71,9 @@ async def get_project_for_user(app: web.Application, project_uuid, user_id, *, i
         raise web.HTTPNotFound(reason="Project not found")
 
 
-async def clone_project(request: web.Request, project: Dict, user_id, forced_copy_project_id: str ="") -> Dict:
+async def clone_project(
+    request: web.Request, project: Dict, user_id: int, forced_copy_project_id: str=""
+) -> Dict:
     """Clones both document and data folders of a project
 
     - document
@@ -87,54 +94,86 @@ async def clone_project(request: web.Request, project: Dict, user_id, forced_cop
     """
     cloned_project, nodes_map = clone_project_document(project, forced_copy_project_id)
 
-    updated_project = await copy_data_folders_from_project(request.app,
-        project, cloned_project, nodes_map, user_id)
+    updated_project = await copy_data_folders_from_project(
+        request.app, project, cloned_project, nodes_map, user_id
+    )
 
     return updated_project
 
-async def start_project_interactive_services(request: web.Request, project: Dict, user_id: str) -> None:
+
+async def start_project_interactive_services(
+    request: web.Request, project: Dict, user_id: str
+) -> None:
     # first get the services if they already exist
-    log.debug("getting running interactive services of project %s for user %s", project["uuid"], user_id)
-    running_services = await director_api.get_running_interactive_services(request.app, user_id, project["uuid"])
+    log.debug(
+        "getting running interactive services of project %s for user %s",
+        project["uuid"],
+        user_id,
+    )
+    running_services = await director_api.get_running_interactive_services(
+        request.app, user_id, project["uuid"]
+    )
     running_service_uuids = [x["service_uuid"] for x in running_services]
     # now start them if needed
-    project_needed_services = {service_uuid:service for service_uuid, service in project["workbench"].items() \
-                                    if _is_node_dynamic(service["key"]) and \
-                                        service_uuid not in running_service_uuids}
-
-    start_service_tasks = [director_api.start_service(request.app,
-                            user_id=user_id,
-                            project_id=project["uuid"],
-                            service_key=service["key"],
-                            service_version=service["version"],
-                            service_uuid=service_uuid) for service_uuid, service in project_needed_services.items()]
+    project_needed_services = {
+        service_uuid: service
+        for service_uuid, service in project["workbench"].items()
+        if _is_node_dynamic(service["key"])
+        and service_uuid not in running_service_uuids
+    }
+
+    start_service_tasks = [
+        director_api.start_service(
+            request.app,
+            user_id=user_id,
+            project_id=project["uuid"],
+            service_key=service["key"],
+            service_version=service["version"],
+            service_uuid=service_uuid,
+        )
+        for service_uuid, service in project_needed_services.items()
+    ]
     await gather(*start_service_tasks)
 
 
-async def delete_project(request: web.Request, project_uuid: str, user_id: str) -> None:
+async def delete_project(request: web.Request, project_uuid: str, user_id: int) -> None:
     await delete_project_from_db(request, project_uuid, user_id)
+
     async def remove_services_and_data():
         await remove_project_interactive_services(user_id, project_uuid, request.app)
         await delete_project_data(request, project_uuid, user_id)
+
     ensure_future(remove_services_and_data())
 
+
 @observe(event="SIGNAL_PROJECT_CLOSE")
-async def remove_project_interactive_services(user_id: Optional[str], project_uuid: Optional[str], app: web.Application) -> None:
+async def remove_project_interactive_services(
+    user_id: Optional[int], project_uuid: Optional[str], app: web.Application
+) -> None:
     if not user_id and not project_uuid:
         raise ValueError("Expected either user or project")
 
-    list_of_services = await director_api.get_running_interactive_services(app,
-                                                                            project_id=project_uuid,
-                                                                            user_id=user_id)
-    stop_tasks = [director_api.stop_service(app, service["service_uuid"]) for service in list_of_services]
+    list_of_services = await director_api.get_running_interactive_services(
+        app, project_id=project_uuid, user_id=user_id
+    )
+    stop_tasks = [
+        director_api.stop_service(app, service["service_uuid"])
+        for service in list_of_services
+    ]
     if stop_tasks:
         await gather(*stop_tasks)
 
-async def delete_project_data(request: web.Request, project_uuid: str, user_id: str) -> None:
+
+async def delete_project_data(
+    request: web.Request, project_uuid: str, user_id: int
+) -> None:
     # requests storage to delete all project's stored data
     await delete_data_folders_of_project(request.app, project_uuid, user_id)
 
-async def delete_project_from_db(request: web.Request, project_uuid: str, user_id: str) -> None:
+
+async def delete_project_from_db(
+    request: web.Request, project_uuid: str, user_id: int
+) -> None:
     db = request.config_dict[APP_PROJECT_DBAPI]
     try:
         await delete_pipeline_db(request.app, project_uuid)
@@ -146,58 +185,104 @@ async def delete_project_from_db(request: web.Request, project_uuid: str, user_i
     # requests storage to delete all project's stored data
     await delete_data_folders_of_project(request.app, project_uuid, user_id)
 
-async def add_project_node(request: web.Request, project_uuid: str, user_id: str, service_key: str, service_version: str, service_id: Optional[str]) -> str: # pylint: disable=too-many-arguments
-    log.debug("starting node %s:%s in project %s for user %s", service_key, service_version, project_uuid, user_id)
+
+async def add_project_node(
+    request: web.Request,
+    project_uuid: str,
+    user_id: int,
+    service_key: str,
+    service_version: str,
+    service_id: Optional[str],
+) -> str:  # pylint: disable=too-many-arguments
+    log.debug(
+        "starting node %s:%s in project %s for user %s",
+        service_key,
+        service_version,
+        project_uuid,
+        user_id,
+    )
     node_uuid = service_id if service_id else str(uuid4())
     if _is_node_dynamic(service_key):
-        await director_api.start_service(request.app, user_id, project_uuid, service_key, service_version, node_uuid)
+        await director_api.start_service(
+            request.app, user_id, project_uuid, service_key, service_version, node_uuid
+        )
     return node_uuid
 
-async def get_project_node(request: web.Request, project_uuid: str, user_id:str, node_id: str):
-    log.debug("getting node %s in project %s for user %s", node_id, project_uuid, user_id)
 
-    list_of_interactive_services = await director_api.get_running_interactive_services(request.app,
-                                                                            project_id=project_uuid,
-                                                                            user_id=user_id)
+async def get_project_node(
+    request: web.Request, project_uuid: str, user_id: int, node_id: str
+):
+    log.debug(
+        "getting node %s in project %s for user %s", node_id, project_uuid, user_id
+    )
+
+    list_of_interactive_services = await director_api.get_running_interactive_services(
+        request.app, project_id=project_uuid, user_id=user_id
+    )
     # get the project if it is running
     for service in list_of_interactive_services:
         if service["service_uuid"] == node_id:
             return service
     # the service is not running, it's a computational service maybe
     # TODO: find out if computational service is running if not throw a 404 since it's not around
-    return {
-        "service_uuid": node_id,
-        "service_state": "idle"
-    }
+    return {"service_uuid": node_id, "service_state": "idle"}
 
-async def delete_project_node(request: web.Request, project_uuid: str, user_id: str, node_uuid: str) -> None:
-    log.debug("deleting node %s in project %s for user %s", node_uuid, project_uuid, user_id)
 
-    list_of_services = await director_api.get_running_interactive_services(request.app,
-                                                                            project_id=project_uuid,
-                                                                            user_id=user_id)
+async def delete_project_node(
+    request: web.Request, project_uuid: str, user_id: int, node_uuid: str
+) -> None:
+    log.debug(
+        "deleting node %s in project %s for user %s", node_uuid, project_uuid, user_id
+    )
+
+    list_of_services = await director_api.get_running_interactive_services(
+        request.app, project_id=project_uuid, user_id=user_id
+    )
     # stop the service if it is running
     for service in list_of_services:
         if service["service_uuid"] == node_uuid:
             await director_api.stop_service(request.app, node_uuid)
             break
     # remove its data if any
-    await delete_data_folders_of_project_node(request.app, project_uuid, node_uuid, user_id)
-
-
-async def update_project_node_progress(app: web.Application, user_id: str, project_id: str, node_id: str, progress: float) -> Optional[Dict]:
-    log.debug("updating node %s progress in project %s for user %s with %s", node_id, project_id, user_id, progress)
+    await delete_data_folders_of_project_node(
+        request.app, project_uuid, node_uuid, user_id
+    )
+
+
+async def update_project_node_progress(
+    app: web.Application, user_id: int, project_id: str, node_id: str, progress: float
+) -> Optional[Dict]:
+    log.debug(
+        "updating node %s progress in project %s for user %s with %s",
+        node_id,
+        project_id,
+        user_id,
+        progress,
+    )
     project = await get_project_for_user(app, project_id, user_id)
     if not node_id in project["workbench"]:
         raise NodeNotFoundError(project_id, node_id)
 
-    project["workbench"][node_id]["progress"] = int(100.0 * float(progress) + .5)
+    project["workbench"][node_id]["progress"] = int(100.0 * float(progress) + 0.5)
     db = app[APP_PROJECT_DBAPI]
     await db.update_user_project(project, user_id, project_id)
     return project["workbench"][node_id]
 
-async def update_project_node_outputs(app: web.Application, user_id: str, project_id: str, node_id: str, data: Optional[Dict]) -> Optional[Dict]:
-    log.debug("updating node %s outputs in project %s for user %s with %s", node_id, project_id, user_id, pprint(data))
+
+async def update_project_node_outputs(
+    app: web.Application,
+    user_id: int,
+    project_id: str,
+    node_id: str,
+    data: Optional[Dict],
+) -> Optional[Dict]:
+    log.debug(
+        "updating node %s outputs in project %s for user %s with %s",
+        node_id,
+        project_id,
+        user_id,
+        pformat(data),
+    )
     project = await get_project_for_user(app, project_id, user_id)
     if not node_id in project["workbench"]:
         raise NodeNotFoundError(project_id, node_id)

services/web/server/src/simcore_service_webserver/resource_manager/websocket_manager.py

@@ -90,4 +90,10 @@ async def find_users_of_resource(self, key: str, value: str) -> List[str]:
 @contextmanager
 def managed_resource(user_id: str, client_session_id: str, app: web.Application) -> WebsocketRegistry:
     registry = WebsocketRegistry(user_id, client_session_id, app)
-    yield registry
+    try:
+        yield registry
+    except Exception:
+        log.exception("Error in web-socket for user:%s, session:%s", user_id, client_session_id)
+        raise
+
+    # TODO: PC->SAN?? exception handling? e.g. remove resource from registry?