Retrieve more states (backend) (#2107)

* compute modified state and dependencies
* moved ComputationTask model to models-library
* modified project json schema
* bugfix: issue webserver wrongly retrieves templates running state, creating log pollution #2108
* removed thumbnail as str in Project
* bugfix: downgrading the projects table now works
* fix node hashes after import based on original hash
* moved ProjectAtDB to the models_library
* fix model to use python snake case
* @pcrespov bonus: sidecar says which mode it uses in the logo
* migration of workbench according to new state object
* small fixes to export/import  (ensure new eTag are generated, ensure new runHash are generated if needed)
* silence logs. sometimes exceptions from requesting are expected and set as a warning

Author: sanderegg

api/specs/common/schemas/project-v0.0.1-converted.yaml

@@ -286,20 +286,42 @@ properties:
                   - '15'
             deprecated: true
           state:
-            title: RunningState
-            description: the node's running state
-            default: NOT_STARTED
-            enum:
-              - UNKNOWN
-              - NOT_STARTED
-              - PUBLISHED
-              - PENDING
-              - STARTED
-              - RETRY
-              - SUCCESS
-              - FAILED
-              - ABORTED
-            type: string
+            title: NodeState
+            type: object
+            properties:
+              modified:
+                title: Modified
+                description: true if the node's outputs need to be re-computed
+                default: true
+                type: boolean
+              dependencies:
+                title: Dependencies
+                description: >-
+                  contains the node inputs dependencies if they need to be
+                  computed first
+                type: array
+                uniqueItems: true
+                items:
+                  type: string
+                  format: uuid
+              currentStatus:
+                description: the node's current state
+                default: NOT_STARTED
+                example:
+                  - RUNNING
+                  - FAILED
+                enum:
+                  - UNKNOWN
+                  - PUBLISHED
+                  - NOT_STARTED
+                  - PENDING
+                  - STARTED
+                  - RETRY
+                  - SUCCESS
+                  - FAILED
+                  - ABORTED
+                type: string
+            additionalProperties: false
     additionalProperties: true
   ui:
     type: object

api/specs/common/schemas/project-v0.0.1.json

@@ -397,21 +397,47 @@
               "deprecated": true
             },
             "state": {
-              "title": "RunningState",
-              "description": "the node's running state",
-              "default": "NOT_STARTED",
-              "enum": [
-                "UNKNOWN",
-                "NOT_STARTED",
-                "PUBLISHED",
-                "PENDING",
-                "STARTED",
-                "RETRY",
-                "SUCCESS",
-                "FAILED",
-                "ABORTED"
-              ],
-              "type": "string"
+              "title": "NodeState",
+              "type": "object",
+              "properties": {
+                "modified": {
+                  "title": "Modified",
+                  "description": "true if the node's outputs need to be re-computed",
+                  "default": true,
+                  "type": "boolean"
+                },
+                "dependencies": {
+                  "title": "Dependencies",
+                  "description": "contains the node inputs dependencies if they need to be computed first",
+                  "type": "array",
+                  "uniqueItems": true,
+                  "items": {
+                    "type": "string",
+                    "format": "uuid"
+                  }
+                },
+                "currentStatus": {
+                  "description": "the node's current state",
+                  "default": "NOT_STARTED",
+                  "examples": [
+                    "RUNNING",
+                    "FAILED"
+                  ],
+                  "enum": [
+                    "UNKNOWN",
+                    "PUBLISHED",
+                    "NOT_STARTED",
+                    "PENDING",
+                    "STARTED",
+                    "RETRY",
+                    "SUCCESS",
+                    "FAILED",
+                    "ABORTED"
+                  ],
+                  "type": "string"
+                }
+              },
+              "additionalProperties": false
             }
           }
         }

packages/models-library/requirements/ci.txt

@@ -12,6 +12,7 @@
 
 # installs this repo's packages
 ../pytest-simcore/
+../postgres-database/
 
 # current module
 .

packages/models-library/requirements/dev.txt

@@ -13,6 +13,7 @@
 
 # installs this repo's packages
 -e ../pytest-simcore/
+-e ../postgres-database/[migration]
 
 # current module
 -e .

packages/models-library/src/models_library/projects.py

@@ -2,7 +2,9 @@
     Models a study's project document
 """
 from copy import deepcopy
-from typing import Any, Dict, List, Optional, Union
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional
 from uuid import UUID
 
 from pydantic import BaseModel, EmailStr, Extra, Field, HttpUrl, validator
@@ -21,7 +23,19 @@
 Workbench = Dict[NodeID_AsDictKey, Node]
 
 
-class Project(BaseModel):
+# NOTE: careful this is in sync with packages/postgres-database/src/simcore_postgres_database/models/projects.py!!!
+class ProjectType(str, Enum):
+    """
+    template: template project
+    standard: standard project
+    """
+
+    TEMPLATE = "TEMPLATE"
+    STANDARD = "STANDARD"
+
+
+class ProjectCommons(BaseModel):
+    # Description of the project
     uuid: ProjectID = Field(
         ...,
         description="project unique identifier",
@@ -30,8 +44,6 @@ class Project(BaseModel):
             "9bcf8feb-c1b1-41b6-b201-639cd6ccdba8",
         ],
     )
-
-    # Description of the project
     name: str = Field(
         ..., description="project name", examples=["Temporal Distortion Simulator"]
     )
@@ -40,33 +52,73 @@ class Project(BaseModel):
         description="longer one-line description about the project",
         examples=["Dabbling in temporal transitions ..."],
     )
-    # NOTE: str is necessary because HttpUrl will not accept and empty string and the
-    # frontend sometimes sends this empty string, which is removed by the validator
-    thumbnail: Union[str, HttpUrl] = Field(
+    thumbnail: Optional[HttpUrl] = Field(
         ...,
         description="url of the project thumbnail",
         examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"],
     )
 
-    # Ownership and Access  (SEE projects_access.py)
-    prjOwner: EmailStr = Field(..., description="user email")
-    accessRights: Dict[GroupID, AccessRights] = Field(
-        ...,
-        description="object containing the GroupID as key and read/write/execution permissions as value",
+    creation_date: datetime = Field(...)
+    last_change_date: datetime = Field(...)
+
+    # Pipeline of nodes (SEE projects_nodes.py)
+    workbench: Workbench = Field(...)
+
+    @validator("thumbnail", always=True, pre=True)
+    @classmethod
+    def convert_empty_str_to_none(v):
+        if isinstance(v, str) and v == "":
+            return None
+        return v
+
+
+class ProjectAtDB(ProjectCommons):
+    # specific DB fields
+    id: int = Field(..., description="The table primary index")
+    project_type: ProjectType = Field(..., alias="type", description="The project type")
+    prj_owner: Optional[int] = Field(..., description="The project owner id")
+    published: Optional[bool] = Field(
+        False, description="Defines if a study is available publicly"
     )
 
+    @validator("project_type", pre=True)
+    @classmethod
+    def convert_sql_alchemy_enum(cls, v):
+        if isinstance(v, Enum):
+            return v.value
+        return v
+
+    class Config:
+        orm_mode = True
+        use_enum_values = True
+
+
+class Project(ProjectCommons):
+    # NOTE: This is the pydantic pendant of project-v0.0.1.json used in the API of the webserver/webclient
+    # NOT for usage with DB!!
+
+    # Ownership and Access  (SEE projects_access.py)
+    prj_owner: EmailStr = Field(..., description="user email", alias="prjOwner")
+
     # Timestamps   TODO: should we use datetime??
-    creationDate: str = Field(
+    creation_date: str = Field(
         ...,
         regex=DATE_RE,
         description="project creation date",
         examples=["2018-07-01T11:13:43Z"],
+        alias="creationDate",
     )
-    lastChangeDate: str = Field(
+    last_change_date: str = Field(
         ...,
         regex=DATE_RE,
         description="last save date",
         examples=["2018-07-01T11:13:43Z"],
+        alias="lastChangeDate",
+    )
+    access_rights: Dict[GroupID, AccessRights] = Field(
+        ...,
+        description="object containing the GroupID as key and read/write/execution permissions as value",
+        alias="accessRights",
     )
 
     # Classification
@@ -77,9 +129,6 @@ class Project(BaseModel):
         examples=["some:id:to:a:classifier"],
     )
 
-    # Pipeline of nodes (SEE projects_nodes.py)
-    workbench: Workbench = ...
-
     # Project state (SEE projects_state.py)
     state: Optional[ProjectState] = None
 
@@ -94,13 +143,6 @@ class Project(BaseModel):
     # Dev only
     dev: Optional[Dict] = Field(description="object used for development purposes only")
 
-    @classmethod
-    @validator("thumbnail")
-    def null_thumbnail(cls, v):
-        if isinstance(v, str) and v == "":
-            return None
-        return v
-
     class Config:
         description = "Document that stores metadata, pipeline and UI setup of a study"
         title = "osparc-simcore project"