Feature export/import study (backend)

.env-devel

@@ -61,6 +61,7 @@ WEBSERVER_PROMETHEUS_PORT=9090
 WEBSERVER_PROMETHEUS_API_VERSION=v1
 WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS=900
 WEBSERVER_GARBAGE_COLLECTION_INTERVAL_SECONDS=30
+WEBSERVER_MAX_UPLOAD_FILE_SIZE_GB=10
 WEBSERVER_MANUAL_MAIN_URL=http://docs.osparc.io/
 WEBSERVER_MANUAL_EXTRA_URL=https://itisfoundation.github.io/osparc-manual-z43/
 WEBSERVER_FOGBUGZ_LOGIN_URL=https://z43.manuscript.com/login

api/specs/webserver/openapi-projects.yaml

@@ -194,6 +194,60 @@ paths:
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
 
+  /projects/{project_id}/export:
+    parameters:
+      - name: project_id
+        in: path
+        required: true
+        schema:
+          type: string
+      - name: compressed
+        in: query
+        required: false
+        schema:
+          type: boolean
+    post:
+      tags:
+        - project
+      summary: creates an archive of the project and downloads it
+      operationId: export_project
+      responses:
+        "200":
+          description: creates an archive from a project file
+          content:
+            application/zip:
+              schema:
+                type: string
+                format: binary
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+
+  /projects/import:
+    post:
+      tags:
+        - project
+      summary: Create new project from an archive
+      operationId: import_project
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              type: object
+              properties:
+                fileName:
+                  type: string
+                  format: binary
+      responses:
+        "201":
+          description: creates a new project from an archive
+          content:
+            application/json:
+              schema:
+                #TODO: change this with an OK response
+                $ref: "#/components/schemas/ProjectEnveloped"
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+
   /projects/{project_id}/close:
     parameters:
       - name: project_id

api/specs/webserver/openapi.yaml

@@ -125,7 +125,6 @@ paths:
   /groups/sparc/classifiers/scicrunch-resources:search:
     $ref: "./openapi-groups.yaml#/paths/~1groups~1sparc~1classifiers~1scicrunch-resources:search"
 
-
   # DATA STORAGE SERVICES ----------------------------------------------------------
 
   /storage/locations:
@@ -169,6 +168,13 @@ paths:
   /projects/{project_id}/state:
     $ref: "./openapi-projects.yaml#/paths/~1projects~1{project_id}~1state"
 
+  # FIXME: /projects/{project_id}:export dose not work
+  /projects/{project_id}/export:
+    $ref: "./openapi-projects.yaml#/paths/~1projects~1{project_id}~1export"
+
+  /projects:import:
+    $ref: "./openapi-projects.yaml#/paths/~1projects~1import"
+
   /projects/{project_id}:close:
     $ref: "./openapi-projects.yaml#/paths/~1projects~1{project_id}~1close"
 

packages/models-library/src/models_library/basic_types.py

@@ -1,6 +1,6 @@
 from enum import Enum
 
-from pydantic import conint, constr, confloat
+from pydantic import confloat, conint, constr
 
 PortInt = conint(gt=0, lt=65535)
 

packages/models-library/src/models_library/projects.py

@@ -2,10 +2,10 @@
     Models a study's project document
 """
 from copy import deepcopy
-from typing import Dict, List, Optional, Any
+from typing import Any, Dict, List, Optional, Union
 from uuid import UUID
 
-from pydantic import BaseModel, EmailStr, Extra, Field, HttpUrl
+from pydantic import BaseModel, EmailStr, Extra, Field, HttpUrl, validator
 
 from .basic_regex import DATE_RE
 from .projects_access import AccessRights, GroupID
@@ -40,7 +40,9 @@ class Project(BaseModel):
         description="longer one-line description about the project",
         examples=["Dabbling in temporal transitions ..."],
     )
-    thumbnail: HttpUrl = Field(
+    # NOTE: str is necessary because HttpUrl will not accept and empty string and the
+    # frontend sometimes sends this empty string, which is removed by the validator
+    thumbnail: Union[str, HttpUrl] = Field(
         ...,
         description="url of the project thumbnail",
         examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"],
@@ -88,9 +90,14 @@ class Project(BaseModel):
     quality: Dict[str, Any] = {}
 
     # Dev only
-    dev: Optional[Dict] = Field(
-        description="object used for development purposes only"
-    )
+    dev: Optional[Dict] = Field(description="object used for development purposes only")
+
+    @classmethod
+    @validator("thumbnail")
+    def null_thumbnail(cls, v):
+        if isinstance(v, str) and v == "":
+            return None
+        return v
 
     class Config:
         description = "Document that stores metadata, pipeline and UI setup of a study"

packages/service-library/src/servicelib/rest_middlewares.py

@@ -172,6 +172,9 @@ async def _middleware_handler(request: web.Request, handler):
 
         resp = await handler(request)
 
+        if isinstance(resp, web.FileResponse):  # allows for files to be downloaded
+            return resp
+
         if not isinstance(resp, web.Response):
             response = create_data_response(
                 data=resp, skip_internal_error_details=_is_prod,

services/director-v2/src/simcore_service_director_v2/utils/client_decorators.py

@@ -75,7 +75,7 @@ async def wrapper_func(*args, **kwargs) -> httpx.Response:
                         service_name,
                         resp.reason_phrase,
                         resp.status_code,
-                        resp.text(),
+                        resp.text,
                     )
                     raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE)
 

services/web/server/requirements/_base.in

@@ -30,5 +30,6 @@ python-socketio
 semantic_version
 aiodebug
 json2html
+parfive
 
 pydantic[email]

services/web/server/requirements/_base.txt

@@ -51,6 +51,7 @@ multidict==4.7.6          # via aiohttp, yarl
 openapi-core==0.12.0      # via -r requirements/../../../../packages/service-library/requirements/_base.in
 openapi-spec-validator==0.2.9  # via openapi-core
 pamqp==2.3.0              # via aiormq
+parfive==1.0.2            # via -r requirements/_base.in
 passlib==1.7.2            # via -r requirements/_base.in
 prometheus-client==0.9.0  # via -r requirements/../../../../packages/service-library/requirements/_base.in
 prompt-toolkit==3.0.8     # via click-repl
@@ -68,6 +69,7 @@ six==1.15.0               # via click-repl, cryptography, isodate, jsonschema, o
 sqlalchemy[postgresql_psycopg2binary]==1.3.20  # via -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt, -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt, -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt, -c requirements/../../../../requirements/constraints.txt, -r requirements/../../../../packages/postgres-database/requirements/_base.in, -r requirements/../../../../packages/service-library/requirements/_base.in, aiopg
 strict-rfc3339==0.7       # via openapi-core
 tenacity==6.2.0           # via -r requirements/../../../../packages/service-library/requirements/_base.in
+tqdm==4.54.1              # via parfive
 trafaret==2.0.2           # via -r requirements/../../../../packages/service-library/requirements/_base.in
 typing-extensions==3.7.4.2  # via aiohttp, yarl
 ujson==3.1.0              # via -r requirements/../../../../packages/service-library/requirements/_base.in, aiohttp-swagger