✨Run dynamic services via dynamic-sidecar (🏗️ OPS + CI action)

.env-devel

@@ -43,6 +43,9 @@ S3_SECURE=0
 SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key
 
+# used by dynamic-sidecar to target the service inside the stack
+DYNAMIC_SIDECAR_IMAGE=local/dynamic-sidecar:development
+
 SMTP_HOST=mail.speag.com
 SMTP_PORT=25
 

README.md

@@ -104,6 +104,26 @@ In **windows**, it works under [WSL] (windows subsystem for linux). Some details
 
 In **MacOS**, [replacing the MacOS utilities with GNU utils](https://apple.stackexchange.com/a/69332) might be required.
 
+#### Upgrading services requirements
+
+Updates are upgraded using a docker container and pip-sync.
+Build and start the container:
+
+    cd requirements/tools
+    make build
+    make shell
+
+Once inside the container navigate to the service's requirements directory.
+
+To upgrade all requirements run:
+
+    make reqs
+
+To upgrade a single requirement named `fastapi`run:
+
+    make reqs upgrade=fastapi
+
+
 ## Releases
 
 **WARNING** This application is **still under development**.

requirements/tools/Makefile

@@ -61,8 +61,8 @@ reqs: _check_py36_version ## updates requirements of all package libraries
 
 
 # SEE https://medium.com/faun/set-current-host-user-for-docker-container-4e521cef9ffc
-.PHONY: build
-build build-nc:
+.PHONY: build	
+build build-nc: ## builds a container which can be used to upgrade depnencies
 	docker build $(if $(findstring -nc,$@),--no-cache,) --tag local/python-devkit:latest .
 
 
@@ -81,7 +81,7 @@ run: build ## Runs upgrade in a container
 
 
 .PHONY: shell
-shell:
+shell:	## mount the currrent project inside the container to update the requirements
 	docker run -it \
 		--workdir="/home/$(USER)/requirements/tools" \
     --volume="/etc/group:/etc/group:ro" \

services/director-v2/docker/entrypoint.sh

@@ -69,6 +69,24 @@ if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ]; then
   pip install --no-cache-dir ptvsd
 fi
 
+# Appends docker group if socket is mounted
+DOCKER_MOUNT=/var/run/docker.sock
+if stat $DOCKER_MOUNT > /dev/null 2>&1
+then
+    echo "$INFO detected docker socket is mounted, adding user to group..."
+    GROUPID=$(stat --format=%g $DOCKER_MOUNT)
+    GROUPNAME=scdocker
+
+    if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1
+    then
+        echo "$WARNING docker group with $GROUPID already exists, getting group name..."
+        # if group already exists in container, then reuse name
+        GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1)
+        echo "$WARNING docker group with $GROUPID has name $GROUPNAME"
+    fi
+    adduser "$SC_USER_NAME" "$GROUPNAME"
+fi
+
 echo "$INFO Starting $* ..."
 echo "  $SC_USER_NAME rights    : $(id "$SC_USER_NAME")"
 echo "  local dir : $(ls -al)"

services/director-v2/requirements/_base.in

@@ -32,3 +32,6 @@ orjson
 
 # clients
 httpx
+
+# other
+asyncio_extras

services/director-v2/requirements/_base.txt

@@ -23,11 +23,14 @@ async-exit-stack==1.0.1 ; python_version < "3.7"
 async-generator==1.10 ; python_version < "3.7"
     # via
     #   -r requirements/_base.in
+    #   asyncio-extras
     #   fastapi
 async-timeout==3.0.1
     # via
     #   aiohttp
     #   aiopg
+asyncio-extras==1.3.2
+    # via -r requirements/_base.in
 attrs==20.3.0
     # via aiohttp
 billiard==3.6.3.0

services/director-v2/src/simcore_service_director_v2/api/entrypoints.py

@@ -33,6 +33,7 @@
 v2_router.include_router(
     dynamic_services.router, tags=["dynamic services"], prefix="/dynamic_services"
 )
+v2_router.include_router(dynamic_services.router, tags=["dynamic-sidecar"])
 
 # root
 api_router = APIRouter()

services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py

@@ -1,9 +1,11 @@
 import logging
 
 import httpx
-from fastapi import APIRouter, Depends
+from typing import Dict, Any
+from fastapi import APIRouter, Depends, Request, Response
 from starlette import status
 from starlette.datastructures import URL
+from starlette.status import HTTP_204_NO_CONTENT
 
 from ...models.domains.dynamic_services import RetrieveDataIn, RetrieveDataOutEnveloped
 from ...utils.logging_utils import log_decorator
@@ -13,6 +15,15 @@
     get_services_client,
 )
 
+from simcore_service_director_v2.modules.dynamic_sidecar.entrypoint import (
+    start_dynamic_sidecar_stack_for_service,
+    get_dynamic_sidecar_stack_status,
+    stop_dynamic_sidecar_stack_for_service,
+)
+from simcore_service_director_v2.models.domains.dynamic_sidecar import (
+    StartDynamicSidecarModel,
+)
+
 router = APIRouter()
 logger = logging.getLogger(__file__)
 
@@ -40,3 +51,45 @@ async def service_retrieve_data_on_ports(
     )
     # validate and return
     return RetrieveDataOutEnveloped.parse_obj(resp.json())
+
+
+@router.post(
+    "/{node_uuid}:start",
+    summary="start the dynamic-sidecar for this service",
+)
+async def start_dynamic_sidecar(
+    node_uuid: str,
+    start_dynamic_sidecar_model: StartDynamicSidecarModel,
+    request: Request,
+) -> Dict[str, str]:
+    return await start_dynamic_sidecar_stack_for_service(
+        app=request.app,
+        user_id=start_dynamic_sidecar_model.user_id,
+        project_id=start_dynamic_sidecar_model.project_id,
+        service_key=start_dynamic_sidecar_model.service_key,
+        service_tag=start_dynamic_sidecar_model.service_tag,
+        paths_mapping=start_dynamic_sidecar_model.paths_mapping,
+        compose_spec=start_dynamic_sidecar_model.compose_spec,
+        target_container=start_dynamic_sidecar_model.target_container,
+        node_uuid=node_uuid,
+        settings=start_dynamic_sidecar_model.settings,
+        request_scheme=start_dynamic_sidecar_model.request_scheme,
+        request_dns=start_dynamic_sidecar_model.request_dns,
+    )
+
+
+@router.post(
+    "/{node_uuid}:status", summary="assembles the status for the dynamic-sidecar"
+)
+async def dynamic_sidecar_status(node_uuid: str, request: Request) -> Dict[str, Any]:
+    return await get_dynamic_sidecar_stack_status(app=request.app, node_uuid=node_uuid)
+
+
+@router.post(
+    "/{node_uuid}:stop",
+    responses={HTTP_204_NO_CONTENT: {"model": None}},
+    summary="stops previously spawned dynamic-sidecar",
+)
+async def stop_dynamic_sidecar(node_uuid: str, request: Request) -> Dict[str, str]:
+    await stop_dynamic_sidecar_stack_for_service(app=request.app, node_uuid=node_uuid)
+    return Response(status_code=HTTP_204_NO_CONTENT)

services/director-v2/src/simcore_service_director_v2/core/application.py

@@ -20,6 +20,7 @@
     docker_registry,
     dynamic_services,
     remote_debug,
+    dynamic_sidecar
 )
 from ..utils.logging_utils import config_all_loggers
 from .events import on_shutdown, on_startup
@@ -66,6 +67,8 @@ def init_app(settings: Optional[AppSettings] = None) -> FastAPI:
     if settings.registry.enabled:
         docker_registry.setup(app, settings.registry)
 
+    dynamic_sidecar.setup_dynamic_sidecar(app)
+
     # setup app --
     app.add_event_handler("startup", on_startup)
     app.add_event_handler("shutdown", on_shutdown)

services/director-v2/src/simcore_service_director_v2/core/settings.py

@@ -100,8 +100,6 @@ def check_auth_credentials(cls, values):
             user, pw = values.get("user"), values.get("pw")
             if user is None or pw is None:
                 raise ValueError("Cannot authenticate without credentials user, pw")
-            if not values["ssl"]:
-                raise ValueError("Authentication REQUIRES a secured channel")
         return values
 
     @property

services/director-v2/src/simcore_service_director_v2/models/domains/dynamic_sidecar.py

@@ -0,0 +1,78 @@
+from typing import Dict, Any, Optional, List
+
+from pydantic import BaseModel, Field, validator
+
+ComposeSpecModel = Optional[Dict[str, Any]]
+
+
+class PathsMappingModel(BaseModel):
+    inputs_path: str = Field(
+        ..., description="path where the service expects all the inputs folder"
+    )
+    outputs_path: str = Field(
+        ..., description="path where the service expects all the outputs folder"
+    )
+    other_paths: List[str] = Field(
+        [],
+        description="optional list of path which contents need to be saved and restored",
+    )
+
+    @validator("other_paths", always=True)
+    @classmethod
+    def other_paths_always_a_list(cls, v):
+        return [] if v is None else v
+
+
+class StartDynamicSidecarModel(BaseModel):
+    user_id: str
+    project_id: str
+    service_key: str
+    service_tag: str
+
+    # these come from the webserver via the director
+    request_scheme: str = Field(
+        ..., description="Used for the proxy configuration either http or https"
+    )
+    request_dns: str = Field(..., description="Used for the proxy configuration")
+
+    settings: List[Dict[str, Any]] = Field(
+        ...,
+        description="settings for the services define by the service maintainer in the labels",
+    )
+    paths_mapping: PathsMappingModel = Field(
+        ...,
+        description=(
+            "the service explicitly requests where to mount all paths "
+            "which will be handeled by the dynamic-sidecar"
+        ),
+    )
+    compose_spec: ComposeSpecModel = Field(
+        ...,
+        description=(
+            "if the user provides a compose_spec, it will be used instead "
+            "of compsing one from the service_key and service_tag"
+        ),
+    )
+    target_container: Optional[str] = Field(
+        ...,
+        description="when the user defines a compose spec, it should pick a container inside the spec to receive traffic on a defined port",
+    )
+
+    @validator("target_container")
+    @classmethod
+    def target_container_must_exist_if_compose_spec_present(
+        cls, v, values, **kwargs
+    ):  # pylint: disable=unused-argument
+        if values.get("compose_spec", None) is not None and v is None:
+            raise ValueError(
+                "simcore.service.target_container is required when compose_spec is defined. "
+                f"The following compose spec was defined: {values['compose_spec']}"
+            )
+        return v
+
+    @validator("request_scheme")
+    @classmethod
+    def validate_protocol(cls, v, values, **kwargs):  # pylint: disable=unused-argument
+        if v not in {"http", "https"}:
+            raise ValueError(f"provided request_scheme={v} must be 'http' or 'https'")
+        return v

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/__init__.py

@@ -0,0 +1,5 @@
+from .entrypoint import (
+    start_dynamic_sidecar_stack_for_service,
+    stop_dynamic_sidecar_stack_for_service,
+)
+from .submodule_setup import setup_dynamic_sidecar