Is788/allow different stack name (#789)

* director/sidecar independant of stack name using SWARM_STACK_NAME
* allow choosing port where simcore platform is available

Author: sanderegg

Makefile

@@ -47,7 +47,7 @@ export VCS_REF:=$(shell git rev-parse --short HEAD)
 export VCS_REF_CLIENT:=$(shell git log --pretty=tformat:"%h" -n1 services/web/client)
 export VCS_STATUS_CLIENT:=$(if $(shell git status -s),'modified/untracked','clean')
 export BUILD_DATE:=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
-
+export SWARM_STACK_NAME ?= services
 # using ?= will only set if absent
 export DOCKER_IMAGE_TAG ?= latest
 $(info DOCKER_IMAGE_TAG set to ${DOCKER_IMAGE_TAG})
@@ -131,13 +131,12 @@ up-devel: up-swarm-devel
 up-swarm: .env docker-swarm-check
 	${DOCKER} swarm init
 	${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.tools.yml config > $(TEMPCOMPOSE).tmp-compose.yml ;
-	${DOCKER} stack deploy -c $(TEMPCOMPOSE).tmp-compose.yml services
+	${DOCKER} stack deploy -c $(TEMPCOMPOSE).tmp-compose.yml ${SWARM_STACK_NAME}
 
 up-swarm-devel: .env docker-swarm-check $(CLIENT_WEB_OUTPUT)
 	${DOCKER} swarm init
 	${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.devel.yml -f services/docker-compose.tools.yml config > $(TEMPCOMPOSE).tmp-compose.yml
-	${DOCKER} stack deploy -c $(TEMPCOMPOSE).tmp-compose.yml services
-
+	${DOCKER} stack deploy -c $(TEMPCOMPOSE).tmp-compose.yml ${SWARM_STACK_NAME}
 
 .PHONY: up-webclient-devel
 # target: up-webclient-devel: – init swarm and deploys all core and tool services up in development mode. Then it stops the webclient service and starts it again with the watcher attached.

services/director/Dockerfile

@@ -27,6 +27,7 @@ ENV REGISTRY_USER = ''
 ENV REGISTRY_PW = ''
 ENV REGISTRY_URL = ''
 ENV PUBLISHED_HOST_NAME=''
+ENV SWARM_STACK_NAME = ''
 ENV EXTRA_HOSTS_SUFFIX = 'undefined'
 
 

services/director/src/simcore_service_director/config.py

@@ -32,3 +32,4 @@
 PUBLISHED_HOST_NAME = os.environ.get("PUBLISHED_HOST_NAME", "")
 NODE_SCHEMA_LOCATION = os.environ.get("NODE_SCHEMA_LOCATION",
     "{root}/{version}/schemas/node-meta-v0.0.1.json".format(root=API_ROOT, version=API_VERSION))
+SWARM_STACK_NAME = os.environ.get("SWARM_STACK_NAME")

services/director/src/simcore_service_director/producer.py

@@ -124,21 +124,27 @@ def __get_service_entrypoint(service_boot_parameters_labels: Dict) -> str:
             return param['value']
     return ''
 
+def _get_swarm_network(client: DockerClient) -> docker.models.networks.Network:
+    network_name = "default"
+    if config.SWARM_STACK_NAME:
+        network_name = config.SWARM_STACK_NAME
+    # try to find the network name (usually named STACKNAME_default)
+    networks = [x for x in client.networks.list(filters={"scope":"swarm"}) if network_name in x.name]
+    if not networks or len(networks)>1:
+        raise exceptions.DirectorException(msg="Swarm network name is not configured, found following networks: {}".format(networks))
+    return networks[0]
+
 
 def __add_to_swarm_network_if_ports_published(
         client: DockerClient,
         docker_service_runtime_parameters: Dict):
-    # TODO: SAN this is a brain killer... change services to something better...
     if "endpoint_spec" in docker_service_runtime_parameters:
-        network_id = "services_default"
-        log.debug(
-            "Adding swarm network with id: %s to docker runtime parameters", network_id)
-        list_of_networks = client.networks.list(
-            names=[network_id], filters={"scope": "swarm"})
-        for network in list_of_networks:
-            __add_network_to_service_runtime_params(
-                docker_service_runtime_parameters, network)
-        log.debug("Added swarm network %s to docker runtime parameters", network_id)
+        try:
+            swarm_network = _get_swarm_network(client)            
+            log.debug("Adding swarm network with id: %s to docker runtime parameters", swarm_network.name)
+            __add_network_to_service_runtime_params(docker_service_runtime_parameters, swarm_network)
+        except exceptions.DirectorException:
+            log.exception("Could not find a swarm network, not in a swarm?")
 
 
 def __add_uuid_label_to_service_runtime_params(

services/docker-compose.yml

@@ -57,8 +57,9 @@ services:
       - S3_SECRET_KEY=${S3_SECRET_KEY}
       - S3_BUCKET_NAME=${S3_BUCKET_NAME}
       - STORAGE_ENDPOINT=${STORAGE_ENDPOINT}
-      - EXTRA_HOSTS_SUFFIX=osparc.io
+      - EXTRA_HOSTS_SUFFIX=${EXTRA_HOSTS_SUFFIX}
       - PUBLISHED_HOST_NAME=${PUBLISHED_HOST_NAME}
+      - SWARM_STACK_NAME=${SWARM_STACK_NAME}
     volumes:
       - '/var/run/docker.sock:/var/run/docker.sock'
     deploy:
@@ -105,7 +106,7 @@ services:
         org.label-schema.vcs-url: "https://github.com/ITISFoundation/osparc-simcore"
         org.label-schema.vcs-ref: "${VCS_REF}"
     ports:
-      - '9081:8080'
+      - ${SIMCORE_PORT:-9081}:8080
     environment:
       - APIHUB_HOST=apihub
       - APIHUB_PORT=8043
@@ -185,6 +186,7 @@ services:
       - REGISTRY_URL=${REGISTRY_URL}
       - REGISTRY_USER=${REGISTRY_USER}
       - REGISTRY_PW=${REGISTRY_PW}
+      - SWARM_STACK_NAME=${SWARM_STACK_NAME}
     depends_on:
       - rabbit
       - postgres

services/sidecar/Dockerfile

@@ -21,6 +21,9 @@ ENV PATH "/home/scu/.local/bin:$PATH"
 ENV SC_PIP pip3 --no-cache-dir
 ENV SC_BUILD_TARGET base
 
+# used for finding the volumes
+ENV SWARM_STACK_NAME = ''
+
 EXPOSE 8080
 
 VOLUME /home/scu/input

services/sidecar/src/sidecar/core.py

@@ -58,6 +58,9 @@ def __init__(self):
         # current task
         self._task = None
 
+        # stack name
+        self._stack_name = None
+
         # executor options
         self._executor = ExecutorSettings()
 
@@ -278,6 +281,9 @@ def initialize(self, task, user_id):
         # volume paths for car container (w/o prefix)
         self._docker.env = ["{}_FOLDER=/{}".format(name.upper(), tail) for name, tail in tails.items()]
 
+        # stack name, should throw if not set
+        self._stack_name = os.environ["SWARM_STACK_NAME"]
+
         # config nodeports
         node_ports.node_config.USER_ID = user_id
         node_ports.node_config.NODE_UUID = task.node_id
@@ -304,9 +310,9 @@ def process(self):
             docker_image = self._docker.image_name + ":" + self._docker.image_tag
             self._docker.client.containers.run(docker_image, "run",
                  detach=False, remove=True,
-                 volumes = {'services_input'  : {'bind' : '/input'},
-                            'services_output' : {'bind' : '/output'},
-                            'services_log'    : {'bind'  : '/log'}},
+                 volumes = {'{}_input'.format(self._stack_name)  : {'bind' : '/input'},
+                            '{}_output'.format(self._stack_name) : {'bind' : '/output'},
+                            '{}_log'.format(self._stack_name)    : {'bind'  : '/log'}},
                  environment=self._docker.env)
         except docker.errors.ContainerError as _e:
             log.exception("Run container returned non zero exit code %s", str(_e))