WIP ITISFoundation#198: non-root user

 - all volumes bound to /home/scu
 - All modules pip installed (in dev w/ edit mode).
 - All files produce in dev mode on bound volumes are not deletable
 - Production stage is further optimized by taking only venv from base
 - Fixes sidecar access to input/output/log volumes

Author: Pedro Crespo

.env-devel

@@ -1,3 +1,7 @@
+# NOTE: write here host gid and docker gid.
+HOST_GID=1000
+DOCKER_GID=1001
+#--------
 POSTGRES_ENDPOINT=postgres:5432
 POSTGRES_USER=simcore
 POSTGRES_PASSWORD=simcore

services/docker-compose.devel.yml

@@ -29,7 +29,9 @@ services:
   sidecar:
     image: services_sidecar:dev
     build:
+      args:
+        - HOST_GID_ARG=${HOST_GID:?Undefined host gid}
       target: development
     volumes:
-      - ./sidecar:/work/services/sidecar
-      - ../packages:/work/packages
+      - ./sidecar:/home/scu/services/sidecar
+      - ../packages:/home/scu/packages

services/docker-compose.swarm.yml.template

@@ -87,9 +87,9 @@ services:
       - S3_SECRET_KEY=<enter secret key>
       - S3_BUCKET_NAME=simcore
     volumes:
-      - input:/input
-      - output:/output
-      - log:/log
+      - input:/home/scu/input
+      - output:/home/scu/output
+      - log:/home/scu/log
       - /var/run/docker.sock:/var/run/docker.sock
     ports:
       - "8000:8000"

services/docker-compose.yml

@@ -95,16 +95,17 @@ services:
       # the packages directory into any docker image
       context: ../
       dockerfile: services/sidecar/Dockerfile
+      args:
+        - DOCKER_GID_ARG=${DOCKER_GID:?Undefined docker gid in host}
       target: production
     volumes:
-      - input:/input
-      - output:/output
-      - log:/log
+      - input:/home/scu/input
+      - output:/home/scu/output
+      - log:/home/scu/log
       - /var/run/docker.sock:/var/run/docker.sock
     ports:
       - "8000:8000"
     environment:
-      - PYTHONPATH=/work/packages/simcore-sdk/src:/work/packages/s3wrapper/src
       - RABBITMQ_USER=${RABBITMQ_USER}
       - RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD}
       - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT}

services/sidecar/Dockerfile

@@ -1,58 +1,91 @@
-FROM python:3.6-alpine as common
+FROM python:3.6-alpine as base
 
-LABEL maintainer="Manuel Guidon <[email protected]"
+LABEL maintainer="Manuel Guidon guidon"
+
+ARG DOCKER_GID_ARG=1001
+
+# create user 'scu' and adds it to host's docker group
+RUN adduser -D -u 8004 scu &&\
+    addgroup -g $DOCKER_GID_ARG docker &&\
+    addgroup scu docker
+
+ENV HOME /home/scu
+ENV PIP  /home/scu/.venv/bin/pip3
+
+EXPOSE 8000
+VOLUME /home/scu/input
+VOLUME /home/scu/output
+VOLUME /home/scu/log
+#VOLUME /var/run/docker.sock TODO: PC->MaG is this using docker??
+
+# -------------------------- Build stage -------------------
+# Keeps same folder structure as in repo so we can reuse relative paths
+#
+# + /home/scu/              $HOME
+#    + services/sidecar
+#        ...
+#    + packages
+#        ...
+FROM base as build
 
 RUN apk add --no-cache \
       postgresql-dev \
       gcc \
       libc-dev
 
-RUN pip install --upgrade \
+RUN python3 -m venv $HOME/.venv &&\
+    $PIP install --no-cache-dir --upgrade \
       pip \
       wheel \
       setuptools
 
-WORKDIR /work
+WORKDIR /home/scu
+
 # Buil context set at repo's root
-COPY services/sidecar/requirements requirements
+COPY --chown=scu:scu services/sidecar/requirements requirements
 
-RUN pip install -r requirements/base.txt &&\
+RUN $PIP install --no-cache-dir -r requirements/base.txt &&\
     rm -rf requirements
 
-# Keeps same folder structure as in repo so we can reuse relative paths
-RUN mkdir -p /work/packages &&\
-    mkdir -p /work/services/sidecar
+# --------------------------Development stage -------------------
+FROM build as development
 
-EXPOSE 8000
+ARG HOST_GID_ARG=1000
 
+# in dev-mode we give access to `scu` to host's mapped volumes
+RUN addgroup -g $HOST_GID_ARG hgrp &&\
+    addgroup scu hgrp && \
+    chown -R scu:scu $HOME/.venv
 
-# --------------------------Development stage -------------------
-FROM common as development
-
-VOLUME /work/packages
-VOLUME /work/services/sidecar
+VOLUME /home/scu/packages
+VOLUME /home/scu/services/sidecar
 
+USER scu
 ENV DEBUG 1
-WORKDIR /work/services/sidecar
+WORKDIR /home/scu/services/sidecar
 CMD ./boot.sh
-# FIXME: executing this as root will create folders (e.g. eggs) in the mapped
 
 
-# --------------------------Production stage -------------------
-FROM common as production
+# --------------------------Production mult-stage -------------------
+FROM build as build-production
 
 # Buil context set at repo's root
-COPY packages /work/packages
-COPY services/sidecar /work/services/sidecar
+COPY --chown=scu:scu packages $HOME/packages
+COPY --chown=scu:scu services/sidecar $HOME/services/sidecar
+
+WORKDIR /home/scu/services/sidecar
+RUN $PIP --no-cache-dir install -r requirements/prod.txt ;\
+    $PIP list
+
+#-------------------
+FROM base as production
 
-WORKDIR /work/services/sidecar
+COPY --from=build-production --chown=scu:scu $HOME/services/sidecar/boot.sh $HOME
+COPY --from=build-production --chown=scu:scu $HOME/.venv $HOME/.venv
 
-RUN pip install -r requirements/prod.txt ;\
-    pip list &&\
-    mv boot.sh /work &&\
-    rm -rf /work/packages &&\
-    rm -rf /work/services/sidecar
+RUN . $HOME/.venv/bin/activate; pip list
 
+WORKDIR /home/scu
+USER scu
 ENV DEBUG 0
-WORKDIR /work
 ENTRYPOINT ./boot.sh

services/sidecar/boot.sh

@@ -1,11 +1,12 @@
 #!/bin/sh
+source $HOME/.venv/bin/activate
 
 if [[ ${DEBUG} == "1" ]]
 then
   echo "Booting in development mode ..."
   echo "Installing director service ..."
 
-  pip install -r requirements/dev.txt
+  pip3 install --no-cache-dir -r requirements/dev.txt
   celery worker --app sidecar --concurrency 2 --loglevel=debug
 else
   echo "Booting in production mode ..."

services/sidecar/src/sidecar/_deprecated.py

@@ -1,3 +1,5 @@
+# TODO: PC->MaG, please check if something missing and delete
+
 import json
 import logging
 import os

services/sidecar/src/sidecar/celery.py

@@ -19,3 +19,9 @@
 celery_obj= Celery(rabbit_config.name,
     broker=rabbit_config.broker,
     backend=rabbit_config.backend)
+
+
+__all__ = [
+    "rabbit_config",
+    "celery_obj"
+]

services/sidecar/src/sidecar/core.py

@@ -20,7 +20,7 @@
                     find_entry_point, is_node_ready)
 
 _LOGGER = get_task_logger(__name__)
-_LOGGER.setLevel(logging.DEBUG)
+_LOGGER.setLevel(logging.DEBUG) # FIXME: set level via config
 
 
 class Sidecar:
@@ -271,11 +271,12 @@ def _process_task_log(self):
     def initialize(self, task):
         self._task = task
 
+        HOMEDIR = os.environ["HOME"]
         self._docker.image_name = self._docker.registry_name + "/" + task.image['name']
         self._docker.image_tag = task.image['tag']
-        self._executor.in_dir = os.path.join("/", "input", task.job_id)
-        self._executor.out_dir = os.path.join("/", "output", task.job_id)
-        self._executor.log_dir = os.path.join("/", "log", task.job_id)
+        self._executor.in_dir = os.path.join(HOMEDIR, "input", task.job_id)
+        self._executor.out_dir = os.path.join(HOMEDIR, "output", task.job_id)
+        self._executor.log_dir = os.path.join(HOMEDIR, "log", task.job_id)
 
         self._docker.env = ["INPUT_FOLDER=" + self._executor.in_dir,
                             "OUTPUT_FOLDER=" + self._executor.out_dir,
@@ -445,7 +446,9 @@ def inspect(self, celery_task, pipeline_id, node_id):
         return next_task_nodes
 
 
-
-
 # TODO: if a singleton, then use
 SIDECAR = Sidecar()
+
+__all__ = [
+    "SIDECAR"
+]