Merge pull request #13 from sanderegg/oetiker-build-update-2

fix simcore-sdk build

Author: oetiker

.env-devel

@@ -1,6 +1,3 @@
-HOST_GID=1000
-DOCKER_GID=1001
-RUN_DOCKER_ENGINE_ROOT=0
 DOCKER_IMAGE_PREFIX=services_
 DOCKER_IMAGE_TAG=latest
 
@@ -32,10 +29,10 @@ S3_SECRET_KEY=12345678
 S3_BUCKET_NAME=simcore
 S3_SECURE=0
 
-SMTP_HOST=mail
+SMTP_HOST=mail.speag.com
 SMTP_PORT=25
 
-VENV2=123
+VENV2=.venv27/
 
 BF_API_KEY=none
 BF_API_SECRET=none

.travis.yml

@@ -2,7 +2,6 @@ dist: xenial
 env:
   global:
     - DOCKER_COMPOSE_VERSION=1.23.2
-    - DOCKER_GID=1042
 services:
   - docker
 addons:

Makefile

@@ -1,50 +1,40 @@
 # author: Sylvain Anderegg
 
-# TODO: add flavours by combinging docker-compose files. Namely development, test and production.
 VERSION := $(shell uname -a)
 
-# SAN this is a hack so that docker-compose works in the linux virtual environment under Windows
+# SAN: this is a hack so that docker-compose works in the linux virtual environment under Windows
 WINDOWS_MODE=OFF
 ifneq (,$(findstring Microsoft,$(VERSION)))
 $(info    detected WSL)
 export DOCKER_COMPOSE=docker-compose
 export DOCKER=docker
-export RUN_DOCKER_ENGINE_ROOT=1
-# Windows does not have these things defined... but they are needed to execute a local swarm
-export DOCKER_GID=1042
-export HOST_GID=1000
+# SAN: Windows does not have these things defined... but they are needed to execute a local swarm
 WINDOWS_MODE=ON
 else ifeq ($(OS), Windows_NT)
 $(info    detected Powershell/CMD)
 export DOCKER_COMPOSE=docker-compose.exe
 export DOCKER=docker.exe
-export RUN_DOCKER_ENGINE_ROOT=1
-export DOCKER_GID=1042
-export HOST_GID=1000
 WINDOWS_MODE=ON
 else ifneq (,$(findstring Darwin,$(VERSION)))
 $(info    detected OSX)
 export DOCKER_COMPOSE=docker-compose
 export DOCKER=docker
-export RUN_DOCKER_ENGINE_ROOT=1
-export DOCKER_GID=1042
-export HOST_GID=1000
 else
 $(info    detected native linux)
-# TODO: DO NOT TOUCH THIS CONFIG --- (ask mguidon)
 export DOCKER_COMPOSE=docker-compose
 export DOCKER=docker
-export RUN_DOCKER_ENGINE_ROOT=0
-export DOCKER_GID=1042
-export HOST_GID=1000
-# TODO: DO NOT TOUCH THIS CONFIG --- (ask mguidon)
-# FIXME: DOCKER_GID and HOST_GID should be removed when issue #90 is resolved
-# TODO: Add a meaningfull call to retrieve the local docker group ID and the user ID in linux.
 endif
 
+export DOCKER_REGISTRY=masu.speag.com
+export SERVICES_VERSION=2.8.0
 
-PY_FILES = $(strip $(shell find services packages -iname '*.py' -not -path "*egg*" -not -path "*contrib*" -not -path "*-sdk/python*" -not -path "*generated_code*" -not -path "*datcore.py" -not -path "*web/server*"))
-
+PY_FILES := $(strip $(shell find services packages -iname '*.py' \
+											-not -path "*egg*" \
+											-not -path "*contrib*" \
+											-not -path "*-sdk/python*" \
+											-not -path "*generated_code*" \
+											-not -path "*datcore.py" \
+											-not -path "*web/server*"))
 TEMPCOMPOSE := $(shell mktemp)
 
 SERVICES_LIST := apihub director sidecar storage webserver
@@ -55,23 +45,17 @@ VCS_URL:=$(shell git config --get remote.origin.url)
 VCS_REF:=$(shell git rev-parse --short HEAD)
 VCS_REF_CLIENT:=$(shell git log --pretty=tformat:"%h" -n1 services/web/client)
 VCS_STATUS_CLIENT:=$(if $(shell git status -s),'modified/untracked','clean')
-
-BUILD_DATE:=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
-
-PLATFORM_VERSION=3.38
-DOCKER_REGISTRY=masu.speag.com
-#DOCKER_REGISTRY=registry.osparc.io
-
 export VCS_URL
 export VCS_REF
 export VCS_REF_CLIENT
 export VCS_STATUS_CLIENT
+
+BUILD_DATE:=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 export BUILD_DATE
-export SERVICES_VERSION=2.8.0
-export DOCKER_REGISTRY=masu.speag.com
 
 
-## Tools
+## Tools ------------------------------------------------------------------------------------------------------
+#
 tools =
 
 ifeq ($(shell uname -s),Darwin)
@@ -87,7 +71,7 @@ endif
 
 ## ------------------------------------------------------------------------------------------------------
 .PHONY: all
-all: help
+all: help info
 ifdef tools
 	$(error "Can't find tools:${tools}")
 endif
@@ -115,7 +99,8 @@ rebuild-devel: .env .tmp-webclient-build
 
 .tmp-webclient-build:
 	# TODO: fixes having services_webclient:build present for services_webserver:production when
-	# targeting services_webserver:development
+	# targeting services_webserver:development and ensures source-output folder at host
+	mkdir -p services\web\client\source-output
 	${DOCKER_COMPOSE} -f services/docker-compose.yml build webclient
 
 
@@ -256,11 +241,16 @@ create-staging-stack-file:
 .PHONY: info
 # target: info – Displays some parameters of makefile environments
 info:
-	@echo '+ vcs ref '
-	@echo '  - origin    : ${VCS_URL}'
-	@echo '  - all       : ${VCS_REF}'
-	@echo '  - web/client (${VCS_STATUS_CLIENT}): ${VCS_REF_CLIENT}'
-	@echo '+ date        : ${BUILD_DATE}'
+	@echo '+ VCS_* '
+	@echo '  - ULR                : ${VCS_URL}'
+	@echo '  - REF                : ${VCS_REF}'
+	@echo '  - (STATUS)REF_CLIENT : (${VCS_STATUS_CLIENT}) ${VCS_REF_CLIENT}'
+	@echo '+ BUILD_DATE           : ${BUILD_DATE}'
+	@echo '+ VERSION              : ${VERSION}'
+	@echo '+ WINDOWS_MODE         : ${WINDOWS_MODE}'
+	@echo '+ DOCKER_REGISTRY      : ${DOCKER_REGISTRY}'
+	@echo '+ SERVICES_VERSION     : ${SERVICES_VERSION}'
+	@echo '+ PY_FILES             : $(shell echo $(PY_FILES) | wc -w) files'
 
 
 .PHONY: pylint

ops/travis/unit-testing/simcore-sdk

@@ -28,6 +28,7 @@ before_script() {
     if [ -n "$(bash ops/travis/helpers/test_for_changes $FOLDER_CHECKS)" ];
     then
         pip freeze
+        make build
         docker images
     fi
 }

packages/simcore-sdk/tests/node_ports/docker-compose.yml

@@ -18,7 +18,6 @@ services:
       - S3_SECRET_KEY=${S3_SECRET_KEY}
       - S3_BUCKET_NAME=${S3_BUCKET_NAME}
       - S3_SECURE=0
-      - RUN_DOCKER_ENGINE_ROOT=1
       - BF_API_SECRET="none"
       - BF_API_KEY="none"
     restart: always

services/director/Dockerfile

@@ -1,106 +1,141 @@
 FROM python:3.6-alpine as base
+#
+#  USAGE:
+#     cd sercices/director
+#     docker build -f Dockerfile -t director:prod --target production ../../
+#     docker run director:prod
+#
+#  REQUIRED: context expected at ``osparc-simcore/`` folder because we need access to osparc-simcore/packages
 
 LABEL maintainer=sanderegg
 
-ARG DOCKER_GID_ARG=1001
+# simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu)
+RUN adduser -D -u 8004 -s /bin/sh -h /home/scu scu
 
 RUN apk add --no-cache \
-    su-exec
+      su-exec
 
-# create user `scu` and `docker` group (with same id as in host)
-RUN adduser -D -u 8004 scu &&\
-    addgroup -g $DOCKER_GID_ARG docker
+ENV PATH "/home/scu/.local/bin:$PATH"
 
-ENV HOME /home/scu
-# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197
-# - Added explicit --cache-dir that is removed at the end of the build
-ENV PIP  /home/scu/.venv/bin/pip3 --cache-dir=/tmp/.pipcache
-# -----
+# All SC_ variables are customized
+ENV SC_PIP pip3 --no-cache-dir
+ENV SC_BUILD_TARGET base
 
-EXPOSE 8001
+# TODO: update environs
 ENV REGISTRY_AUTH = ''
 ENV REGISTRY_USER = ''
 ENV REGISTRY_PW = ''
 ENV REGISTRY_URL = ''
 ENV PUBLISHED_HOST_NAME=''
 ENV EXTRA_HOSTS_SUFFIX = 'undefined'
 
-# ----------------------------------------------------------------
-COPY --chown=scu:scu scripts/docker/healthcheck_curl_host.py /healthcheck/healthcheck_curl_host.py
-# will start to run after interval every interval. fails after timeout. fail do not count if during start-period. will do # retries
-HEALTHCHECK --interval=10s --timeout=30s --start-period=20s --retries=3 CMD [ "python3", "/healthcheck/healthcheck_curl_host.py", "http://localhost:8001/v0/" ]
 
-WORKDIR /home/scu
+EXPOSE 8080
 
 # -------------------------- Build stage -------------------
+# Installs build/package management tools and third party dependencies
 #
-# - Preserves relative folder structure
+# + /build             WORKDIR
 #
-# + /home/scu/              $HOME
-#    + services/director
-#        ...
-#    + packages
-#        ...
+
 FROM base as build
 
+ENV SC_BUILD_TARGET build
+
 RUN apk add --no-cache \
     gcc \
     libc-dev \
     git
 
-# create virtual environment
-RUN python3 -m venv $HOME/.venv &&\
-    $PIP install --upgrade \
+RUN $SC_PIP install --upgrade \
       pip \
       wheel \
       setuptools
 
-# this is done to accelerate installation of director later on
-COPY --chown=scu:scu services/director/requirements/base.txt requirements-base.txt
-RUN $PIP install -r requirements-base.txt &&\
-    rm requirements-base.txt
+WORKDIR /build
 
-COPY --chown=scu:scu services/director/docker docker
+# install base 3rd party dependencies
+COPY  services/director/requirements/*.txt \
+      services/director/requirements/
 
-# --------------------------Development stage -------------------
-FROM build as development
+RUN $SC_PIP install \
+      -r services/director/requirements/base.txt
 
-ARG HOST_GID_ARG=1000
+#--------------------------Cache stage -------------------
+# CI in master buils & pushes this target to speed-up image build
+#
+#  + /build
+#    + services/director [scu:scu] WORKDIR
+#
+FROM build as cache
 
-# in dev-mode we give access to `scu` to host's mapped volumes
-# FIXME: files created by scu cannot be deleted by host! we need to do the same group in host?
-RUN addgroup -g $HOST_GID_ARG hgrp &&\
-    addgroup scu hgrp && \
-    chown -R scu:scu $HOME/.venv &&\
-    rm -rf /tmp/.pipcache
+ENV SC_BUILD_TARGET cache
 
-VOLUME /home/scu/services/director
-VOLUME /home/scu/packages/service-library
 
-ENV DEBUG 1
-ENV RUN_DOCKER_ENGINE_ROOT=0
-ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh" ]
-CMD ["/bin/sh", "docker/boot.sh"]
+COPY --chown=scu:scu packages/service-library /build/packages/service-library
+COPY --chown=scu:scu services/director /build/services/director
 
-# --------------------------Cache stage -------------------
-FROM build as cache
-COPY --chown=scu:scu services/director $HOME/services/director
-COPY --chown=scu:scu packages/service-library $HOME/packages/service-library
-WORKDIR $HOME/services/director
-RUN $PIP install -r requirements/prod.txt
-WORKDIR $HOME/packages/service-library
-# FIXME: is this really requirements/dev or rather requirements/prod.txt?
-RUN $PIP install -r requirements/dev.txt
+WORKDIR /build/services/director
+
+
+RUN $SC_PIP install -r requirements/prod.txt &&\
+    $SC_PIP list -v
 
 
 # --------------------------Production stage -------------------
+# Final cleanup up to reduce image size and startup setup
+# Runs as scu (non-root user)
+#
+#  + /home/scu     $HOME = WORKDIR
+#    + services/director [scu:scu]
+#
 FROM cache as production
-WORKDIR $HOME
-ENV DEBUG 0
-ENV RUN_DOCKER_ENGINE_ROOT=0
-RUN rm -rf $HOME/services &&\
-    rm -rf $HOME/packages &&\
-    rm -rf /tmp/.pipcache
-
-ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh" ]
-CMD ["/bin/sh", "docker/boot.sh"]
+
+
+ENV SC_BUILD_TARGET production
+ENV SC_BOOT_MODE production
+
+WORKDIR /home/scu
+
+RUN mkdir -p services/director &&\
+    chown scu:scu services/director &&\
+    mv /build/services/director/docker services/director/docker &&\
+    rm -rf /build
+
+RUN apk del --no-cache\
+      gcc \
+      git
+
+HEALTHCHECK --interval=30s \
+            --timeout=120s \
+            --start-period=30s \
+            --retries=3 \
+            CMD ["python3", "/home/scu/services/director/docker/healthcheck.py", "http://localhost:8080/v0/"]
+
+ENTRYPOINT [ "/bin/sh", "services/director/docker/entrypoint.sh" ]
+CMD ["/bin/sh", "services/director/docker/boot.sh"]
+
+
+# --------------------------Development stage -------------------
+# Source code accessible in host but runs in container
+# Runs as myu with same gid/uid as host
+# Placed at the end to speed-up the build if images targeting production
+#
+#  + /devel         WORKDIR
+#    + services  (mounted volume)
+#
+FROM build as development
+
+ENV SC_BUILD_TARGET development
+ENV SC_BOOT_MODE development
+
+
+# WORKDIR /build
+# NOTE: can copy from /build if necessary
+
+WORKDIR /devel
+VOLUME  /devel/packages
+VOLUME  /devel/services/director/
+
+ENTRYPOINT [ "/bin/sh", "services/director/docker/entrypoint.sh" ]
+CMD ["/bin/sh", "services/director/docker/boot.sh"]

services/director/README.md

@@ -66,7 +66,7 @@ Using the main Makefile of the oSparc platform allows for testing the director:
   make up-swarm-devel
 ```
 
-Then open [director-swagger-ui](http://localhost:8001/apidoc/) to see the director API and try out the different routes.
+Then open [director-swagger-ui](http://localhost:8080/apidoc/) to see the director API and try out the different routes.
 
 ## code generation from REST API "server side"