Is539/master build fails (#540)

* minor cleanup

* Fixed storage Dockerfile image build:
 - Added workaround for pypa/pip#6197
 - Froze python2 requirements (virtualenv py2.7 would fail to install numpy)

* Fixes director Dockerfile using same workaround in pip as with storage

* Applies pip workaround to sidecar Dockerfile

* Applies pip workaround to webserver Dockerfile

* Applied pip workaround to backend dynamic services

* pip compiled py27.in -> py27.txt (pip-tools)
 Changes to be committed:
	new file:   services/storage/requirements/py27.in
	modified:   services/storage/requirements/py27.txt

Author: pcrespov

.travis.yml

@@ -127,7 +127,7 @@ jobs:
 
     # test python, linting ----------------------------------------------------------------------
     - stage: unit testing
-      name: Python linting
+      name: python linting
       language: python
       python:
         - "3.6"

ops/travis/system-testing/tests/test_swarm_runs.py

@@ -25,7 +25,7 @@ def osparc_simcore_root_dir(here) -> Path:
     root_dir = here.parent.parent.parent.parent.resolve()
     assert root_dir.exists(), "Is this service within osparc-simcore repo?"
     assert any(root_dir.glob("services/web/server")), "%s not look like rootdir" % root_dir
-    return root_dir    
+    return root_dir
 
 @pytest.fixture("session")
 def services_docker_compose(osparc_simcore_root_dir) -> Dict[str, str]:
@@ -53,7 +53,9 @@ def docker_client():
     client = docker.from_env()
     yield client
 
-@tenacity.retry(stop=tenacity.stop_after_delay(240), wait=tenacity.wait_fixed(5), retry=tenacity.retry_if_exception_type(AssertionError))
+@tenacity.retry(stop=tenacity.stop_after_delay(240),
+                wait=tenacity.wait_fixed(5),
+                retry=tenacity.retry_if_exception_type(AssertionError))
 def try_checking_task_state(running_service, service_name):
     tasks = running_service.tasks()
     assert tasks is not None
@@ -72,17 +74,21 @@ def try_checking_task_state(running_service, service_name):
     difference = now - creation_time
     assert difference.total_seconds() > 5
 
-# the swarm should be up prior to testing... using make up-swarm
+
 def test_services_running(docker_client, services_docker_compose, tools_docker_compose):
+    """
+    the swarm should be up prior to testing... using make up-swarm
+
+    """
     running_services = docker_client.services.list()
 
     assert (len(services_docker_compose["services"]) + len(tools_docker_compose["services"])) == len(running_services)
 
     # all the services shall be available here
     for service_name in services_docker_compose["services"].keys():
         # find the service
-        running_service = [x for x in running_services if service_name in x.name]
+        running_service = [s for s in running_services if service_name in s.name]
         assert len(running_service) == 1
         running_service = running_service[0]
-        # check health        
-        try_checking_task_state(running_service, service_name)        
+        # check health
+        try_checking_task_state(running_service, service_name)

services/director/Dockerfile

@@ -12,7 +12,10 @@ RUN adduser -D -u 8004 scu &&\
     addgroup -g $DOCKER_GID_ARG docker
 
 ENV HOME /home/scu
-ENV PIP  /home/scu/.venv/bin/pip3
+# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197
+# - Added explicit --cache-dir that is removed at the end of the build
+ENV PIP  /home/scu/.venv/bin/pip3 --cache-dir=/pipcache
+# -----
 
 EXPOSE 8001
 ENV REGISTRY_AUTH = ''
@@ -47,14 +50,14 @@ RUN apk add --no-cache \
 
 # create virtual environment
 RUN python3 -m venv $HOME/.venv &&\
-    $PIP install --no-cache-dir --upgrade \
+    $PIP install --upgrade \
       pip \
       wheel \
       setuptools
 
 # this is done to accelerate installation of director later on
 COPY --chown=scu:scu services/director/requirements/base.txt requirements-base.txt
-RUN $PIP install --no-cache-dir -r requirements-base.txt &&\
+RUN $PIP install -r requirements-base.txt &&\
     rm requirements-base.txt
 
 COPY --chown=scu:scu services/director/docker docker
@@ -83,16 +86,18 @@ FROM build as cache
 COPY --chown=scu:scu services/director $HOME/services/director
 COPY --chown=scu:scu packages/service-library $HOME/packages/service-library
 WORKDIR $HOME/services/director
-RUN $PIP install --no-cache-dir -r requirements/prod.txt
+RUN $PIP install -r requirements/prod.txt
 WORKDIR $HOME/packages/service-library
-RUN $PIP install --no-cache-dir -r requirements/dev.txt
+RUN $PIP install -r requirements/dev.txt
 
 # --------------------------Production stage -------------------
 FROM cache as production
 WORKDIR $HOME
 ENV DEBUG 0
 ENV RUN_DOCKER_ENGINE_ROOT=0
-RUN rm -rf $HOME/services \
-      && rm -rf $HOME/packages
+RUN rm -rf $HOME/services &&\
+    rm -rf $HOME/packages &&\
+    rm -rf /pipcache
+
 ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh" ]
 CMD ["/bin/sh", "docker/boot.sh"]

services/director/docker/boot.sh

@@ -7,16 +7,14 @@ then
   echo "INFO: Booting in development mode ..."
   echo "DEBUG: Director running as `id $(whoami)`"
   echo "DEBUG: Director running groups `groups`"
-  
+
   echo "Installing director service ..."
   cd $HOME/services/director
-  $PIP install --no-cache-dir -r requirements/dev.txt
+  $PIP install -r requirements/dev.txt
   $PIP list
   cd $HOME
   simcore-service-director --loglevel=debug
 else
   echo "INFO: Booting in production mode ..."
   simcore-service-director --loglevel=info
 fi
-
-

services/docker-compose.cache.yml

@@ -11,10 +11,10 @@ services:
       target: cache
   storage:
     build:
-      target: cache    
+      target: cache
   webclient:
     build:
       target: build
   webserver:
     build:
-      target: cache
+      target: cache

services/dy-2Dgraph/use-cases/cc/Dockerfile

@@ -11,7 +11,9 @@ USER $NB_USER
 
 # install requirements --------------------------------------------------------
 COPY --chown=jovyan:users cc/requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197
+RUN pip --cache-dir=/pipcache install -r requirements.txt &&\
+    rm -rf /pipcache
 # prepare for booting ---------------------------------------------------------
 COPY --chown=jovyan:users docker /docker
 # set of arguments to copy the right notebook ---------------------------------
@@ -36,4 +38,4 @@ ARG NOTEBOOK_NAME
 ARG NOTEBOOK_FOLDER_NAME
 # copy the notebook in the image
 COPY --chown=jovyan:users cc/${NOTEBOOK_FOLDER_NAME}/${NOTEBOOK_NAME} ${NOTEBOOK_NAME}
-ENTRYPOINT [ "/bin/bash", "/docker/boot.sh" ]
+ENTRYPOINT [ "/bin/bash", "/docker/boot.sh" ]

services/dy-2Dgraph/use-cases/kember/Dockerfile

@@ -4,7 +4,10 @@ LABEL maintainer="sanderegg"
 
 # install requirements --------------------------------------------------------
 COPY --chown=jovyan:users kember/requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197
+RUN pip --cache-dir=/pipcache install -r requirements.txt &&\
+    rm -rf /pipcache
+
 
 # prepare for booting ---------------------------------------------------------
 COPY --chown=jovyan:users docker /docker
@@ -29,4 +32,4 @@ FROM common AS production
 ARG NOTEBOOK_NAME
 # copy the notebook in the image
 COPY --chown=jovyan:users kember/${NOTEBOOK_NAME} ${NOTEBOOK_NAME}
-ENTRYPOINT [ "/bin/bash", "/docker/boot.sh" ]
+ENTRYPOINT [ "/bin/bash", "/docker/boot.sh" ]

services/sidecar/Dockerfile

@@ -12,7 +12,9 @@ RUN adduser -D -u 8004 scu &&\
     addgroup -g $DOCKER_GID_ARG docker
 
 ENV HOME /home/scu
-ENV PIP  /home/scu/.venv/bin/pip3
+# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197
+# - Added explicit --cache-dir that is removed at the end of the build
+ENV PIP  /home/scu/.venv/bin/pip3 --cache-dir=/pipcache
 
 EXPOSE 8000
 VOLUME /home/scu/input
@@ -38,14 +40,14 @@ RUN apk add --no-cache \
       libc-dev
 
 RUN python3 -m venv $HOME/.venv &&\
-    $PIP install --no-cache-dir --upgrade \
+    $PIP install --upgrade \
       pip \
       wheel \
       setuptools
 
 # TODO: check if scu:scu copy is necessary!?
 COPY --chown=scu:scu services/sidecar/requirements/base.txt requirements-base.txt
-RUN $PIP install --no-cache-dir -r requirements-base.txt &&\
+RUN $PIP install -r requirements-base.txt &&\
     rm requirements-base.txt
 
 COPY --chown=scu:scu services/sidecar/docker docker
@@ -80,7 +82,7 @@ COPY --chown=scu:scu services/sidecar $HOME/services/sidecar
 COPY --chown=scu:scu services/storage/client-sdk $HOME/services/storage/client-sdk
 
 WORKDIR /home/scu/services/sidecar
-RUN $PIP --no-cache-dir install -r requirements/prod.txt ;\
+RUN $PIP install -r requirements/prod.txt ;\
     $PIP list
 
 
@@ -101,7 +103,8 @@ FROM cache as production
 WORKDIR /home/scu/
 
 RUN . $HOME/.venv/bin/activate; pip list &&\
-    rm -rf $HOME/services
+    rm -rf $HOME/services &&\
+    rm -rf /pipcache
 
 ENV DEBUG 0
 ENV RUN_DOCKER_ENGINE_ROOT=0

services/sidecar/boot.sh

@@ -9,8 +9,8 @@ then
   echo "DEBUG: Sidecar running groups `groups`"
 
   cd $HOME/services/sidecar
-  pip3 install --no-cache-dir -r requirements/dev.txt
-  pip3 list
+  $PIP install -r requirements/dev.txt
+  $PIP list
 
   cd $HOME
   celery worker --app sidecar.celery:app --concurrency 2 --loglevel=debug

services/storage/Dockerfile

@@ -1,4 +1,3 @@
-# TODO: this is still not finished!!
 FROM python:3.6-alpine as base
 
 LABEL maintainer=mguidon
@@ -20,8 +19,12 @@ ENV HOME /home/scu
 ENV VENV /home/scu/.venv/
 ENV VENV3 $VENV
 ENV VENV2 /home/scu/.venv27/
-ENV PIP  /home/scu/.venv/bin/pip3
-ENV PIP2 $VENV2/bin/pip
+
+# TODO: tmp workaround for https://github.com/pypa/pip/issues/6197
+# - Added explicit --cache-dir that is removed at the end of the build
+ENV PIP  /home/scu/.venv/bin/pip3 --cache-dir=/pipcache
+ENV PIP2 $VENV2/bin/pip --cache-dir=/pipcache2
+# -----
 
 EXPOSE 8080
 
@@ -48,16 +51,18 @@ RUN apk add --no-cache \
       libc-dev \
       libffi-dev \
       python2 \
-      python2-dev
+      python2-dev \
+      linux-headers
 
 RUN python3 -m venv $HOME/.venv &&\
-    $PIP install --no-cache-dir --upgrade \
+    $PIP install --upgrade \
       pip \
       wheel \
-      setuptools
+      setuptools &&\
+    $PIP list
 
 RUN $PIP install virtualenv &&\
-    $VENV3/bin/virtualenv --python=python2  $VENV2
+    $VENV3/bin/virtualenv --python=python2 $VENV2
 
 WORKDIR /home/scu
 
@@ -66,35 +71,34 @@ WORKDIR /home/scu
 COPY --chown=scu:scu services/storage/requirements/py27.txt requirements-py27.txt
 COPY --chown=scu:scu services/storage/requirements/base.txt requirements-base.txt
 COPY --chown=scu:scu services/storage/docker docker
-RUN $PIP install --no-cache-dir -r requirements-base.txt
 
-RUN apk add --no-cache \
-  linux-headers
-RUN $PIP2 install --no-cache-dir -r requirements-py27.txt
+RUN $PIP install -r requirements-base.txt
+RUN $PIP2 install numpy==1.16.0 &&\
+    $PIP2 install -r requirements-py27.txt
+
 # --------------------------Development stage -------------------
 FROM build as development
 
 ARG HOST_GID_ARG=1000
 
 # install test 3rd party packages to accelerate runtime installs
 COPY --chown=scu:scu services/storage/tests/requirements.txt requirements-tests.txt
-RUN $PIP install --no-cache-dir -r requirements-tests.txt
+RUN $PIP install -r requirements-tests.txt
 
 # in dev mode we give access to `scu` to host's mapped volumes
 RUN addgroup -g $HOST_GID_ARG hgrp &&\
-    addgroup scu hgrp && \
+    addgroup scu hgrp &&\
     chown -R scu:scu $HOME/.venv
 
 VOLUME /home/scu/packages
-VOLUME /home/scu/services/storage/
+VOLUME /home/scu/services/storage
 
 ENV DEBUG 1
 ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh" ]
 CMD ["/bin/sh", "docker/boot.sh"]
 
 
 # --------------------------Production multi-stage -------------------
-#FROM build as build-production
 FROM build as cache
 
 ENV SIMCORE_WEB_CONFIG production
@@ -105,28 +109,22 @@ COPY --chown=scu:scu packages $HOME/packages
 COPY --chown=scu:scu services/storage $HOME/services/storage
 
 WORKDIR /home/scu/services/storage
-RUN $PIP --no-cache-dir install -r requirements/prod.txt &&\
+RUN $PIP install -r requirements/prod.txt &&\
     $PIP list
 
-#-------------------
-#FROM base as production
-# TODO: PC some basic package missing
-
-#COPY --from=build-production --chown=scu:scu $HOME/services/server/boot.sh $HOME
-#COPY --from=build-production --chown=scu:scu $HOME/.venv $HOME/.venv
-#RUN . $HOME/.venv/bin/activate; pip list
 
-# FIXME: temporary solution until found missing packages
 WORKDIR /home/scu
-
 ENV DEBUG 0
 
 FROM cache as production
+
 # remove any unnecessary stuff
 RUN . $HOME/.venv/bin/activate; pip list &&\
     rm -rf $HOME/packages &&\
-    rm -rf $HOME/services/storage
+    rm -rf $HOME/services/storage &&\
+    rm -rf /pipcache &&\
+    rm -rf /pipcache2
+
 
 ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh" ]
 CMD ["/bin/sh", "docker/boot.sh"]
-

services/storage/docker/boot.sh

@@ -9,8 +9,8 @@ then
   echo "DEBUG: Workdir :`pwd`"
 
   cd $HOME/services/storage
-  pip install -r requirements/dev.txt
-  pip list
+  $PIP install -r requirements/dev.txt
+  $PIP list
 
   cd $HOME/
   simcore-service-storage --config docker-dev-config.yaml

services/storage/requirements/py27.in

@@ -0,0 +1 @@
+blackfynn==2.5.0

services/storage/requirements/py27.txt

@@ -1 +1,27 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file services/storage/requirements/py27.txt services/storage/requirements/py27.in
+#
 blackfynn==2.5.0
+boto3==1.9.86             # via blackfynn
+botocore==1.12.86         # via boto3, s3transfer
+certifi==2018.11.29       # via requests
+chardet==3.0.4            # via requests
+configparser==3.7.1       # via blackfynn
+docopt==0.6.2             # via blackfynn
+docutils==0.14            # via botocore
+futures==3.2.0            # via s3transfer
+idna==2.8                 # via requests
+jmespath==0.9.3           # via boto3, botocore
+numpy==1.16.0             # via blackfynn, pandas
+pandas==0.24.0            # via blackfynn
+protobuf==3.2.0           # via blackfynn
+psutil==5.5.0             # via blackfynn
+python-dateutil==2.7.5    # via botocore, pandas
+pytz==2018.4              # via blackfynn, pandas
+requests==2.21.0          # via blackfynn
+s3transfer==0.1.13        # via boto3
+six==1.12.0               # via protobuf, python-dateutil
+urllib3==1.24.1           # via botocore, requests