WIP #198: setup for sidecar

 - created setup and requirements for dev&prod
 - installs package in docker (instead of reference via sys.path). this way all third-parties are installed.
 - minor cleanup of code

Author: Pedro Crespo

services/sidecar/Dockerfile

@@ -2,32 +2,54 @@ FROM python:3.6-alpine as common
 
 LABEL maintainer="Manuel Guidon <[email protected]"
 
-WORKDIR /work/sidecar
+RUN apk add --no-cache \
+      postgresql-dev \
+      gcc \
+      libc-dev
 
-RUN apk add --no-cache postgresql-dev gcc libc-dev
+RUN pip install --upgrade \
+      pip \
+      wheel \
+      setuptools
 
-COPY services/sidecar/requirements.txt requirements.txt
+WORKDIR /work
+# Buil context set at repo's root
+COPY services/sidecar/requirements requirements
 
-RUN pip install --upgrade pip \
-    && pip install -r requirements.txt \
-    && pip list --format=columns
+RUN pip install -r requirements/base.txt &&\
+    rm -rf requirements
+
+# Keeps same folder structure as in repo so we can reuse relative paths
+RUN mkdir -p /work/packages &&\
+    mkdir -p /work/services/sidecar
 
 EXPOSE 8000
 
+
+# --------------------------Development stage -------------------
 FROM common as development
 
-VOLUME /work/sidecar
 VOLUME /work/packages
+VOLUME /work/services/sidecar
+WORKDIR /work/services/sidecar
+
+CMD pip install -r requirements/dev.txt &&\
+    celery -A sidecar worker -c 2 --loglevel=info
 
-# NO clue why this does not work without explicitly specifying
-ENTRYPOINT celery -A sidecar worker -c 2 --loglevel=info
 
+# --------------------------Production stage -------------------
 FROM common as production
 
-# the context for the build is the git repo root directory
-COPY services/sidecar/src /work
+# Buil context set at repo's root
 COPY packages /work/packages
+COPY services/sidecar /work/services/sidecar
+
+WORKDIR /work/services/sidecar
+
+RUN pip install -r requirements/prod.txt && pip list;\
+    rm -rf /work/packages &&\
+    rm -rf /work/services/sidecar
+
+WORKDIR /work
 
-# NO clue why this does not work without explicitly specifying
-ENV PYTHONPATH="/work/packages/simcore-sdk/src:/work/packages/s3wrapper/src"
 ENTRYPOINT celery -A sidecar worker -c 2 --loglevel=info

services/sidecar/README.md

@@ -1,3 +1,29 @@
 # Sidecar
 
 Use sidecar container to control computational service.
+
+
+
+```bash
+
+# create an prepare a clean virtual environment ...
+python3 -m venv .venv
+source .venv/bin/activate
+pip3 install --upgrade pip setuptools wheel
+# ..or
+make .venv
+source .venv/bin/activate
+
+
+cd services/sidecar
+
+# for development (edit mode)
+# see how this packages is listed with a path to it src/ folder
+pip3 install -r requirements/dev.txt
+pip3 list
+
+
+# for production
+pip3 install -r requirements/prod.txt
+pip3 list
+```

services/sidecar/requirements.txt renamed to services/sidecar/requirements/base.txt

@@ -1,3 +1,5 @@
+# Common third party packages
+# Please keep alphabetical order
 celery==4.1.0
 docker==3.3.0
 kombu==4.1.0

services/sidecar/requirements/dev.txt

@@ -0,0 +1,8 @@
+-e .
+-e ../../packages/s3wrapper/
+-e ../../packages/simcore-sdk/
+
+
+# extra dev tools
+autopep8
+pylint

services/sidecar/requirements/prod.txt

@@ -0,0 +1,3 @@
+.
+../../packages/s3wrapper/
+../../packages/simcore-sdk/

services/sidecar/setup.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+import sys
+import pathlib
+
+from setuptools import (
+    find_packages,
+    setup,
+)
+
+_CDIR = pathlib.Path(sys.argv[0] if __name__ == "__main__" else __file__).parent
+_PACKAGES_DIR = _CDIR.absolute().parent.parent / "packages"
+
+def list_requirements_in(filename):
+    requires = []
+    with (_CDIR / "requirements" / filename).open() as fh:
+        requires = [line.strip() for line in fh.readlines() if not line.lstrip().startswith("#")]
+    return requires
+
+
+INSTALL_REQUIRES = list_requirements_in("base.txt")
+
+# FIXME: not sure how to add these dependencies *here* and *only* in production. Now using requirements/prod.txt
+SC_PACKAGES = [
+    "s3wrapper",
+    "simcore-sdk"
+]
+
+setup(
+    name='simcore-service-sidecar',
+    version='0.0.1',
+    packages=find_packages(where='src'),
+    package_dir={
+        '': 'src',
+    },
+    install_requires=INSTALL_REQUIRES,
+    python_requires='>=3.6',
+)

services/sidecar/src/sidecar/__init__.py

@@ -0,0 +1 @@
+from .core import *

services/sidecar/src/sidecar/sidecar.py renamed to services/sidecar/src/sidecar/core.py

@@ -12,14 +12,15 @@
 from sqlalchemy import and_, exc
 from sqlalchemy.orm.attributes import flag_modified
 
-from sidecar_utils import (DbSettings, DockerSettings, ExecutorSettings,
-                           RabbitSettings, S3Settings, delete_contents,
-                           find_entry_point, is_node_ready)
 from simcore_sdk.config.rabbit import Config as rabbit_config
 from simcore_sdk.models.pipeline_models import (RUNNING, SUCCESS,
                                                 ComputationalPipeline,
                                                 ComputationalTask)
 
+from .utils import (DbSettings, DockerSettings, ExecutorSettings,
+                    RabbitSettings, S3Settings, delete_contents,
+                    find_entry_point, is_node_ready)
+
 rabbit_config = rabbit_config()
 celery= Celery(rabbit_config.name, broker=rabbit_config.broker, backend=rabbit_config.backend)
 

services/sidecar/src/sidecar/sidecar_utils.py renamed to services/sidecar/src/sidecar/utils.py

@@ -29,14 +29,18 @@ def list_requirements_in(filename):
 setup(
     name='simcore-web-server',
     version='0.0.0',
-    package_dir={'': 'src'},
-    packages=find_packages('src'),
+    packages=find_packages(where='src'),
+    package_dir={
+        '': 'src',
+    },
     include_package_data=True,
     package_data={
         'config': ['../config/*.yaml'] #FIXME: this is still not copied!??
     },
     entry_points={
-        'console_scripts': ['service-web-server=server.__main__:main']},
+        'console_scripts': [
+            'service-web-server=server.__main__:main', ]
+        },
     python_requires='>=3.6',
     install_requires=INSTALL_REQUIRES,
     tests_require=TESTS_REQUIRE,