Feature#19/auth serve2

.travis.yml

@@ -35,18 +35,14 @@ matrix:
         - while sudo lsof -Pi :5432 -sTCP:LISTEN -t; do sleep 1; done
 
       install:
-        - pip3 install pylint
-        - pip3 install pytest-cov
-        - pip3 install coveralls
-        - pip3 install -r packages/pytest_docker/requirements.txt
-        - pip3 install -r packages/s3wrapper/requirements.txt
-        - pip3 install -r packages/simcore-sdk/requirements.txt
-        - pip3 --version
+        - pip install --upgrade pip wheel setuptools && pip3 --version
+        - pip3 install packages/s3wrapper[test]
+        - pip3 install packages/simcore-sdk[test]
+        - pushd services/web/server; pip3 install -r requirements/ci.txt; popd
 
       before_script:
         - pylint --version
-        # change python path for 2nd party libraries
-        - export PYTHONPATH=${PYTHONPATH}:${TRAVIS_BUILD_DIR}/packages/s3wrapper/src
+        - pip3 list
 
       script:
         - make pylint

Makefile

@@ -20,6 +20,9 @@ all:
 	@echo 'run `make up-devel` to start your dev environment.'
 	@echo 'see Makefile for further targets'
 
+clean:
+	@git clean -dxf -e .vscode/
+
 build-devel:
 	${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.devel.yml build
 
@@ -76,9 +79,10 @@ before_test:
 	${DOCKER_COMPOSE} -f packages/simcore-sdk/tests/docker-compose.yml build
 
 run_test:
-	pytest --cov=pytest_docker -v packages/pytest_docker/
-	pytest --cov=s3wrapper -v packages/s3wrapper/
-	pytest -v packages/simcore-sdk/
+	pytest --cov=pytest_docker -v packages/pytest_docker/tests
+	pytest --cov=s3wrapper -v packages/s3wrapper/tests
+	pytest --cov=simcore_sdk -v packages/simcore-sdk/tests
+	pytest --cov=server -v services/web/server/tests
 
 after_test:
 	# leave a clean slate (not sure whether this is actually needed)
@@ -114,7 +118,10 @@ push_platform_images:
 	@diff -uN $@ $<
 	@false
 
-venv:
+.venv:
 	python3 -m venv .venv
 	.venv/bin/pip3 install --upgrade pip wheel setuptools
-	echo "To activate the venv, execute 'source .venv/bin/activate'"
+	@echo "To activate the venv, execute 'source .venv/bin/activate' or '.venv/bin/activate.bat' (WIN)"
+
+
+.PHONY: all clean build-devel rebuild-devel up-devel build up down test after_test push_platform_images

packages/s3wrapper/requirements-dev.txt

@@ -0,0 +1,4 @@
+# develop mode
+-e . ".[test]"
+
+autopep8>=1.3.5

packages/s3wrapper/setup.py

@@ -1,6 +1,29 @@
 from setuptools import setup
 
+install_requires = [
+    'minio==4.0.0',
+]
+
+tests_require = [
+    'coveralls~=1.3',
+    'mock~=2.0',
+    'pylint~=2.0',
+    'pytest~=3.6',
+    'pytest-cov~=2.5',
+    'pytest-docker~=0.6',
+    'requests~=2.19'
+]
+
 setup(
     name='s3wrapper',
     version='0.1.0',
+    package_dir={'': 'src'},
+    packages=['s3wrapper'],
+    python_requires='>=3.6',
+    install_requires=install_requires,
+    tests_require=tests_require,
+    extras_require= {
+        'test': tests_require
+    },
+    setup_requires=['pytest-runner']
 )

packages/simcore-sdk/requirements-dev.txt

@@ -0,0 +1,5 @@
+# develop mode
+-e . ".[test]"
+-e ../s3wrapper/
+
+autopep8>=1.3.5

packages/simcore-sdk/setup.py

@@ -1,6 +1,33 @@
 from setuptools import setup
 
+INSTALL_REQUIRES = [
+    'networkx==2.1',
+    'psycopg2==2.7.4',
+    'sqlalchemy==1.2.8',
+    'tenacity==4.12.0'
+    'trafaret-config==2.0.1'
+]
+
+TEST_REQUIRE = [
+    'coveralls~=1.3',
+    'mock~=2.0',
+    'pylint~=2.0',
+    'pytest~=3.6',
+    'pytest-cov~=2.5',
+    'pytest-docker~=0.6',
+    'requests~=2.19'
+]
+
 setup(
     name='simcore-sdk',
     version='0.1.0',
+    package_dir={'': 'src'},
+    packages=['simcore_sdk'],
+    python_requires='>=3.6',
+    INSTALL_REQUIRES=INSTALL_REQUIRES,
+    TEST_REQUIRE=TEST_REQUIRE,
+    extras_require= {
+        'test': TEST_REQUIRE
+    },
+    setup_requires=['pytest-runner']
 )

packages/simcore-sdk/src/simcore_sdk/config/db.py

@@ -1,12 +1,27 @@
-""" Basic configuration file for postgres
+""" Basic configuration file for postgres service
 
 """
 from os import environ as env
 
+import trafaret as T
 
+CONFIG_SCHEMA = T.Dict({
+    "database": T.String(),
+    "user": T.String(),
+    "password": T.String(),
+    T.Key("minsize", default=1 ,optional=True): T.Int(),
+    T.Key("maxsize", default=4, optional=True): T.Int(),
+    "host": T.Or( T.String, T.Null),
+    "port": T.Or( T.Int, T.Null),
+    "endpoint": T.Or( T.String, T.Null)
+})
 
+
+# TODO: deprecate!
 class Config():
+
     def __init__(self):
+        # TODO: uniform config classes . see server.config file
         POSTGRES_URL = env.get("POSTGRES_ENDPOINT", "postgres:5432")
         POSTGRES_USER = env.get("POSTGRES_USER", "simcore")
         POSTGRES_PW = env.get("POSTGRES_PASSWORD", "simcore")
@@ -18,7 +33,7 @@ def __init__(self):
         self._db = POSTGRES_DB
         self._endpoint = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}'.format(
             user=self._user, pw=self._pwd, url=self._url, db=self._db)
-            
+
     @property
     def endpoint(self):
-        return self._endpoint
+        return self._endpoint

packages/simcore-sdk/src/simcore_sdk/config/docker.py

@@ -1,8 +1,18 @@
 """ Basic configuration file for docker registry
 
 """
+import trafaret as T
+
+# TODO: adapt all data below!
+CONFIG_SCHEMA = T.Dict({
+    "user": T.String(),
+    "password": T.String(),
+    "registry": T.String()
+})
+
 
 class Config():
+    # TODO: uniform config classes . see server.config file
     def __init__(self):
         self._registry = "masu.speag.com"
         self._user = "z43"

packages/simcore-sdk/src/simcore_sdk/config/rabbit.py

@@ -5,7 +5,66 @@
 from os import environ as env
 
 import pika
+import yaml
+import trafaret as T
 
+
+# TODO: adapt all data below!
+# TODO: can use venv as defaults? e.g. $RABBITMQ_LOG_CHANNEL
+CONFIG_SCHEMA = T.Dict({
+    T.Key("host", default='rabbit', optional=True): T.String(),
+    T.Key("port", default=5672, optional=True): T.Int(),
+    "user": T.String(),
+    "password": T.String(),
+    "channels": T.Dict({
+        "progress": T.String(),
+        "log": T.String(),
+    T.Key("celery", default=dict(result_backend="rpc://"), optional=True): T.Dict({
+        T.Key("result_backend", default="${CELERY_RESULT_BACKEND}", optional=True): T.String()
+        })
+    })
+})
+
+
+CONFIG_EXAMPLES = map(yaml.load,[
+"""
+  user: simcore
+  password: simcore
+  channels:
+    log: comp.backend.channels.log
+    progress: comp.backend.channels.progress
+""",
+"""
+  host: rabbito
+  port: 1234
+  user: foo
+  password: secret
+  channels:
+    log: comp.backend.channels.log
+    progress: comp.backend.channels.progress
+""",
+"""
+  user: bar
+  password: secret
+  channels:
+    log: comp.backend.channels.log
+    progress: comp.backend.channels.progress
+  celery:
+    result_backend: 'rpc://'
+"""])
+
+
+def eval_broker(config):
+    """
+        Raises trafaret.DataError if config validation fails
+    """
+    CONFIG_SCHEMA.check(config) # raise exception
+    url = 'amqp://{user}:{password}@{host}:{port}'.format(**config)
+    return url
+
+
+# TODO: deprecate! -----------------------------------------------------------------------------
+# TODO: uniform config classes . see server.config file
 RABBITMQ_USER = env.get('RABBITMQ_USER','simcore')
 RABBITMQ_PASSWORD = env.get('RABBITMQ_PASSWORD','simcore')
 RABBITMQ_HOST="rabbit"
@@ -17,21 +76,37 @@
 
 CELERY_BROKER_URL = AMQ_URL
 CELERY_RESULT_BACKEND=env.get('CELERY_RESULT_BACKEND','rpc://')
+# -------------------------------------
 
-class Config():
-    def __init__(self):
+class Config:
+    def __init__(self, config=None):
+        if config is not None:
+            CONFIG_SCHEMA.check(config) # raise exception
+        else:
+            config = {}
+
+        # FIXME: get variables via config.get('') or
         # rabbit
-        self._broker_url = CELERY_BROKER_URL
-        self._result_backend = CELERY_RESULT_BACKEND
+        try:
+            self._broker_url = eval_broker(config)
+        except:                                     # pylint: disable=W0702
+            self._broker_url = CELERY_BROKER_URL
+
+        self._result_backend = config.get("celery", {}).get("result_backend") or CELERY_RESULT_BACKEND
         self._module_name = "tasks"
 
         # pika
-        self._pika_credentials = pika.PlainCredentials(RABBITMQ_USER, RABBITMQ_PASSWORD)
-        self._pika_parameters = pika.ConnectionParameters(host=RABBITMQ_HOST, 
-            port=RABBITMQ_PORT, credentials=self._pika_credentials, connection_attempts=100)
+        self._pika_credentials = pika.PlainCredentials(
+                config.get("user") or RABBITMQ_USER,
+                config.get("password") or RABBITMQ_PASSWORD)
+        self._pika_parameters = pika.ConnectionParameters(
+            host=config.get("host") or RABBITMQ_HOST,
+            port=config.get("port") or RABBITMQ_PORT,
+            credentials=self._pika_credentials,
+            connection_attempts=100)
 
-        self._log_channel = RABBITMQ_LOG_CHANNEL
-        self._progress_channel = RABBITMQ_PROGRESS_CHANNEL
+        self._log_channel = config.get("celery", {}).get("result_backend") or RABBITMQ_LOG_CHANNEL
+        self._progress_channel = config.get("celery", {}).get("result_backend") or RABBITMQ_PROGRESS_CHANNEL
 
     @property
     def parameters(self):
@@ -44,7 +119,7 @@ def log_channel(self):
     @property
     def progress_channel(self):
         return self._progress_channel
-    
+
     @property
     def broker(self):
         return self._broker_url
@@ -55,4 +130,4 @@ def backend(self):
 
     @property
     def name(self):
-        return self._module_name
+        return self._module_name

packages/simcore-sdk/src/simcore_sdk/config/s3.py

@@ -4,11 +4,23 @@
 from os import environ as env
 import logging
 
+import trafaret as T
 
 _LOGGER = logging.getLogger(__name__)
 
+
+CONFIG_SCHEMA = T.Dict({
+    "endpoint": T.String(),
+    "access_key": T.String(),
+    "secret_key": T.String(),
+    "bucket_name": T.String(),
+})
+
+
+# TODO: deprecate!
 class Config():
     def __init__(self):
+        # TODO: uniform config classes . see server.config file
         S3_ENDPOINT = env.get("S3_ENDPOINT", "minio:9000")
         S3_ACCESS_KEY = env.get("S3_ACCESS_KEY", "12345678")
         S3_SECRET_KEY = env.get("S3_SECRET_KEY", "12345678")

packages/simcore-sdk/src/simcore_sdk/models/pipeline_models.py

@@ -53,4 +53,4 @@ class ComputationalTask(Base):
     # utc timestamps for submission/start/end
     submit = Column(DateTime)
     start = Column(DateTime)
-    end = Column(DateTime)
+    end = Column(DateTime)

services/director/Dockerfile

@@ -29,8 +29,8 @@ COPY --chown=app:app requirements.txt requirements.txt
 RUN python -m venv venv \
  && chown -R app:app venv
 
-RUN venv/bin/pip install --upgrade pip \
- && venv/bin/pip install -r requirements.txt
+RUN venv/bin/pip3 install --upgrade pip wheel setuptools\
+ && venv/bin/pip3 install -r requirements.txt
 
 COPY --chown=app:app boot.sh boot.sh
 RUN chmod +x boot.sh

services/docker-compose.yml

@@ -48,6 +48,10 @@ services:
       - S3_ACCESS_KEY=${S3_ACCESS_KEY}
       - S3_SECRET_KEY=${S3_SECRET_KEY}
       - S3_BUCKET_NAME=${S3_BUCKET_NAME}
+      - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT}
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB}
     depends_on:
       - webclient
   #--------------------------------------------------------------------