Is200/auth business logic

.env-devel

@@ -1,4 +1,5 @@
 # NOTE: write here host gid and docker gid.
+# TODO: keep hierarchy
 HOST_GID=1000
 DOCKER_GID=1001
 RUN_DOCKER_ENGINE_ROOT=0
@@ -18,4 +19,5 @@ S3_ENDPOINT=minio:9000
 S3_ACCESS_KEY=12345678
 S3_SECRET_KEY=12345678
 S3_BUCKET_NAME=simcore
-
+SMTP_HOST=smtp.gmail.com
+SMTP_PORT=465

.github/CODEOWNERS

@@ -1,14 +1,15 @@
 # Order is important. The last matching pattern has the most precedence.
 
 # files and folders recursively
-/api/                    @sanderegg, @pcrespov
-/docs/                   @pcrespov
-/services/computation    @mguidon
-/services/dy*            @sanderegg
-/services/sidecar        @pcrespov, @mguidon
-/services/web/client     @odeimaiz, @oetiker
-/services/web/server     @pcrespov
-/services/storage        @mguidon
+/api/                      @sanderegg, @pcrespov
+/docs/                     @pcrespov
+/packages/service-library  @pcrespov
+/services/computation      @mguidon
+/services/dy*              @sanderegg
+/services/sidecar          @pcrespov, @mguidon
+/services/web/client       @odeimaiz, @oetiker
+/services/web/server       @pcrespov
+/services/storage          @mguidon
 
 # any change in travis
 /.travis.yml     @odeimaiz

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,5 @@
+<!--  **WIP-** prefix in title if still work in progress -->
+
 ## What do these changes do?
 
 <!-- Please give a short brief about these changes. -->
@@ -6,9 +8,8 @@
 ## Related issue number
 
 <!--
- - Add **WIP-** prefix in title if it is still work in progress (i.e. should not be merged but is ready to start reviewing it)
+ Use [waffle.io] keywords in title/descriptions to trigger bot actions:
 
- - Use [waffle.io] keywords in title/descriptions to trigger bot actions:
    - close, closes, close, closes, closed, fix, fixes, fixed, resolve, resolves, resolved #[issueNumber]
    - connect to, connects to, connected to, connect, connects, connected #[issueNumber]
 -->
@@ -19,6 +20,7 @@
 - [ ] I think the code is well written
 - [ ] Unit tests for the changes exist
 - [ ] Documentation reflects the changes
+- [ ] **Runs in the swarm**
 - [ ] If you design a new module, add your user to .github/CODEOWNERS
 
 [waffle.io]:https://waffle.io/marketing-assets/documents/waffleio_cheatsheet_v1.pdf?utm_source=blog&utm_medium=cheatsheet-ctabutton&utm_campaign=cheatsheet

Makefile

@@ -101,7 +101,8 @@ run_test:
 	pytest --cov=pytest_docker -v packages/pytest_docker/tests
 	pytest --cov=s3wrapper -v packages/s3wrapper/tests
 	pytest --cov=simcore_sdk -v packages/simcore-sdk/tests
-	pytest --cov=simcore_service_webserver -v services/web/server/tests
+	pytest --cov=simcore_service_webserver -v services/web/server/tests/unit
+	pytest --cov=simcore_service_webserver -v services/web/server/tests/login
 	pytest --cov=simcore_service_director -v services/director/tests
 	pytest --cov=simcore_service_storage -v -m "not travis" services/storage/tests
 
@@ -116,7 +117,7 @@ test:
 	make run_test
 	make after_test
 
-PLATFORM_VERSION=3.17
+PLATFORM_VERSION=3.18
 
 push_platform_images:
 	${DOCKER} login masu.speag.com

packages/deploy-tools/Makefile

@@ -0,0 +1,35 @@
+SHELL = /bin/bash
+
+##
+# Definitions.
+
+.SUFFIXES:
+
+# Python virtual environment
+VENV_DIR = $(CURDIR)/../../.venv
+OUTPUT_DIR = $(CURDIR)/out
+
+
+##
+# Targets .
+
+.PHONE: up-swarm
+up-swarm:
+	docker swarm init
+	docker stack deploy -c "$(OUTPUT_DIR)/docker-compose.yml"  services
+
+
+.PHONE: down-swarm
+down-swarm:
+	docker swarm leave -f
+
+
+$(VENV_DIR):
+	@python3 -m venv "$(VENV_DIR)"
+	@"$(VENV_DIR)/bin/pip3" install --upgrade pip wheel setuptools
+	@"$(VENV_DIR)/bin/pip3" install pylint
+	@echo "To activate the virtual environment, execute 'source env/bin/activate'"
+
+.PHONY: venv
+# venv – Create the virtual environment into venv folder
+venv: $(VENV_DIR)

packages/deploy-tools/README.md

@@ -0,0 +1,8 @@
+# Deploy tools [Under development]
+
+
+### Features
+
+  - finds ``osparc-simcore/services/docker-compose`` files
+  - merges docker-compose files
+  - resolves services/docker-compose files and produces a  docker-compose for deployment

packages/deploy-tools/requirements.txt

@@ -0,0 +1,2 @@
+typing==3.6.6
+PyYAML==3.13

packages/deploy-tools/setup.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""The setup script
+
+"""
+import sys
+import re
+from pathlib import Path
+from setuptools import setup, find_packages
+from typing import List
+
+CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
+COMMENT = re.compile(r'^\s*#')
+
+def list_packages(reqpath: Path) -> List[str]:
+    pkg_names = []
+    with reqpath.open() as f:
+        pkg_names = [line.strip() for line in f.readlines() if not COMMENT.match(line)]
+    return pkg_names
+
+
+requirements = list_packages(CURRENT_DIR / 'requirements.txt')
+
+setup_requirements = ['pytest-runner', ]
+
+test_requirements = list_packages(CURRENT_DIR / 'tests' / 'requirements.txt')
+
+setup(
+    name='simcore-deploy-tools',
+    version='0.1.0',
+    author="Pedro Crespo (pcrespov)",
+    description="Collection of deploy tools",
+    classifiers=[
+        'Development Status :: 2 - Pre-Alpha',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Natural Language :: English',
+        'Programming Language :: Python :: 3.6'
+    ],
+    license="MIT license",
+    install_requires=requirements,
+    packages=find_packages(where='src'),
+    package_dir={
+        '': 'src',
+    },
+    python_requires='>3.6, <3.7',
+    include_package_data=True,
+    setup_requires=setup_requirements,
+    test_suite='tests',
+    tests_require=test_requirements,
+    zip_safe=False,
+    entry_points={
+        'console_scripts': [
+            'simcore-deploy-tools = deploytools.cli:main',
+        ],
+    }
+)

packages/deploy-tools/src/deploytools/cli.py

@@ -0,0 +1,13 @@
+# TODO: redo with cookie-cutter
+import sys
+
+from .utils import output_dir
+from .swarm_composer import create_deploy
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    # TODO: use click to create cli!
+    outdir = output_dir()
+    create_deploy(outdir)

packages/deploy-tools/src/deploytools/merge.py

@@ -0,0 +1,42 @@
+""" Merges based on docker-compose tools
+"""
+
+from itertools import accumulate, chain
+from typing import Dict, List, TypeVar
+
+T = TypeVar('T')
+
+
+def merge_lists(lhs: List[T], rhs: List[T]) -> List:
+    # NOTE assumes all elements in the list are of the same type!
+    # NOTE: does not respect order
+    res = list( set(lhs + rhs) )
+    return res
+
+
+def merge_dicts(lhs: Dict, rhs: Dict) -> Dict:
+
+    merged_keys = set( chain(lhs.keys(), rhs.keys()) )
+    res = dict.fromkeys(merged_keys, None)
+    for key in merged_keys:
+        if key in lhs:
+            lhs_value = lhs[key]
+            if key not in rhs:
+                res[key] = lhs_value
+            else: # rhs overrides
+                assert key in rhs
+                rhs_value = rhs[key]
+                if isinstance(rhs_value, dict):
+                    res[key] = merge_dicts(lhs_value, rhs_value)
+                elif isinstance(rhs_value, list):
+                    res[key] = merge_lists(lhs_value, rhs_value)
+                else:
+                    res[key] = rhs_value
+        else:
+            res[key] = rhs[key]
+
+    return res
+
+
+def merge_docker_compose(*compose_dicts):
+    return list(accumulate(compose_dicts, merge_dicts))[-1]

packages/deploy-tools/src/deploytools/readers.py

@@ -0,0 +1,34 @@
+""" File readers
+
+"""
+
+from typing import Dict, Optional
+
+import yaml
+
+from .utils import env_devel_file, osparc_simcore_root_dir
+
+
+def load_docker_compose(suffix: Optional[str]=None) -> Dict:
+    name = "docker-compose"
+    if suffix:
+        name += suffix
+    name += ".yml"
+    dcpath = osparc_simcore_root_dir() / "services" / name
+    assert dcpath.exists(), dcpath
+
+    dc = dict()
+    with dcpath.open() as f:
+        dc = yaml.safe_load(f)
+    return dc
+
+
+def load_devel_environ(name: Optional[str]=None):
+    environ = {}
+    with env_devel_file(name).open() as f:
+        for line in f:
+            line = line.strip()
+            if line and not line.startswith("#"):
+                key, value = line.split("=")
+                environ[key] = value
+    return environ

packages/deploy-tools/src/deploytools/swarm_composer.py

@@ -0,0 +1,112 @@
+import re
+from collections import defaultdict
+from copy import deepcopy
+from functools import lru_cache
+from typing import Dict, List, Optional
+from pathlib import Path
+
+from .merge import merge_docker_compose
+from .readers import load_devel_environ, load_docker_compose
+from .utils import dump_to_file, dump_to_stdout, output_dir
+
+
+# SWARM OPTIONS ------------
+@lru_cache(None)
+def get_constraints() -> Dict:
+    # TODO: check that these constraints are correct
+    _constraints = defaultdict.fromkeys(["director", "webserver"],
+        ['node.platform.os == linux', "node.role == manager"])
+    _constraints.default_factory = list
+
+    _constraints["apihub"].append('node.platform.os == linux')
+    return _constraints
+
+#-------------------------------
+
+
+
+def resolve_environs(service_environ: List[str]) -> Dict:
+    """ Creates a dict with the environment variables
+        inside of a webserver container
+    """
+    host_environ = load_devel_environ()
+
+    container_environ = {
+       # 'SIMCORE_WEB_OUTDIR': 'home/scu/services/web/client' # defined in Dockerfile
+    }
+
+    MATCH = re.compile(r'\$\{(\w+)+')
+
+    for item in service_environ:
+        key, value = item.split("=")
+        m = MATCH.match(value)
+        if m:
+            envkey = m.groups()[0]
+            value = host_environ[envkey]
+        container_environ[key] = value
+
+    return container_environ
+
+
+
+def resolve_deploy(dc_source: Dict, exclude: Optional[List[str]]=None) -> Dict:
+
+    constraints = get_constraints()
+    if exclude is None:
+        exclude = []
+
+    dc_deploy = deepcopy(dc_source)
+
+    for name in dc_source["services"]:
+        service = dc_deploy["services"][name]
+        if name in exclude:
+            dc_deploy["services"].pop(name)
+            continue
+
+        # remove builds
+        service.pop("build", None)
+
+        # replace by image
+        service.setdefault("image", "services_{}:latest".format(name))
+
+        # add deploy info
+        constraint = constraints.get(name, None)
+        if constraint is not None:
+            service["deploy"] = {
+                "placement": {
+                    "constraints": list(constraint)
+                }
+            }
+
+        # resolve environs
+        if "environment" in service:
+            environs = resolve_environs(service["environment"])
+            service["environment"] = [ "{}={}".format(k,v) for k,v in environs.items()]
+
+        print("{:-^20}".format(name))
+        dump_to_stdout(service)
+
+    return dc_deploy
+
+
+def create_deploy(outdir: Path):
+
+    prod = load_docker_compose()
+    tools = load_docker_compose(".tools")
+
+    merged = merge_docker_compose(prod, tools)
+
+    dc_deploy = resolve_deploy(merged, exclude=["webclient"])
+    fpath = outdir / "docker-compose.yml"
+    dump_to_file(dc_deploy, fpath)
+
+    # dc_devel = merge(dc_deploy, docker_compose(".devel"))
+    # fpath = here() / "out" / "docker-compose.devel.yml"
+    # yaml_dump(fpath, dc_deploy)
+
+
+
+
+
+if __name__ == "__main__":
+    create_deploy(output_dir())