Cleanup catalog service (#1582)

Cleanup of catalog service before adding more functionality

* Major refactoring
- api/dependencies: dependencies injected in handlers
- api/routes: endpoint handlers and routing
- api/root.py: join all openapi-specs into a single router
- models: domain (orm and business logic models)  and schema (i/o schemas for openapi)  models
- db: database tables (i.e. sa schemas for tables) and repositories (crud layer between handlers and db calls)
- core: init application and settings (parses values by args, environs, .env or default, in this order)
- services: modules/plugins with logic for the app
- __main__,__version__ : main entrypoint and version

Author: pcrespov

ci/github/unit-testing/catalog.bash

@@ -12,7 +12,7 @@ install() {
 test() {
     pytest --cov=simcore_service_catalog --durations=10 --cov-append \
           --color=yes --cov-report=term-missing --cov-report=xml \
-          -v -m "not travis" services/catalog/tests
+          -v -m "not travis" services/catalog/tests/unit
 }
 
 # Check if the function exists (bash specific)

ci/helpers/ensure_python_pip.bash

@@ -9,7 +9,7 @@ set -euo pipefail
 IFS=$'\n\t'
 
 # Pin pip version to a compatible release https://www.python.org/dev/peps/pep-0440/#compatible-release
-PIP_VERSION=19.3.1
+PIP_VERSION=20.1.1
 
 echo "INFO:" "$(python --version)" "@" "$(command -v python)"
 

ci/travis/unit-testing/catalog.bash

@@ -34,7 +34,7 @@ script() {
     then
         pytest --cov=simcore_service_catalog --durations=10 --cov-append \
           --color=yes --cov-report=term-missing --cov-report=xml \
-          -v -m "not travis" services/catalog/tests
+          -v -m "not travis" services/catalog/tests/unit
     else
         echo "No changes detected. Skipping unit-testing of catalog."
     fi

packages/postgres-database/docker/Dockerfile

@@ -23,7 +23,7 @@ RUN apt-get update &&\
 RUN python -m venv ${VIRTUAL_ENV}
 
 RUN pip --no-cache-dir install --upgrade \
-  pip~=20.0.2  \
+  pip~=20.1.1  \
   wheel \
   setuptools
 

packages/pytest-simcore/src/pytest_simcore/postgres_service2.py

@@ -0,0 +1,148 @@
+"""
+    sets up a docker-compose
+
+IMPORTANT: incompatible with pytest_simcore.docker_compose and pytest_simcore.postgres_service
+
+"""
+# pylint:disable=unused-variable
+# pylint:disable=unused-argument
+# pylint:disable=redefined-outer-name
+
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Callable, Coroutine, Dict, Union
+
+import aiopg.sa
+import pytest
+import sqlalchemy as sa
+import yaml
+from dotenv import dotenv_values
+
+import simcore_postgres_database.cli as pg_cli
+from simcore_postgres_database.models.base import metadata
+
+current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
+
+
+@pytest.fixture(scope="session")
+def env_devel_file(project_slug_dir: Path) -> Path:
+    # takes as a bas
+    env_devel_path = project_slug_dir / ".env-devel"
+    assert env_devel_path.exists()
+    return env_devel_path
+
+
+@pytest.fixture(scope="session")
+def test_environment(env_devel_file: Path) -> Dict[str, str]:
+    env = dotenv_values(env_devel_file, verbose=True, interpolate=True)
+    return env
+
+
+@pytest.fixture(scope="session")
+def test_docker_compose_file(pytestconfig) -> Path:
+    """Get an absolute path to the  `docker-compose.yml` file.
+       Override this fixture in your tests if you need a custom location.
+    """
+    return os.path.join(str(pytestconfig.rootdir), "tests", "docker-compose.yml")
+
+
+@pytest.fixture(scope="session")
+def docker_compose_file(test_environment: Dict[str, str], tmpdir_factory, test_docker_compose_file) -> Path:
+    # Overrides fixture in https://github.com/avast/pytest-docker
+
+    environ = dict(
+        os.environ
+    )  # NOTE: do not forget to add the current environ here, otherwise docker-compose fails
+    environ.update(test_environment)
+
+    # assumes prototype in cwd
+    src_path = test_docker_compose_file
+    assert src_path.exists, f"Expected prototype at cwd, i.e. {src_path.resolve()}"
+
+    dst_path = Path(
+        str(
+            tmpdir_factory.mktemp("docker_compose_file_fixture").join(
+                "docker-compose.yml"
+            )
+        )
+    )
+
+    shutil.copy(src_path, dst_path.parent)
+    assert dst_path.exists()
+
+    # configs
+    subprocess.run(
+        f'docker-compose --file "{src_path}" config > "{dst_path}"',
+        shell=True,
+        check=True,
+        env=environ,
+    )
+
+    return dst_path
+
+
+@pytest.fixture(scope="session")
+def postgres_service2(docker_services, docker_ip, docker_compose_file: Path) -> Dict:
+
+    # check docker-compose's environ is resolved properly
+    config = yaml.safe_load(docker_compose_file.read_text())
+    environ = config["services"]["postgres"]["environment"]
+
+    # builds DSN
+    config = dict(
+        user=environ["POSTGRES_USER"],
+        password=environ["POSTGRES_PASSWORD"],
+        host=docker_ip,
+        port=docker_services.port_for("postgres", 5432),
+        database=environ["POSTGRES_DB"],
+    )
+
+    dsn = "postgresql://{user}:{password}@{host}:{port}/{database}".format(**config)
+
+    def _create_checker() -> Callable:
+        def is_postgres_responsive() -> bool:
+            try:
+                engine = sa.create_engine(dsn)
+                conn = engine.connect()
+                conn.close()
+            except sa.exc.OperationalError:
+                return False
+            return True
+
+        return is_postgres_responsive
+
+    # Wait until service is responsive.
+    docker_services.wait_until_responsive(
+        check=_create_checker(), timeout=30.0, pause=0.1,
+    )
+
+    config["dsn"] = dsn
+    return config
+
+
+@pytest.fixture(scope="session")
+def make_engine(postgres_service2: Dict) -> Callable:
+    dsn = postgres_service2["dsn"]  # session scope freezes dsn
+
+    def maker(is_async=True) -> Union[Coroutine, Callable]:
+        return aiopg.sa.create_engine(dsn) if is_async else sa.create_engine(dsn)
+
+    return maker
+
+
+@pytest.fixture
+def apply_migration(postgres_service2: Dict, make_engine) -> None:
+    kwargs = postgres_service2.copy()
+    kwargs.pop("dsn")
+    pg_cli.discover.callback(**kwargs)
+    pg_cli.upgrade.callback("head")
+    yield
+    pg_cli.downgrade.callback("base")
+    pg_cli.clean.callback()
+
+    # FIXME: deletes all because downgrade is not reliable!
+    engine = make_engine(False)
+    metadata.drop_all(engine)

requirements.txt

@@ -15,6 +15,7 @@
 
 # formatter
 black
+isort
 # dependency manager
 pip-tools
 # version manager

scripts/openapi/oas_resolver/Dockerfile

@@ -12,7 +12,7 @@ WORKDIR /src
 
 # update pip
 RUN pip install --no-cache-dir --upgrade \
-      pip~=20.0.2  \
+      pip~=20.1.1  \
       wheel \
       setuptools
 

services/api-server/Dockerfile

@@ -1,5 +1,5 @@
 ARG PYTHON_VERSION="3.6.10"
-FROM python:${PYTHON_VERSION}-slim as base
+FROM python:${PYTHON_VERSION}-slim-buster as base
 #
 #  USAGE:
 #     cd sercices/api-server
@@ -43,7 +43,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 # those from our virtualenv.
 ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 
-EXPOSE 8001
+EXPOSE 8000
 EXPOSE 3000
 
 # -------------------------- Build stage -------------------
@@ -64,7 +64,7 @@ RUN apt-get update &&\
 RUN python -m venv ${VIRTUAL_ENV}
 
 RUN pip install --upgrade --no-cache-dir \
-    pip~=20.0.2  \
+    pip~=20.1.1  \
     wheel \
     setuptools
 
@@ -111,7 +111,10 @@ ENV PYTHONOPTIMIZE=TRUE
 
 WORKDIR /home/scu
 
+# Starting from clean base image, copies pre-installed virtualenv from cache
 COPY --chown=scu:scu --from=cache  ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+# Copies booting scripts
 COPY --chown=scu:scu services/api-server/docker services/api-server/docker
 RUN chmod +x services/api-server/docker/*.sh
 

services/api-server/Makefile

@@ -4,10 +4,13 @@
 include ../../scripts/common.Makefile
 
 # Custom variables
-APP_NAME     := $(notdir $(CURDIR))
-APP_CLI_NAME := simcore-service-$(APP_NAME)
-export APP_VERSION = $(shell cat VERSION)
-SRC_DIR      := $(abspath $(CURDIR)/src/$(subst -,_,$(APP_CLI_NAME)))
+APP_NAME         := $(notdir $(CURDIR))
+APP_CLI_NAME     := simcore-service-$(APP_NAME)
+APP_PACKAGE_NAME := $(subst -,_,$(APP_CLI_NAME))
+APP_VERSION      := $(shell cat VERSION)
+SRC_DIR          := $(abspath $(CURDIR)/src/$(APP_PACKAGE_NAME))
+
+export APP_VERSION
 
 .PHONY: reqs
 reqs: ## compiles pip requirements (.in -> .txt)
@@ -71,18 +74,32 @@ down: docker-compose.yml ## stops pg fixture
 	# killing any process using port 8000
 	-@fuser --kill --verbose --namespace tcp 8000
 
-######################
 
 
-.PHONY: build
-build: ## builds docker image (using main services/docker-compose-build.yml)
-	@$(MAKE_C) ${REPO_BASE_DIR} target=${APP_NAME} $@
+# BUILD ########
+
+
+.PHONY: build build-nc build-devel build-devel-nc build-cache build-cache-nc
+build build-nc build-devel build-devel-nc build-cache build-cache-nc: ## builds docker image (using main services/docker-compose-build.yml)
+	@$(MAKE_C) ${REPO_BASE_DIR} $@ target=${APP_NAME}
+
 
+.PHONY: openapi-specs openapi.json
+openapi-specs: openapi.json
+openapi.json: .env
+	# generating openapi specs file
+	python3 -c "import json; from $(APP_PACKAGE_NAME).__main__ import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@
 
-# GENERATION python client -------------------------------------------------
+
+# GENERATION python client ########
 .PHONY: python-client generator-help
+
 # SEE https://openapi-generator.tech/docs/usage#generate
 # SEE https://openapi-generator.tech/docs/generators/python
+#
+# TODO: put instead to additional-props.yaml and --config=openapi-generator/python-config.yaml
+# TODO: copy this code to https://github.com/ITISFoundation/osparc-simcore-python-client/blob/master/Makefile
+#
 
 # NOTE: assumes this repo exists
 GIT_USER_ID := ITISFoundation
@@ -91,7 +108,6 @@ GIT_REPO_ID := osparc-simcore-python-client
 SCRIPTS_DIR := $(abspath $(CURDIR)/../../scripts)
 GENERATOR_NAME := python
 
-# TODO: put instead to additional-props.yaml and --config=openapi-generator/python-config.yaml
 ADDITIONAL_PROPS := \
 	generateSourceCodeOnly=false\
 	hideGenerationTimestamp=true\
@@ -106,21 +122,13 @@ null  :=
 space := $(null) #
 comma := ,
 
-# TODO: fix this, shall be generated upon start when flag is provided
-
-
-
-# TODO: code_samples still added by hand!
 client:
 	# cloning $(GIT_USER_ID)/$(GIT_REPO_ID) -> $@
 	git clone [email protected]:$(GIT_USER_ID)/$(GIT_REPO_ID).git $@
 	cd client; git checkout -b "upgrade-${APP_VERSION}"
 
 
-python-client: client ## runs python client generator
-	# download openapi.json
-	curl -O http://localhost:8000/api/v0/openapi.json
-
+python-client: client openapi.json ## runs python client generator
 	cd $(CURDIR); \
 	$(SCRIPTS_DIR)/openapi-generator-cli.bash generate \
 		--generator-name=$(GENERATOR_NAME) \
@@ -134,8 +142,6 @@ python-client: client ## runs python client generator
 		--release-note="Updated to $(APP_VERSION)"
 
 
-
-
 generator-help: ## help on client-api generator
 	# generate help
 	@$(SCRIPTS_DIR)/openapi-generator-cli.bash help generate

services/api-server/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.0.2",
   "info": {
     "title": "Public API Server",
-    "description": "**osparc-simcore Public RESTful API Specifications**\n## Python Client\n- Github [repo](https://github.com/ITISFoundation/osparc-simcore-python-client)\n- Quick install: ``pip install git+https://github.com/ITISFoundation/osparc-simcore-python-client.git``\n",
+    "description": "**osparc-simcore Public RESTful API Specifications**\n## Python Library\n- Check the [documentation](https://itisfoundation.github.io/osparc-simcore-python-client)\n- Quick install: ``pip install git+https://github.com/ITISFoundation/osparc-simcore-python-client.git``\n",
     "version": "0.3.0",
     "x-logo": {
       "url": "https://raw.githubusercontent.com/ITISFoundation/osparc-manual/b809d93619512eb60c827b7e769c6145758378d0/_media/osparc-logo.svg",

services/catalog/.cookiecutterrc

@@ -0,0 +1,13 @@
+#
+# Environment variables used to configure this service
+#
+
+LOG_LEVEL=DEBUG
+
+POSTGRES_USER=test
+POSTGRES_PASSWORD=test
+POSTGRES_DB=test
+POSTGRES_HOST=localhost
+
+# Enables debug
+SC_BOOT_MODE=debug-ptvsd

services/catalog/.env-devel

@@ -0,0 +1,2 @@
+docker-compose.yml
+.env