[maintenance] pytest-simcore package

.codeclimate.yml

@@ -77,3 +77,4 @@ exclude_patterns:
   - "**/generated_code/"
   - "**/migrations/"
   - "**/*.js"
+  - "**/pytest-simcore/"

.github/CODEOWNERS

@@ -8,6 +8,7 @@ Makefile                                  @pcrespov, @sanderegg
 /api/                                     @sanderegg, @pcrespov
 /ci/                                      @sanderegg, @pcrespov
 /docs/                                    @pcrespov
+/packages/pytest-simcore                  @pcrespov, @sanderegg
 /packages/service-library                 @pcrespov
 /scripts/demo                             @odeimaiz, @pcrespov
 /scripts/json-schema-to-openapi-schema    @sanderegg

packages/pytest-simcore/Makefile

@@ -0,0 +1,16 @@
+#
+# Targets for DEVELOPMENT of Service Library
+#
+include ../../scripts/common.Makefile
+
+
+.PHONY: install-dev install-prod
+install-dev install-prod: _check_venv_active ## install app in development/production or CI mode
+	# installing in $(subst install-,,$@) mode
+	python -m pip install $(if $(findstring dev,$@),--editable,) .
+
+
+.PHONY: tests
+tests: ## runs unit tests
+	# running unit tests
+	@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests

packages/pytest-simcore/README.md

@@ -0,0 +1,22 @@
+# pytest-simcore plugin
+
+pytest plugin with fixtures and test helpers for osparc-simcore repo modules
+
+## Installation
+
+To install in a modules (e.g. web/server):
+
+- add relative path in ``module/requirements/dev.txt`` and ``module/requirements/ci.txt`` to install for testing
+- in the code, activate different fixtures by declaring the different submodules
+- helper functions are imported as in a normal module
+
+
+```python
+from pytest_simcore.helpers.utils_assert import foo
+
+pytest_plugins = ["pytest_simcore.environs"]
+
+
+def test_something( some_pytest_simcore_fixture, ... )
+    ...
+```

packages/pytest-simcore/setup.py

@@ -0,0 +1,37 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="pytest-simcore",
+    version="0.1.0",
+    maintainer="pcrespov, sanderegg",
+    description="pytest plugin with fixtures and test helpers for osparc-simcore repo modules",
+    py_modules=["pytest_simcore"],
+    python_requires=">=3.6.*",
+    # TODO create partial extensions:
+    install_requires=["pytest>=3.5.0"],
+    extras_require={
+        "all": [
+            "aio-pika",
+            "aiohttp",
+            "aioredis",
+            "celery",
+            "docker",
+            "python-socketio",
+            "PyYAML",
+            "sqlalchemy[postgresql_psycopg2binary]",
+            "tenacity",
+            "yarl",
+        ],
+    },
+    packages=find_packages(where="src"),
+    package_dir={"": "src"},
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Framework :: Pytest",
+        "Intended Audience :: Developers",
+        "Topic :: Software Development :: Testing",
+        "Operating System :: OS Independent",
+        "License :: OSI Approved :: MIT License",
+    ],
+    entry_points={"pytest11": ["simcore = pytest_simcore"]},
+)

packages/pytest-simcore/src/pytest_simcore/__init__.py

@@ -0,0 +1,13 @@
+# Collection of tests fixtures for integration testing
+
+def pytest_addoption(parser):
+    group = parser.getgroup("simcore")
+    group.addoption(
+        "--keep-docker-up",
+        action="store_true",
+        default=False,
+        help="Keep stack/registry up after fixtures closes",
+    )
+
+    # DUMMY
+    parser.addini('HELLO', 'Dummy pytest.ini setting')

packages/pytest-simcore/src/pytest_simcore/celery_service.py

@@ -0,0 +1,45 @@
+# pylint:disable=unused-variable
+# pylint:disable=unused-argument
+# pylint:disable=redefined-outer-name
+
+from typing import Dict
+
+import celery
+import celery.bin.base
+import celery.bin.celery
+import celery.platforms
+import pytest
+import tenacity
+
+from servicelib.celery_utils import CeleryRetryPolicyUponInitialization
+
+from .helpers.utils_docker import get_service_published_port
+
+
+@pytest.fixture(scope="module")
+def celery_config(docker_stack: Dict, devel_environ: Dict) -> Dict:
+    assert "simcore_rabbit" in docker_stack["services"]
+
+    config = {
+        "host": "127.0.0.1",
+        "port": get_service_published_port("rabbit", devel_environ["RABBIT_PORT"]),
+        "user": devel_environ["RABBIT_USER"],
+        "password": devel_environ["RABBIT_PASSWORD"],
+    }
+    yield config
+
+
+@pytest.fixture(scope="module")
+def celery_service(celery_config: Dict, docker_stack: Dict) -> str:
+    url = "amqp://{user}:{password}@{host}:{port}".format(**celery_config)
+    wait_till_celery_responsive(url)
+    yield url
+
+
+@tenacity.retry(**CeleryRetryPolicyUponInitialization().kwargs)
+def wait_till_celery_responsive(url: str) -> None:
+    app = celery.Celery("tasks", broker=url)
+
+    status = celery.bin.celery.CeleryCommand.commands["status"]()
+    status.app = status.get_app()
+    status.run()  # raises celery.bin.base.Error if cannot run

services/web/server/tests/integration/fixtures/docker_compose.py → packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -1,8 +1,6 @@
-"""
-
-    Main Makefile produces a set of docker-compose configuration files
-    Here we can find fixtures of most of these configurations
+""" Fixtures to create docker-compose.yaml configururation files (as in Makefile)
 
+    Basically runs `docker-compose config
 """
 # pylint:disable=unused-variable
 # pylint:disable=unused-argument
@@ -19,24 +17,25 @@
 
 import pytest
 import yaml
-from utils_docker import run_docker_compose_config
+
+from .helpers.utils_docker import run_docker_compose_config
 
 
 @pytest.fixture("session")
 def devel_environ(env_devel_file: Path) -> Dict[str, str]:
-    """ Loads and extends .env-devel
-
+    """ Loads and extends .env-devel returning
+        all environment variables key=value
     """
-    PATTERN_ENVIRON_EQUAL = re.compile(r"^(\w+)=(.*)$")
+    key_eq_value_pattern = re.compile(r"^(\w+)=(.*)$")
     env_devel = {}
-    with env_devel_file.open() as f:
-        for line in f:
-            m = PATTERN_ENVIRON_EQUAL.match(line)
-            if m:
-                key, value = m.groups()
+    with env_devel_file.open() as fh:
+        for line in fh:
+            match = key_eq_value_pattern.match(line)
+            if match:
+                key, value = match.groups()
                 env_devel[key] = str(value)
 
-    # Customized EXTENSION: change some of the environ to accomodate the test case ----
+    # Customized EXTENSION: overrides some of the environ to accomodate the test case ----
     if "REGISTRY_SSL" in env_devel:
         env_devel["REGISTRY_SSL"] = "False"
     if "REGISTRY_URL" in env_devel:
@@ -54,14 +53,6 @@ def devel_environ(env_devel_file: Path) -> Dict[str, str]:
     return env_devel
 
 
-@pytest.fixture(scope="module")
-def temp_folder(request, tmpdir_factory) -> Path:
-    tmp = Path(
-        tmpdir_factory.mktemp("docker_compose_{}".format(request.module.__name__))
-    )
-    yield tmp
-
-
 @pytest.fixture(scope="module")
 def env_file(osparc_simcore_root_dir: Path, devel_environ: Dict[str, str]) -> Path:
     """
@@ -179,6 +170,15 @@ def ops_services_config_file(request, temp_folder, ops_docker_compose):
 
 
 # HELPERS ---------------------------------------------
+def _minio_fix(service_environs: Dict) -> Dict:
+    """this hack ensures that S3 is accessed from the host at all time, thus pre-signed links work.
+        172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface.
+    """
+    if "S3_ENDPOINT" in service_environs:
+        service_environs["S3_ENDPOINT"] = "172.17.0.1:9001"
+    return service_environs
+
+
 def _get_ip() -> str:
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     try:
@@ -207,6 +207,8 @@ def _filter_services_and_dump(
         # removes builds (No more)
         if "build" in service:
             service.pop("build", None)
+        if "environment" in service:
+            service["environment"] = _minio_fix(service["environment"])
 
     # updates current docker-compose (also versioned ... do not change by hand)
     with docker_compose_path.open("wt") as fh:

packages/pytest-simcore/src/pytest_simcore/docker_registry.py

@@ -0,0 +1,137 @@
+# pylint:disable=unused-variable
+# pylint:disable=unused-argument
+# pylint:disable=redefined-outer-name
+
+import json
+import os
+import time
+from typing import Dict
+
+import docker
+import pytest
+import tenacity
+
+
+@pytest.fixture(scope="session")
+def docker_registry(keep_docker_up: bool) -> str:
+    # run the registry outside of the stack
+    docker_client = docker.from_env()
+    # try to login to private registry
+    host = "127.0.0.1"
+    port = 5000
+    url = "{host}:{port}".format(host=host, port=port)
+    container = None
+    try:
+        docker_client.login(registry=url, username="simcore")
+        container = docker_client.containers.list({"name": "pytest_registry"})[0]
+    except Exception: # pylint: disable=broad-except
+        print("Warning: docker registry is already up!")
+        container = docker_client.containers.run(
+            "registry:2",
+            ports={"5000": "5000"},
+            name="pytest_registry",
+            environment=["REGISTRY_STORAGE_DELETE_ENABLED=true"],
+            restart_policy={"Name": "always"},
+            detach=True,
+        )
+
+        # Wait until we can connect
+        assert _wait_till_registry_is_responsive(url)
+
+    # get the hello world example from docker hub
+    hello_world_image = docker_client.images.pull("hello-world", "latest")
+    # login to private registry
+    docker_client.login(registry=url, username="simcore")
+    # tag the image
+    repo = url + "/hello-world:dev"
+    assert hello_world_image.tag(repo) == True
+    # push the image to the private registry
+    docker_client.images.push(repo)
+    # wipe the images
+    docker_client.images.remove(image="hello-world:latest")
+    docker_client.images.remove(image=hello_world_image.id)
+    # pull the image from the private registry
+    private_image = docker_client.images.pull(repo)
+    docker_client.images.remove(image=private_image.id)
+
+    # necessary for old school configs
+    os.environ["REGISTRY_URL"] = url
+    os.environ["REGISTRY_USER"] = "simcore"
+    os.environ["REGISTRY_PW"] = ""
+
+    yield url
+
+    if not keep_docker_up:
+        container.stop()
+
+        while docker_client.containers.list(filters={"name": container.name}):
+            time.sleep(1)
+
+
+@tenacity.retry(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_delay(60))
+def _wait_till_registry_is_responsive(url: str) -> bool:
+    docker_client = docker.from_env()
+    docker_client.login(registry=url, username="simcore")
+    return True
+
+
+# pull from itisfoundation/sleeper and push into local registry
+@pytest.fixture(scope="session")
+def sleeper_service(docker_registry: str) -> Dict[str, str]:
+    """ Adds a itisfoundation/sleeper in docker registry
+
+    """
+    client = docker.from_env()
+    TAG = "1.0.0"
+    image = client.images.pull("itisfoundation/sleeper", tag=TAG)
+    assert not image is None
+    repo = f"{docker_registry}/simcore/services/comp/itis/sleeper:{TAG}"
+    assert image.tag(repo) == True
+    client.images.push(repo)
+    image = client.images.pull(repo)
+    assert image
+    image_labels = image.labels
+
+    yield {
+        "schema": {
+            key[len("io.simcore.") :]: json.loads(value)[key[len("io.simcore.") :]]
+            for key, value in image_labels.items()
+            if key.startswith("io.simcore.")
+        },
+        "image": repo,
+    }
+
+
+@pytest.fixture(scope="session")
+def jupyter_service(docker_registry: str) -> Dict[str, str]:
+    """ Adds a itisfoundation/jupyter-base-notebook in docker registry
+
+    """
+    client = docker.from_env()
+
+    # TODO: cleanup
+
+    # pull from dockerhub
+    reponame, tag = "itisfoundation/jupyter-base-notebook:2.13.0".split(":")
+    image = client.images.pull(reponame, tag=tag)
+    assert not image is None
+
+    # push to fixture registry (services/{dynamic|comp})
+    image_name = reponame.split("/")[-1]
+    repo = f"{docker_registry}/simcore/services/dynamic/{image_name}:{tag}"
+    assert image.tag(repo) == True
+    client.images.push(repo)
+
+    # check
+    image = client.images.pull(repo)
+    assert image
+    image_labels = image.labels
+
+    yield {
+        "schema": {
+            key[len("io.simcore.") :]: json.loads(value)[key[len("io.simcore.") :]]
+            for key, value in image_labels.items()
+            if key.startswith("io.simcore.")
+        },
+        "image": repo,
+    }