diff --git a/.codeclimate.yml b/.codeclimate.yml index 17c4fd2cfed..d9bd3b34903 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -77,3 +77,4 @@ exclude_patterns: - "**/generated_code/" - "**/migrations/" - "**/*.js" + - "**/pytest-simcore/" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f90bb41b57d..36f37eb20fe 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,6 +8,7 @@ Makefile @pcrespov, @sanderegg /api/ @sanderegg, @pcrespov /ci/ @sanderegg, @pcrespov /docs/ @pcrespov +/packages/pytest-simcore @pcrespov, @sanderegg /packages/service-library @pcrespov /scripts/demo @odeimaiz, @pcrespov /scripts/json-schema-to-openapi-schema @sanderegg diff --git a/packages/pytest-simcore/Makefile b/packages/pytest-simcore/Makefile new file mode 100644 index 00000000000..325f67629f7 --- /dev/null +++ b/packages/pytest-simcore/Makefile @@ -0,0 +1,16 @@ +# +# Targets for DEVELOPMENT of Service Library +# +include ../../scripts/common.Makefile + + +.PHONY: install-dev install-prod +install-dev install-prod: _check_venv_active ## install app in development/production or CI mode + # installing in $(subst install-,,$@) mode + python -m pip install $(if $(findstring dev,$@),--editable,) . + + +.PHONY: tests +tests: ## runs unit tests + # running unit tests + @pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests diff --git a/packages/pytest-simcore/README.md b/packages/pytest-simcore/README.md new file mode 100644 index 00000000000..3d0e39003c2 --- /dev/null +++ b/packages/pytest-simcore/README.md @@ -0,0 +1,22 @@ +# pytest-simcore plugin + +pytest plugin with fixtures and test helpers for osparc-simcore repo modules + +## Installation + +To install in a modules (e.g. web/server): + +- add relative path in ``module/requirements/dev.txt`` and ``module/requirements/ci.txt`` to install for testing +- in the code, activate different fixtures by declaring the different submodules +- helper functions are imported as in a normal module + + +```python +from pytest_simcore.helpers.utils_assert import foo + +pytest_plugins = ["pytest_simcore.environs"] + + +def test_something( some_pytest_simcore_fixture, ... ) + ... +``` diff --git a/packages/pytest-simcore/setup.py b/packages/pytest-simcore/setup.py new file mode 100644 index 00000000000..62106d9188f --- /dev/null +++ b/packages/pytest-simcore/setup.py @@ -0,0 +1,37 @@ +from setuptools import setup, find_packages + +setup( + name="pytest-simcore", + version="0.1.0", + maintainer="pcrespov, sanderegg", + description="pytest plugin with fixtures and test helpers for osparc-simcore repo modules", + py_modules=["pytest_simcore"], + python_requires=">=3.6.*", + # TODO create partial extensions: + install_requires=["pytest>=3.5.0"], + extras_require={ + "all": [ + "aio-pika", + "aiohttp", + "aioredis", + "celery", + "docker", + "python-socketio", + "PyYAML", + "sqlalchemy[postgresql_psycopg2binary]", + "tenacity", + "yarl", + ], + }, + packages=find_packages(where="src"), + package_dir={"": "src"}, + classifiers=[ + "Development Status :: 4 - Beta", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Topic :: Software Development :: Testing", + "Operating System :: OS Independent", + "License :: OSI Approved :: MIT License", + ], + entry_points={"pytest11": ["simcore = pytest_simcore"]}, +) diff --git a/packages/pytest-simcore/src/pytest_simcore/__init__.py b/packages/pytest-simcore/src/pytest_simcore/__init__.py new file mode 100644 index 00000000000..1018744d838 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/__init__.py @@ -0,0 +1,13 @@ +# Collection of tests fixtures for integration testing + +def pytest_addoption(parser): + group = parser.getgroup("simcore") + group.addoption( + "--keep-docker-up", + action="store_true", + default=False, + help="Keep stack/registry up after fixtures closes", + ) + + # DUMMY + parser.addini('HELLO', 'Dummy pytest.ini setting') diff --git a/packages/pytest-simcore/src/pytest_simcore/celery_service.py b/packages/pytest-simcore/src/pytest_simcore/celery_service.py new file mode 100644 index 00000000000..b6f73c4d0f3 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/celery_service.py @@ -0,0 +1,45 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +from typing import Dict + +import celery +import celery.bin.base +import celery.bin.celery +import celery.platforms +import pytest +import tenacity + +from servicelib.celery_utils import CeleryRetryPolicyUponInitialization + +from .helpers.utils_docker import get_service_published_port + + +@pytest.fixture(scope="module") +def celery_config(docker_stack: Dict, devel_environ: Dict) -> Dict: + assert "simcore_rabbit" in docker_stack["services"] + + config = { + "host": "127.0.0.1", + "port": get_service_published_port("rabbit", devel_environ["RABBIT_PORT"]), + "user": devel_environ["RABBIT_USER"], + "password": devel_environ["RABBIT_PASSWORD"], + } + yield config + + +@pytest.fixture(scope="module") +def celery_service(celery_config: Dict, docker_stack: Dict) -> str: + url = "amqp://{user}:{password}@{host}:{port}".format(**celery_config) + wait_till_celery_responsive(url) + yield url + + +@tenacity.retry(**CeleryRetryPolicyUponInitialization().kwargs) +def wait_till_celery_responsive(url: str) -> None: + app = celery.Celery("tasks", broker=url) + + status = celery.bin.celery.CeleryCommand.commands["status"]() + status.app = status.get_app() + status.run() # raises celery.bin.base.Error if cannot run diff --git a/services/web/server/tests/integration/fixtures/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py similarity index 82% rename from services/web/server/tests/integration/fixtures/docker_compose.py rename to packages/pytest-simcore/src/pytest_simcore/docker_compose.py index 5555c919a7e..7a3b0d35e72 100644 --- a/services/web/server/tests/integration/fixtures/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -1,8 +1,6 @@ -""" - - Main Makefile produces a set of docker-compose configuration files - Here we can find fixtures of most of these configurations +""" Fixtures to create docker-compose.yaml configururation files (as in Makefile) + Basically runs `docker-compose config """ # pylint:disable=unused-variable # pylint:disable=unused-argument @@ -19,24 +17,25 @@ import pytest import yaml -from utils_docker import run_docker_compose_config + +from .helpers.utils_docker import run_docker_compose_config @pytest.fixture("session") def devel_environ(env_devel_file: Path) -> Dict[str, str]: - """ Loads and extends .env-devel - + """ Loads and extends .env-devel returning + all environment variables key=value """ - PATTERN_ENVIRON_EQUAL = re.compile(r"^(\w+)=(.*)$") + key_eq_value_pattern = re.compile(r"^(\w+)=(.*)$") env_devel = {} - with env_devel_file.open() as f: - for line in f: - m = PATTERN_ENVIRON_EQUAL.match(line) - if m: - key, value = m.groups() + with env_devel_file.open() as fh: + for line in fh: + match = key_eq_value_pattern.match(line) + if match: + key, value = match.groups() env_devel[key] = str(value) - # Customized EXTENSION: change some of the environ to accomodate the test case ---- + # Customized EXTENSION: overrides some of the environ to accomodate the test case ---- if "REGISTRY_SSL" in env_devel: env_devel["REGISTRY_SSL"] = "False" if "REGISTRY_URL" in env_devel: @@ -54,14 +53,6 @@ def devel_environ(env_devel_file: Path) -> Dict[str, str]: return env_devel -@pytest.fixture(scope="module") -def temp_folder(request, tmpdir_factory) -> Path: - tmp = Path( - tmpdir_factory.mktemp("docker_compose_{}".format(request.module.__name__)) - ) - yield tmp - - @pytest.fixture(scope="module") def env_file(osparc_simcore_root_dir: Path, devel_environ: Dict[str, str]) -> Path: """ @@ -179,6 +170,15 @@ def ops_services_config_file(request, temp_folder, ops_docker_compose): # HELPERS --------------------------------------------- +def _minio_fix(service_environs: Dict) -> Dict: + """this hack ensures that S3 is accessed from the host at all time, thus pre-signed links work. + 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface. + """ + if "S3_ENDPOINT" in service_environs: + service_environs["S3_ENDPOINT"] = "172.17.0.1:9001" + return service_environs + + def _get_ip() -> str: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: @@ -207,6 +207,8 @@ def _filter_services_and_dump( # removes builds (No more) if "build" in service: service.pop("build", None) + if "environment" in service: + service["environment"] = _minio_fix(service["environment"]) # updates current docker-compose (also versioned ... do not change by hand) with docker_compose_path.open("wt") as fh: diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py new file mode 100644 index 00000000000..24e7249e07f --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py @@ -0,0 +1,137 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import json +import os +import time +from typing import Dict + +import docker +import pytest +import tenacity + + +@pytest.fixture(scope="session") +def docker_registry(keep_docker_up: bool) -> str: + # run the registry outside of the stack + docker_client = docker.from_env() + # try to login to private registry + host = "127.0.0.1" + port = 5000 + url = "{host}:{port}".format(host=host, port=port) + container = None + try: + docker_client.login(registry=url, username="simcore") + container = docker_client.containers.list({"name": "pytest_registry"})[0] + except Exception: # pylint: disable=broad-except + print("Warning: docker registry is already up!") + container = docker_client.containers.run( + "registry:2", + ports={"5000": "5000"}, + name="pytest_registry", + environment=["REGISTRY_STORAGE_DELETE_ENABLED=true"], + restart_policy={"Name": "always"}, + detach=True, + ) + + # Wait until we can connect + assert _wait_till_registry_is_responsive(url) + + # get the hello world example from docker hub + hello_world_image = docker_client.images.pull("hello-world", "latest") + # login to private registry + docker_client.login(registry=url, username="simcore") + # tag the image + repo = url + "/hello-world:dev" + assert hello_world_image.tag(repo) == True + # push the image to the private registry + docker_client.images.push(repo) + # wipe the images + docker_client.images.remove(image="hello-world:latest") + docker_client.images.remove(image=hello_world_image.id) + # pull the image from the private registry + private_image = docker_client.images.pull(repo) + docker_client.images.remove(image=private_image.id) + + # necessary for old school configs + os.environ["REGISTRY_URL"] = url + os.environ["REGISTRY_USER"] = "simcore" + os.environ["REGISTRY_PW"] = "" + + yield url + + if not keep_docker_up: + container.stop() + + while docker_client.containers.list(filters={"name": container.name}): + time.sleep(1) + + +@tenacity.retry(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_delay(60)) +def _wait_till_registry_is_responsive(url: str) -> bool: + docker_client = docker.from_env() + docker_client.login(registry=url, username="simcore") + return True + + +# pull from itisfoundation/sleeper and push into local registry +@pytest.fixture(scope="session") +def sleeper_service(docker_registry: str) -> Dict[str, str]: + """ Adds a itisfoundation/sleeper in docker registry + + """ + client = docker.from_env() + TAG = "1.0.0" + image = client.images.pull("itisfoundation/sleeper", tag=TAG) + assert not image is None + repo = f"{docker_registry}/simcore/services/comp/itis/sleeper:{TAG}" + assert image.tag(repo) == True + client.images.push(repo) + image = client.images.pull(repo) + assert image + image_labels = image.labels + + yield { + "schema": { + key[len("io.simcore.") :]: json.loads(value)[key[len("io.simcore.") :]] + for key, value in image_labels.items() + if key.startswith("io.simcore.") + }, + "image": repo, + } + + +@pytest.fixture(scope="session") +def jupyter_service(docker_registry: str) -> Dict[str, str]: + """ Adds a itisfoundation/jupyter-base-notebook in docker registry + + """ + client = docker.from_env() + + # TODO: cleanup + + # pull from dockerhub + reponame, tag = "itisfoundation/jupyter-base-notebook:2.13.0".split(":") + image = client.images.pull(reponame, tag=tag) + assert not image is None + + # push to fixture registry (services/{dynamic|comp}) + image_name = reponame.split("/")[-1] + repo = f"{docker_registry}/simcore/services/dynamic/{image_name}:{tag}" + assert image.tag(repo) == True + client.images.push(repo) + + # check + image = client.images.pull(repo) + assert image + image_labels = image.labels + + yield { + "schema": { + key[len("io.simcore.") :]: json.loads(value)[key[len("io.simcore.") :]] + for key, value in image_labels.items() + if key.startswith("io.simcore.") + }, + "image": repo, + } diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py new file mode 100644 index 00000000000..1326ba64725 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py @@ -0,0 +1,124 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import subprocess +import time +from pathlib import Path +from pprint import pprint +from typing import Dict + +import docker +import pytest +import tenacity +from servicelib.simcore_service_utils import SimcoreRetryPolicyUponInitialization + + +@pytest.fixture(scope="session") +def docker_client() -> docker.client.DockerClient: + client = docker.from_env() + yield client + + +@pytest.fixture(scope="module") +def docker_swarm(docker_client: docker.client.DockerClient) -> None: + try: + docker_client.swarm.reload() + print("CAUTION: Already part of a swarm") + yield + except docker.errors.APIError: + docker_client.swarm.init() + yield + assert docker_client.swarm.leave(force=True) + + +@pytest.fixture(scope="session") +def keep_docker_up(request) -> bool: + return request.config.getoption("--keep-docker-up") == True + + +@tenacity.retry(**SimcoreRetryPolicyUponInitialization().kwargs) +def _wait_for_services(docker_client: docker.client.DockerClient) -> None: + pre_states = ["NEW", "PENDING", "ASSIGNED", "PREPARING", "STARTING"] + services = docker_client.services.list() + for service in services: + print(f"Waiting for {service.name}...") + if service.tasks(): + task = service.tasks()[0] + if task["Status"]["State"].upper() not in pre_states: + if not task["Status"]["State"].upper() == "RUNNING": + raise Exception(f"service {service} not running") + + +def _print_services(docker_client: docker.client.DockerClient, msg: str) -> None: + print("{:*^100}".format("docker services running " + msg)) + for service in docker_client.services.list(): + pprint(service.attrs) + print("-" * 100) + + +@pytest.fixture(scope="module") +def docker_stack( + docker_swarm, + docker_client: docker.client.DockerClient, + core_services_config_file: Path, + ops_services_config_file: Path, + keep_docker_up: bool, +) -> Dict: + stacks = {"simcore": core_services_config_file, "ops": ops_services_config_file} + + # make up-version + stacks_up = [] + for stack_name, stack_config_file in stacks.items(): + subprocess.run( + f"docker stack deploy -c {stack_config_file.name} {stack_name}", + shell=True, + check=True, + cwd=stack_config_file.parent, + ) + stacks_up.append(stack_name) + + _wait_for_services(docker_client) + _print_services(docker_client, "[BEFORE TEST]") + + yield { + "stacks": stacks_up, + "services": [service.name for service in docker_client.services.list()], + } + + _print_services(docker_client, "[AFTER TEST]") + + if keep_docker_up: + # skip bringing the stack down + return + + # clean up. Guarantees that all services are down before creating a new stack! + # + # WORKAROUND https://github.com/moby/moby/issues/30942#issue-207070098 + # + # docker stack rm services + # until [ -z "$(docker service ls --filter label=com.docker.stack.namespace=services -q)" ] || [ "$limit" -lt 0 ]; do + # sleep 1; + # done + # until [ -z "$(docker network ls --filter label=com.docker.stack.namespace=services -q)" ] || [ "$limit" -lt 0 ]; do + # sleep 1; + # done + + # make down + # NOTE: remove them in reverse order since stacks share common networks + WAIT_BEFORE_RETRY_SECS = 1 + stacks_up.reverse() + for stack in stacks_up: + subprocess.run(f"docker stack rm {stack}", shell=True, check=True) + + while docker_client.services.list( + filters={"label": f"com.docker.stack.namespace={stack}"} + ): + time.sleep(WAIT_BEFORE_RETRY_SECS) + + while docker_client.networks.list( + filters={"label": f"com.docker.stack.namespace={stack}"} + ): + time.sleep(WAIT_BEFORE_RETRY_SECS) + + _print_services(docker_client, "[AFTER REMOVED]") diff --git a/packages/pytest-simcore/src/pytest_simcore/environs.py b/packages/pytest-simcore/src/pytest_simcore/environs.py new file mode 100644 index 00000000000..43bafaa39a1 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/environs.py @@ -0,0 +1,47 @@ +# pylint: disable=redefined-outer-name + +import sys +from pathlib import Path + +import pytest + +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + + +@pytest.fixture(scope="session") +def osparc_simcore_root_dir(request) -> Path: + """ osparc-simcore repo root dir """ + WILDCARD = "packages/pytest-simcore/src/pytest_simcore/environs.py" + ROOT = Path("/") + + test_dir = Path(request.session.fspath) # expected test dir in simcore + + for start_dir in (current_dir, test_dir): + root_dir = start_dir + while not any(root_dir.glob(WILDCARD)) and root_dir != ROOT: + root_dir = root_dir.parent + + if root_dir!=ROOT: + break + + msg = f"'{root_dir}' does not look like the git root directory of osparc-simcore" + + assert root_dir != ROOT, msg + assert root_dir.exists(), msg + assert any(root_dir.glob(WILDCARD)), msg + assert any(root_dir.glob(".git")), msg + + return root_dir + + +@pytest.fixture(scope="session") +def env_devel_file(osparc_simcore_root_dir) -> Path: + env_devel_fpath = osparc_simcore_root_dir / ".env-devel" + assert env_devel_fpath.exists() + return env_devel_fpath + + +@pytest.fixture(scope="module") +def temp_folder(request, tmpdir_factory) -> Path: + tmp = Path(tmpdir_factory.mktemp(f"tmp_module_{request.module.__name__}")) + yield tmp diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/__init__.py b/packages/pytest-simcore/src/pytest_simcore/helpers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/web/server/tests/helpers/utils_assert.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_assert.py similarity index 88% rename from services/web/server/tests/helpers/utils_assert.py rename to packages/pytest-simcore/src/pytest_simcore/helpers/utils_assert.py index c09d3aea5a5..6286300ad72 100644 --- a/services/web/server/tests/helpers/utils_assert.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_assert.py @@ -1,3 +1,6 @@ +""" Extends assertions for testing + +""" from aiohttp import web from pprint import pformat @@ -17,7 +20,8 @@ async def assert_status( assert not data, pformat(data) assert not error, pformat(error) else: - # with a 200, data may still be empty see https://medium.com/@santhoshkumarkrishna/http-get-rest-api-no-content-404-vs-204-vs-200-6dd869e3af1d + # with a 200, data may still be empty see + # https://medium.com/@santhoshkumarkrishna/http-get-rest-api-no-content-404-vs-204-vs-200-6dd869e3af1d # assert data is not None, pformat(data) assert not error, pformat(error) @@ -40,7 +44,6 @@ def do_assert_error( assert not data, pformat(data) assert error, pformat(error) - # TODO: improve error messages assert len(error["errors"]) == 1 err = error["errors"][0] diff --git a/services/web/server/tests/helpers/utils_docker.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py similarity index 92% rename from services/web/server/tests/helpers/utils_docker.py rename to packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py index 9caf8c9d492..0818d869d75 100644 --- a/services/web/server/tests/helpers/utils_docker.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py @@ -18,22 +18,24 @@ def get_service_published_port( service_name: str, target_port: Optional[int] = None ) -> str: - """ - WARNING: ENSURE that service name exposes a port in Dockerfile file or docker-compose config file - """ + # WARNING: ENSURE that service name exposes a port in + # Dockerfile file or docker-compose config file + # NOTE: retries since services can take some time to start client = docker.from_env() services = [x for x in client.services.list() if service_name in x.name] if not services: raise RuntimeError( - f"Cannot find published port for service '{service_name}'. Probably services still not started." + f"Cannot find published port for service '{service_name}'." + "Probably services still not started." ) service_ports = services[0].attrs["Endpoint"].get("Ports") if not service_ports: raise RuntimeError( - f"Cannot find published port for service '{service_name}' in endpoint. Probably services still not started." + f"Cannot find published port for service '{service_name}' in endpoint." + "Probably services still not started." ) published_port = None diff --git a/services/web/server/tests/helpers/utils_environs.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_environs.py similarity index 98% rename from services/web/server/tests/helpers/utils_environs.py rename to packages/pytest-simcore/src/pytest_simcore/helpers/utils_environs.py index f02d4b1822e..3823aee42d5 100644 --- a/services/web/server/tests/helpers/utils_environs.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_environs.py @@ -4,7 +4,7 @@ import re from copy import deepcopy from pathlib import Path -from typing import Dict +from typing import Dict, List import yaml @@ -49,9 +49,6 @@ def eval_environs_in_docker_compose( return content -from typing import List - - def replace_environs_in_docker_compose_service( service_section: Dict, docker_compose_dir: Path, diff --git a/services/web/server/tests/helpers/utils_login.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_login.py similarity index 97% rename from services/web/server/tests/helpers/utils_login.py rename to packages/pytest-simcore/src/pytest_simcore/helpers/utils_login.py index 1b42e940875..1d21eb93553 100644 --- a/services/web/server/tests/helpers/utils_login.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_login.py @@ -7,7 +7,8 @@ from simcore_service_webserver.login.cfg import cfg, get_storage from simcore_service_webserver.login.registration import create_invitation from simcore_service_webserver.login.utils import encrypt_password, get_random_string -from utils_assert import assert_status + +from .utils_assert import assert_status TEST_MARKS = re.compile(r"TEST (\w+):(.*)") @@ -63,9 +64,6 @@ async def log_client_in(client, user_data=None, *, enable_check=True): return user -# CONTEXT MANAGERS ------------------------------ - - class NewUser: def __init__(self, params=None, app: web.Application = None): self.params = params diff --git a/services/web/server/tests/helpers/utils_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_projects.py similarity index 100% rename from services/web/server/tests/helpers/utils_projects.py rename to packages/pytest-simcore/src/pytest_simcore/helpers/utils_projects.py diff --git a/services/web/server/tests/helpers/utils_tokens.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_tokens.py similarity index 100% rename from services/web/server/tests/helpers/utils_tokens.py rename to packages/pytest-simcore/src/pytest_simcore/helpers/utils_tokens.py diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py new file mode 100644 index 00000000000..60f74adf700 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -0,0 +1,61 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import os +from distutils.util import strtobool +from typing import Dict + +import pytest +import tenacity + +from s3wrapper.s3_client import S3Client +from servicelib.minio_utils import MinioRetryPolicyUponInitialization + +from .helpers.utils_docker import get_service_published_port + + +@pytest.fixture(scope="module") +def minio_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: + assert "ops_minio" in docker_stack["services"] + + # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a + # container onto the host network interface. + config = { + "client": { + "endpoint": f"172.17.0.1:{get_service_published_port('minio', devel_environ['S3_ENDPOINT'].split(':')[1])}", + "access_key": devel_environ["S3_ACCESS_KEY"], + "secret_key": devel_environ["S3_SECRET_KEY"], + "secure": strtobool(devel_environ["S3_SECURE"]) != 0, + }, + "bucket_name": devel_environ["S3_BUCKET_NAME"], + } + + # nodeports takes its configuration from env variables + for key, value in config["client"].items(): + os.environ[f"S3_{key.upper()}"] = str(value) + os.environ[f"S3_BUCKET_NAME"] = devel_environ["S3_BUCKET_NAME"] + + return config + + +@pytest.fixture(scope="module") +def minio_service(minio_config: Dict[str, str], docker_stack: Dict) -> S3Client: + assert wait_till_minio_responsive(minio_config) + + client = S3Client(**minio_config["client"]) + assert client.create_bucket(minio_config["bucket_name"]) + + yield client + + assert client.remove_bucket(minio_config["bucket_name"], delete_contents=True) + + +@tenacity.retry(**MinioRetryPolicyUponInitialization().kwargs) +def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool: + """Check if something responds to ``url`` """ + client = S3Client(**minio_config["client"]) + if client.create_bucket("pytest"): + client.remove_bucket("pytest") + return True + raise Exception(f"Minio not responding to {minio_config}") diff --git a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py new file mode 100644 index 00000000000..31adbb47df3 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py @@ -0,0 +1,63 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import os +from typing import Dict + +import pytest +import sqlalchemy as sa +from sqlalchemy.orm import sessionmaker +from tenacity import Retrying + +from servicelib.aiopg_utils import DSN, PostgresRetryPolicyUponInitialization +from simcore_postgres_database.models.base import metadata + +from .helpers.utils_docker import get_service_published_port + + +@pytest.fixture(scope="module") +def postgres_dsn(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: + assert "simcore_postgres" in docker_stack["services"] + + pg_config = { + "user": devel_environ["POSTGRES_USER"], + "password": devel_environ["POSTGRES_PASSWORD"], + "database": devel_environ["POSTGRES_DB"], + "host": "127.0.0.1", + "port": get_service_published_port("postgres", devel_environ["POSTGRES_PORT"]), + } + # nodeports takes its configuration from env variables + os.environ["POSTGRES_ENDPOINT"] = f"127.0.0.1:{pg_config['port']}" + os.environ["POSTGRES_USER"] = devel_environ["POSTGRES_USER"] + os.environ["POSTGRES_PASSWORD"] = devel_environ["POSTGRES_PASSWORD"] + os.environ["POSTGRES_DB"] = devel_environ["POSTGRES_DB"] + return pg_config + + +@pytest.fixture(scope="module") +def postgres_db(postgres_dsn: Dict[str, str], docker_stack: Dict) -> sa.engine.Engine: + url = DSN.format(**postgres_dsn) + + # Attempts until responsive + for attempt in Retrying(**PostgresRetryPolicyUponInitialization().kwargs): + with attempt: + engine = sa.create_engine(url, isolation_level="AUTOCOMMIT") + conn = engine.connect() + conn.close() + + # Configures db and initializes tables + metadata.create_all(bind=engine, checkfirst=True) + + yield engine + + metadata.drop_all(engine) + engine.dispose() + + +@pytest.fixture(scope="module") +def postgres_session(postgres_db: sa.engine.Engine) -> sa.orm.session.Session: + Session = sessionmaker(postgres_db) + session = Session() + yield session + session.close() diff --git a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py new file mode 100644 index 00000000000..aa1c1cacff6 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py @@ -0,0 +1,50 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import os +from typing import Dict + +import aio_pika +import pytest +import tenacity + +from servicelib.rabbitmq_utils import RabbitMQRetryPolicyUponInitialization + +from .helpers.utils_docker import get_service_published_port + + +@pytest.fixture(scope="module") +def rabbit_config(docker_stack: Dict, devel_environ: Dict) -> Dict: + assert "simcore_rabbit" in docker_stack["services"] + + config = { + "host": "127.0.0.1", + "port": get_service_published_port("rabbit", devel_environ["RABBIT_PORT"]), + "user": devel_environ["RABBIT_USER"], + "password": devel_environ["RABBIT_PASSWORD"], + } + + # sidecar takes its configuration from env variables + os.environ["RABBIT_HOST"] = "127.0.0.1" + os.environ["RABBIT_PORT"] = config["port"] + os.environ["RABBIT_USER"] = devel_environ["RABBIT_USER"] + os.environ["RABBIT_PASSWORD"] = devel_environ["RABBIT_PASSWORD"] + os.environ["RABBIT_PROGRESS_CHANNEL"] = devel_environ["RABBIT_PROGRESS_CHANNEL"] + + yield config + + +@pytest.fixture(scope="function") +async def rabbit_service(rabbit_config: Dict, docker_stack: Dict) -> str: + url = "amqp://{user}:{password}@{host}:{port}".format(**rabbit_config) + wait_till_rabbit_responsive(url) + yield url + + +# HELPERS -- + + +@tenacity.retry(**RabbitMQRetryPolicyUponInitialization().kwargs) +async def wait_till_rabbit_responsive(url: str) -> None: + await aio_pika.connect(url) diff --git a/packages/pytest-simcore/src/pytest_simcore/redis_service.py b/packages/pytest-simcore/src/pytest_simcore/redis_service.py new file mode 100644 index 00000000000..8722da306d2 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/redis_service.py @@ -0,0 +1,53 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +from typing import Dict + +import aioredis +import pytest +import tenacity +from yarl import URL + +from servicelib.redis_utils import RedisRetryPolicyUponInitialization + +from .helpers.utils_docker import get_service_published_port + + +@pytest.fixture(scope="module") +def redis_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: + assert "simcore_redis" in docker_stack["services"] + + config = { + "host": "127.0.0.1", + "port": get_service_published_port("redis", devel_environ["REDIS_PORT"]), + } + return config + + +@pytest.fixture(scope="module") +async def redis_service(redis_config: Dict[str, str], docker_stack: Dict) -> URL: + url = URL("redis://{host}:{port}".format(**redis_config)) + await wait_till_redis_responsive(url) + return url + + +@pytest.fixture(scope="module") +async def redis_client(loop, redis_service: URL) -> aioredis.Redis: + client = await aioredis.create_redis_pool(str(redis_service), encoding="utf-8") + + yield client + + await client.flushall() + client.close() + await client.wait_closed() + + +# HELPERS -- + + +@tenacity.retry(**RedisRetryPolicyUponInitialization().kwargs) +async def wait_till_redis_responsive(redis_url: URL) -> None: + client = await aioredis.create_redis_pool(str(redis_url), encoding="utf-8") + client.close() + await client.wait_closed() diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py new file mode 100644 index 00000000000..588cdc524ae --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -0,0 +1,51 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import os +from typing import Dict + +import aiohttp +import pytest +import tenacity +from yarl import URL + +from s3wrapper.s3_client import S3Client +from servicelib.minio_utils import MinioRetryPolicyUponInitialization + +from .helpers.utils_docker import get_service_published_port + + +@pytest.fixture(scope="module") +def storage_endpoint(docker_stack: Dict, devel_environ: Dict) -> URL: + assert "simcore_storage" in docker_stack["services"] + default_port = devel_environ["STORAGE_ENDPOINT"].split(":")[1] + endpoint = f"127.0.0.1:{get_service_published_port('storage', default_port)}" + + # nodeports takes its configuration from env variables + os.environ[f"STORAGE_ENDPOINT"] = endpoint + + return URL(f"http://{endpoint}") + + +@pytest.fixture(scope="function") +async def storage_service( + minio_service: S3Client, storage_endpoint: URL, docker_stack: Dict +) -> URL: + assert await wait_till_storage_responsive(storage_endpoint) + + yield storage_endpoint + + +# HELPERS -- + +# TODO: this can be used by ANY of the simcore services! +@tenacity.retry(**MinioRetryPolicyUponInitialization().kwargs) +async def wait_till_storage_responsive(storage_endpoint: URL): + async with aiohttp.ClientSession() as session: + async with session.get(storage_endpoint.with_path("/v0/")) as resp: + assert resp.status == 200 + data = await resp.json() + assert "data" in data + assert "status" in data["data"] + assert data["data"]["status"] == "SERVICE_RUNNING" diff --git a/services/web/server/tests/integration/fixtures/websocket_client.py b/packages/pytest-simcore/src/pytest_simcore/websocket_client.py similarity index 95% rename from services/web/server/tests/integration/fixtures/websocket_client.py rename to packages/pytest-simcore/src/pytest_simcore/websocket_client.py index 174a6452827..0128d913ca3 100644 --- a/services/web/server/tests/integration/fixtures/websocket_client.py +++ b/packages/pytest-simcore/src/pytest_simcore/websocket_client.py @@ -1,5 +1,3 @@ -# pylint:disable=wildcard-import -# pylint:disable=unused-import # pylint:disable=unused-variable # pylint:disable=unused-argument # pylint:disable=redefined-outer-name diff --git a/packages/pytest-simcore/tests/conftest.py b/packages/pytest-simcore/tests/conftest.py new file mode 100644 index 00000000000..cc55b6b458b --- /dev/null +++ b/packages/pytest-simcore/tests/conftest.py @@ -0,0 +1,15 @@ +# pylint: disable=unused-import + +pytest_plugins = "pytester" + + +try: + import pytest_sugar + + raise Exception( + "Cannot run these tests with this module installed: " + "pip uninstall pytest_sugar" + ) +except ImportError: + # GOOD + pass diff --git a/packages/pytest-simcore/tests/test_fixture_environs.py b/packages/pytest-simcore/tests/test_fixture_environs.py new file mode 100644 index 00000000000..28662d630b8 --- /dev/null +++ b/packages/pytest-simcore/tests/test_fixture_environs.py @@ -0,0 +1,34 @@ +import sys +from pathlib import Path + +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +repo_base_dir = current_dir.parent.parent.parent +assert any(repo_base_dir.glob(".git")) + + +def test_root_dir_with_installed_plugin(testdir): + """ Make sure osparc_simcore_root_dir is correct + even when pytest-simcore installed + """ + + # create a temporary pytest test module + testdir.makepyfile( + f""" + from pathlib import Path + pytest_plugins = ["pytest_simcore.environs"] + + def test_sth(osparc_simcore_root_dir): + assert osparc_simcore_root_dir == Path('{repo_base_dir}') + """ + ) + + result = testdir.runpytest("-v") + # fnmatch_lines does an assertion internally + # WARNING: this does not work with pytest-sugar! + result.stdout.fnmatch_lines( + ["*::test_sth PASSED*",] + ) + + # make sure that that we get a '0' exit code for the testsuite + assert result.ret == 0 diff --git a/packages/pytest-simcore/tests/test_simcore_plugin.py b/packages/pytest-simcore/tests/test_simcore_plugin.py new file mode 100644 index 00000000000..d44af5ef0c0 --- /dev/null +++ b/packages/pytest-simcore/tests/test_simcore_plugin.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + + +def test_using_pytest_simcore_fixture(testdir): + """Make sure that pytest accepts our fixture.""" + + # create a temporary pytest test module + testdir.makepyfile(""" + pytest_plugins = ["pytest_simcore.environs"] + + def test_sth(request): + assert request.config.getoption("--keep-docker-up") == True + """) + + # run pytest with the following cmd args + result = testdir.runpytest( + '--keep-docker-up', + '-v' + ) + + # fnmatch_lines does an assertion internally + # WARNING: this does not work with pytest-sugar! + result.stdout.fnmatch_lines([ + '*::test_sth PASSED*', + ]) + + # make sure that that we get a '0' exit code for the testsuite + assert result.ret == 0 + + +def test_help_message(testdir): + result = testdir.runpytest( + '--help', + ) + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines([ + 'simcore:', + '*--keep-docker-up*Keep stack/registry up after fixtures closes', + ]) + + +def test_hello_ini_setting(testdir): + testdir.makeini(""" + [pytest] + HELLO = world + """) + + testdir.makepyfile(""" + import pytest + + @pytest.fixture + def hello(request): + return request.config.getini('HELLO') + + def test_hello_world(hello): + assert hello == 'world' + """) + + result = testdir.runpytest('-v') + + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines([ + '*::test_hello_world PASSED*', + ]) + + # make sure that that we get a '0' exit code for the testsuite + assert result.ret == 0 diff --git a/packages/s3wrapper/src/s3wrapper/s3_client.py b/packages/s3wrapper/src/s3wrapper/s3_client.py index 981cba36d5e..7f483e92533 100644 --- a/packages/s3wrapper/src/s3wrapper/s3_client.py +++ b/packages/s3wrapper/src/s3wrapper/s3_client.py @@ -1,19 +1,23 @@ import logging - import re from datetime import timedelta -from minio import Minio, CopyConditions +from minio import CopyConditions, Minio from minio.error import ResponseError log = logging.getLogger(__name__) - class S3Client: """ Wrapper around minio """ - def __init__(self, endpoint, access_key=None, secret_key=None, secure=False): + def __init__( + self, + endpoint: str, + access_key: str = None, + secret_key: str = None, + secure: bool = False, + ): self.__metadata_prefix = "x-amz-meta-" self.client = None self.endpoint = endpoint @@ -22,10 +26,9 @@ def __init__(self, endpoint, access_key=None, secret_key=None, secure=False): self.secure = secure self.endpoint_url = ("https://" if secure else "http://") + endpoint try: - self.client = Minio(endpoint, - access_key=access_key, - secret_key=secret_key, - secure=secure) + self.client = Minio( + endpoint, access_key=access_key, secret_key=secret_key, secure=secure + ) except ResponseError as _err: logging.exception("Could not create minio client") @@ -37,7 +40,6 @@ def __remove_objects_recursively(self, bucket_name): self.remove_objects(bucket_name, to_del) - def create_bucket(self, bucket_name, delete_contents_if_exists=False): try: if not self.exists_bucket(bucket_name): @@ -95,9 +97,10 @@ def upload_file(self, bucket_name, object_name, filepath, metadata=None): _metadata = {} if metadata is not None: for key in metadata.keys(): - _metadata[self.__metadata_prefix+key] = metadata[key] - self.client.fput_object(bucket_name, object_name, filepath, - metadata=_metadata) + _metadata[self.__metadata_prefix + key] = metadata[key] + self.client.fput_object( + bucket_name, object_name, filepath, metadata=_metadata + ) except ResponseError as _err: logging.exception("Could not upload file") return False @@ -117,7 +120,7 @@ def get_metadata(self, bucket_name, object_name): _metadata = obj.metadata metadata = {} for key in _metadata.keys(): - _key = key[len(self.__metadata_prefix):] + _key = key[len(self.__metadata_prefix) :] metadata[_key] = _metadata[key] return metadata @@ -128,7 +131,9 @@ def get_metadata(self, bucket_name, object_name): def list_objects(self, bucket_name, prefix=None, recursive=False): try: - return self.client.list_objects(bucket_name, prefix=prefix, recursive=recursive) + return self.client.list_objects( + bucket_name, prefix=prefix, recursive=recursive + ) except ResponseError as _err: logging.exception("Could not list objects") @@ -153,8 +158,8 @@ def remove_objects(self, bucket_name, objects): return True def exists_object(self, bucket_name, object_name, recursive=False): - ''' This seems to be pretty heavy, should be used with care - ''' + """ This seems to be pretty heavy, should be used with care + """ try: objects = self.list_objects(bucket_name, recursive=recursive) for obj in objects: @@ -181,15 +186,18 @@ def search(self, bucket_name, query, recursive=True, include_metadata=False): results.append(obj) for r in results: - msg = "Object {} in bucket {} matches query {}".format(r.object_name, r.bucket_name, query) + msg = "Object {} in bucket {} matches query {}".format( + r.object_name, r.bucket_name, query + ) log.debug(msg) return results def create_presigned_put_url(self, bucket_name, object_name, dt=timedelta(days=3)): try: - return self.client.presigned_put_object(bucket_name, object_name, - expires=dt) + return self.client.presigned_put_object( + bucket_name, object_name, expires=dt + ) except ResponseError as _err: logging.exception("Could create presigned put url") @@ -198,8 +206,9 @@ def create_presigned_put_url(self, bucket_name, object_name, dt=timedelta(days=3 def create_presigned_get_url(self, bucket_name, object_name, dt=timedelta(days=3)): try: - return self.client.presigned_get_object(bucket_name, object_name, - expires=dt) + return self.client.presigned_get_object( + bucket_name, object_name, expires=dt + ) except ResponseError as _err: logging.exception("Could create presigned get url") @@ -208,8 +217,12 @@ def create_presigned_get_url(self, bucket_name, object_name, dt=timedelta(days=3 def copy_object(self, to_bucket_name, to_object_name, from_bucket_object_name): try: - ret = self.client.copy_object(to_bucket_name, to_object_name, - from_bucket_object_name, CopyConditions()) + ret = self.client.copy_object( + to_bucket_name, + to_object_name, + from_bucket_object_name, + CopyConditions(), + ) print(ret) return True except ResponseError as _err: diff --git a/packages/service-library/src/servicelib/celery_utils.py b/packages/service-library/src/servicelib/celery_utils.py new file mode 100644 index 00000000000..22f06270e98 --- /dev/null +++ b/packages/service-library/src/servicelib/celery_utils.py @@ -0,0 +1,24 @@ +import logging +from typing import Optional + +from tenacity import before_sleep_log, stop_after_attempt, wait_fixed + +log = logging.getLogger(__file__) + + +class CeleryRetryPolicyUponInitialization: + """ Retry policy upon service initialization + """ + + WAIT_SECS = 2 + ATTEMPTS_COUNT = 20 + + def __init__(self, logger: Optional[logging.Logger] = None): + logger = logger or log + + self.kwargs = dict( + wait=wait_fixed(self.WAIT_SECS), + stop=stop_after_attempt(self.ATTEMPTS_COUNT), + before_sleep=before_sleep_log(logger, logging.INFO), + reraise=True, + ) diff --git a/packages/service-library/src/servicelib/minio_utils.py b/packages/service-library/src/servicelib/minio_utils.py new file mode 100644 index 00000000000..32ace235d87 --- /dev/null +++ b/packages/service-library/src/servicelib/minio_utils.py @@ -0,0 +1,24 @@ +import logging +from typing import Optional + +from tenacity import before_sleep_log, stop_after_attempt, wait_fixed + +log = logging.getLogger(__name__) + + +class MinioRetryPolicyUponInitialization: + """ Retry policy upon service initialization + """ + + WAIT_SECS = 2 + ATTEMPTS_COUNT = 20 + + def __init__(self, logger: Optional[logging.Logger] = None): + logger = logger or log + + self.kwargs = dict( + wait=wait_fixed(self.WAIT_SECS), + stop=stop_after_attempt(self.ATTEMPTS_COUNT), + before_sleep=before_sleep_log(logger, logging.INFO), + reraise=True, + ) diff --git a/packages/service-library/src/servicelib/rabbitmq_utils.py b/packages/service-library/src/servicelib/rabbitmq_utils.py new file mode 100644 index 00000000000..e0e02151136 --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq_utils.py @@ -0,0 +1,24 @@ +import logging +from typing import Optional + +from tenacity import before_sleep_log, stop_after_attempt, wait_fixed + +log = logging.getLogger(__file__) + + +class RabbitMQRetryPolicyUponInitialization: + """ Retry policy upon service initialization + """ + + WAIT_SECS = 2 + ATTEMPTS_COUNT = 20 + + def __init__(self, logger: Optional[logging.Logger] = None): + logger = logger or log + + self.kwargs = dict( + wait=wait_fixed(self.WAIT_SECS), + stop=stop_after_attempt(self.ATTEMPTS_COUNT), + before_sleep=before_sleep_log(logger, logging.INFO), + reraise=True, + ) diff --git a/packages/service-library/src/servicelib/redis_utils.py b/packages/service-library/src/servicelib/redis_utils.py new file mode 100644 index 00000000000..ef17e454ed2 --- /dev/null +++ b/packages/service-library/src/servicelib/redis_utils.py @@ -0,0 +1,24 @@ +import logging +from typing import Optional + +from tenacity import before_sleep_log, stop_after_attempt, wait_fixed + +log = logging.getLogger(__file__) + + +class RedisRetryPolicyUponInitialization: + """ Retry policy upon service initialization + """ + + WAIT_SECS = 2 + ATTEMPTS_COUNT = 20 + + def __init__(self, logger: Optional[logging.Logger] = None): + logger = logger or log + + self.kwargs = dict( + wait=wait_fixed(self.WAIT_SECS), + stop=stop_after_attempt(self.ATTEMPTS_COUNT), + before_sleep=before_sleep_log(logger, logging.INFO), + reraise=True, + ) diff --git a/packages/service-library/src/servicelib/simcore_service_utils.py b/packages/service-library/src/servicelib/simcore_service_utils.py new file mode 100644 index 00000000000..8f537b930c6 --- /dev/null +++ b/packages/service-library/src/servicelib/simcore_service_utils.py @@ -0,0 +1,24 @@ +import logging +from typing import Optional + +from tenacity import before_sleep_log, stop_after_attempt, wait_fixed + +log = logging.getLogger(__name__) + + +class SimcoreRetryPolicyUponInitialization: + """ Retry policy upon service initialization + """ + + WAIT_SECS = 5 + ATTEMPTS_COUNT = 12 + + def __init__(self, logger: Optional[logging.Logger] = None): + logger = logger or log + + self.kwargs = dict( + wait=wait_fixed(self.WAIT_SECS), + stop=stop_after_attempt(self.ATTEMPTS_COUNT), + before_sleep=before_sleep_log(logger, logging.INFO), + reraise=True, + ) diff --git a/services/web/server/tests/integration/fixtures/docker_registry.py b/services/sidecar/tests/integration/fixtures/docker_registry.py similarity index 89% rename from services/web/server/tests/integration/fixtures/docker_registry.py rename to services/sidecar/tests/integration/fixtures/docker_registry.py index a0d49514f7e..bea2ab0412c 100644 --- a/services/web/server/tests/integration/fixtures/docker_registry.py +++ b/services/sidecar/tests/integration/fixtures/docker_registry.py @@ -9,17 +9,15 @@ import tenacity import time - @pytest.fixture(scope="session") def docker_registry(): # run the registry outside of the stack docker_client = docker.from_env() - container = docker_client.containers.run( - "registry:2", - ports={"5000": "5000"}, + container = docker_client.containers.run("registry:2", + ports={"5000":"5000"}, environment=["REGISTRY_STORAGE_DELETE_ENABLED=true"], - restart_policy={"Name": "always"}, - detach=True, + restart_policy={"Name":"always"}, + detach=True ) host = "127.0.0.1" port = 5000 @@ -30,7 +28,7 @@ def docker_registry(): # test the registry docker_client = docker.from_env() # get the hello world example from docker hub - hello_world_image = docker_client.images.pull("hello-world", "latest") + hello_world_image = docker_client.images.pull("hello-world","latest") # login to private registry docker_client.login(registry=url, username="simcore") # tag the image @@ -52,7 +50,6 @@ def docker_registry(): while docker_client.containers.list(filters={"name": container.name}): time.sleep(1) - @tenacity.retry(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_delay(60)) def _wait_till_registry_is_responsive(url): docker_client = docker.from_env() @@ -60,7 +57,7 @@ def _wait_till_registry_is_responsive(url): return True -# pull from itisfoundation/sleeper and push into local registry +#pull from itisfoundation/sleeper and push into local registry @pytest.fixture(scope="session") def sleeper_service(docker_registry) -> str: """ Adds a itisfoundation/sleeper in docker registry @@ -76,7 +73,6 @@ def sleeper_service(docker_registry) -> str: assert image yield repo - @pytest.fixture(scope="session") def jupyter_service(docker_registry) -> str: """ Adds a itisfoundation/jupyter-base-notebook in docker registry diff --git a/services/web/server/requirements/ci.txt b/services/web/server/requirements/ci.txt index e166f7cf253..1f5e6ce51b8 100644 --- a/services/web/server/requirements/ci.txt +++ b/services/web/server/requirements/ci.txt @@ -14,6 +14,7 @@ ../../../packages/postgres-database/ ../../../packages/simcore-sdk/ ../../../packages/service-library/ +../../../packages/pytest-simcore/ # installs current package . diff --git a/services/web/server/requirements/dev.txt b/services/web/server/requirements/dev.txt index 26d8db55523..0b7144c13c3 100644 --- a/services/web/server/requirements/dev.txt +++ b/services/web/server/requirements/dev.txt @@ -18,6 +18,7 @@ watchdog[watchmedo] -e ../../../packages/postgres-database/ -e ../../../packages/simcore-sdk/ -e ../../../packages/service-library/ +-e ../../../packages/pytest-simcore/ # installs current package -e . diff --git a/services/web/server/src/simcore_service_webserver/computation_subscribe.py b/services/web/server/src/simcore_service_webserver/computation_subscribe.py index 1d30855d164..af952421019 100644 --- a/services/web/server/src/simcore_service_webserver/computation_subscribe.py +++ b/services/web/server/src/simcore_service_webserver/computation_subscribe.py @@ -3,44 +3,25 @@ import logging from functools import wraps from pprint import pformat -from typing import Callable, Coroutine, Dict, Optional +from typing import Callable, Coroutine, Dict +import aio_pika from aiohttp import web -from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed +from tenacity import retry -import aio_pika from servicelib.application_keys import APP_CONFIG_KEY +from servicelib.rabbitmq_utils import RabbitMQRetryPolicyUponInitialization from simcore_sdk.config.rabbit import eval_broker -from .computation_config import ( - APP_CLIENT_RABBIT_DECORATED_HANDLERS_KEY, - CONFIG_SECTION_NAME, -) +from .computation_config import (APP_CLIENT_RABBIT_DECORATED_HANDLERS_KEY, + CONFIG_SECTION_NAME) from .projects import projects_api -from .projects.projects_exceptions import NodeNotFoundError, ProjectNotFoundError +from .projects.projects_exceptions import (NodeNotFoundError, + ProjectNotFoundError) from .socketio.events import post_messages log = logging.getLogger(__file__) - -class RabbitMQRetryPolicyUponInitialization: - """ Retry policy upon service initialization - """ - - WAIT_SECS = 2 - ATTEMPTS_COUNT = 20 - - def __init__(self, logger: Optional[logging.Logger] = None): - logger = logger or log - - self.kwargs = dict( - wait=wait_fixed(self.WAIT_SECS), - stop=stop_after_attempt(self.ATTEMPTS_COUNT), - before_sleep=before_sleep_log(logger, logging.INFO), - reraise=True, - ) - - def rabbit_adapter(app: web.Application) -> Callable: """this decorator allows passing additional paramters to python-socketio compatible handlers. I.e. aiopika handler expect functions of type `async def function(message)` diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index b71afc71d47..3aa6b5b7f65 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -5,7 +5,7 @@ """ # pylint: disable=unused-argument # pylint: disable=bare-except -# pylint:disable=redefined-outer-name +# pylint: disable=redefined-outer-name import logging import sys @@ -22,12 +22,6 @@ logging.getLogger("openapi_spec_validator").setLevel(logging.WARNING) logging.getLogger("sqlalchemy").setLevel(logging.WARNING) -## HELPERS -sys.path.append(str(current_dir / "helpers")) - - -## FIXTURES: standard paths - @pytest.fixture(scope="session") def package_dir() -> Path: @@ -37,30 +31,6 @@ def package_dir() -> Path: return dirpath -@pytest.fixture(scope="session") -def osparc_simcore_root_dir() -> Path: - """ osparc-simcore repo root dir """ - WILDCARD = "services/web/server" - - root_dir = Path(current_dir) - while not any(root_dir.glob(WILDCARD)) and root_dir != Path("/"): - root_dir = root_dir.parent - - msg = f"'{root_dir}' does not look like the git root directory of osparc-simcore" - assert root_dir.exists(), msg - assert any(root_dir.glob(WILDCARD)), msg - assert any(root_dir.glob(".git")), msg - - return root_dir - - -@pytest.fixture(scope="session") -def env_devel_file(osparc_simcore_root_dir) -> Path: - env_devel_fpath = osparc_simcore_root_dir / ".env-devel" - assert env_devel_fpath.exists() - return env_devel_fpath - - @pytest.fixture(scope="session") def api_specs_dir(osparc_simcore_root_dir: Path) -> Path: specs_dir = osparc_simcore_root_dir / "api" / "specs" / "webserver" diff --git a/services/web/server/tests/integration/computation/conftest.py b/services/web/server/tests/integration/computation/conftest.py index 77e6cc02961..ed21816646c 100644 --- a/services/web/server/tests/integration/computation/conftest.py +++ b/services/web/server/tests/integration/computation/conftest.py @@ -9,9 +9,9 @@ import pytest +from pytest_simcore.helpers.utils_login import LoggedUser +from pytest_simcore.helpers.utils_projects import NewProject from simcore_service_webserver.security_roles import UserRole -from utils_login import LoggedUser -from utils_projects import NewProject current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent diff --git a/services/web/server/tests/integration/computation/test_computation.py b/services/web/server/tests/integration/computation/test_computation.py index 5dbf108f8ba..aa612a94cf5 100644 --- a/services/web/server/tests/integration/computation/test_computation.py +++ b/services/web/server/tests/integration/computation/test_computation.py @@ -5,7 +5,6 @@ import json import sys import time - from pathlib import Path from pprint import pprint @@ -13,6 +12,7 @@ from aiohttp import web from yarl import URL +from pytest_simcore.helpers.utils_assert import assert_status from servicelib.application import create_safe_application from servicelib.application_keys import APP_CONFIG_KEY from simcore_sdk.models.pipeline_models import ( @@ -28,8 +28,6 @@ from simcore_service_webserver.security import setup_security from simcore_service_webserver.security_roles import UserRole from simcore_service_webserver.session import setup_session -from utils_assert import assert_status - current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent diff --git a/services/web/server/tests/integration/conftest.py b/services/web/server/tests/integration/conftest.py index 3152a13bdef..e78dc358c6f 100644 --- a/services/web/server/tests/integration/conftest.py +++ b/services/web/server/tests/integration/conftest.py @@ -23,25 +23,26 @@ import trafaret_config import yaml +from pytest_simcore.helpers.utils_docker import get_service_published_port from simcore_service_webserver.application_config import app_schema from simcore_service_webserver.cli import create_environ from simcore_service_webserver.resources import resources as app_resources -from utils_docker import get_service_published_port + +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent # imports the fixtures for the integration tests pytest_plugins = [ - "fixtures.docker_compose", - "fixtures.docker_swarm", - "fixtures.docker_registry", - "fixtures.rabbit_service", - "fixtures.celery_service", - "fixtures.postgres_service", - "fixtures.redis_service", - "fixtures.websocket_client", + "pytest_simcore.environs", + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", + "pytest_simcore.docker_registry", + "pytest_simcore.rabbit_service", + "pytest_simcore.celery_service", + "pytest_simcore.postgres_service", + "pytest_simcore.redis_service", + "pytest_simcore.websocket_client" ] -current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent - log = logging.getLogger(__name__) diff --git a/services/web/server/tests/integration/fixtures/__init__.py b/services/web/server/tests/integration/fixtures/__init__.py deleted file mode 100644 index b472f27c93f..00000000000 --- a/services/web/server/tests/integration/fixtures/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Collection of tests fixtures for integration testing diff --git a/services/web/server/tests/integration/fixtures/celery_service.py b/services/web/server/tests/integration/fixtures/celery_service.py deleted file mode 100644 index a0e17ab8e8d..00000000000 --- a/services/web/server/tests/integration/fixtures/celery_service.py +++ /dev/null @@ -1,33 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -from copy import deepcopy - -import celery -import celery.bin.base -import celery.bin.celery -import celery.platforms -import pytest -import tenacity - - -@pytest.fixture(scope="module") -def celery_service(_webserver_dev_config, docker_stack): - cfg = deepcopy(_webserver_dev_config["rabbit"]) - host = cfg["host"] - port = cfg["port"] - user = cfg["user"] - password = cfg["password"] - url = "amqp://{}:{}@{}:{}".format(user, password, host, port) - wait_till_celery_responsive(url) - yield url - - -@tenacity.retry(wait=tenacity.wait_fixed(0.1), stop=tenacity.stop_after_delay(60)) -def wait_till_celery_responsive(url): - app = celery.Celery("tasks", broker=url) - - status = celery.bin.celery.CeleryCommand.commands["status"]() - status.app = status.get_app() - status.run() # raises celery.bin.base.Error if cannot run diff --git a/services/web/server/tests/integration/fixtures/docker_swarm.py b/services/web/server/tests/integration/fixtures/docker_swarm.py deleted file mode 100644 index 8b1abf5f571..00000000000 --- a/services/web/server/tests/integration/fixtures/docker_swarm.py +++ /dev/null @@ -1,124 +0,0 @@ -# pylint:disable=wildcard-import -# pylint:disable=unused-import -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -import os -import subprocess -import time -from pathlib import Path - -import docker -import pytest -import yaml - - -@pytest.fixture(scope="session") -def docker_client(): - client = docker.from_env() - yield client - - -@pytest.fixture(scope="module") -def docker_swarm(docker_client): - try: - docker_client.swarm.reload() - print("CAUTION: Already part of a swarm") - yield - except docker.errors.APIError: - docker_client.swarm.init() - yield - # teardown - assert docker_client.swarm.leave(force=True) - - -@pytest.fixture(scope="module") -def docker_stack( - docker_swarm, - docker_client, - core_services_config_file: Path, - ops_services_config_file: Path, -): - stacks = {"simcore": core_services_config_file, "ops": ops_services_config_file} - - # make up-version - stacks_up = [] - for stack_name, stack_config_file in stacks.items(): - subprocess.run( - f"docker stack deploy -c {stack_config_file.name} {stack_name}", - shell=True, - check=True, - cwd=stack_config_file.parent, - ) - stacks_up.append(stack_name) - - # wait for the stack to come up - def _wait_for_services(retry_count, max_wait_time_s): - pre_states = ["NEW", "PENDING", "ASSIGNED", "PREPARING", "STARTING"] - services = docker_client.services.list() - WAIT_TIME_BEFORE_RETRY = 5 - start_time = time.time() - for service in services: - for n in range(retry_count): - assert (time.time() - start_time) < max_wait_time_s - if service.tasks(): - task = service.tasks()[0] - if task["Status"]["State"].upper() not in pre_states: - assert task["Status"]["State"].upper() == "RUNNING" - break - print(f"Waiting for {service.name}...") - time.sleep(WAIT_TIME_BEFORE_RETRY) - - def _print_services(msg): - from pprint import pprint - - print("{:*^100}".format("docker services running " + msg)) - for service in docker_client.services.list(): - pprint(service.attrs) - print("-" * 100) - - RETRY_COUNT = 12 - WAIT_TIME_BEFORE_FAILING = 60 - _wait_for_services(RETRY_COUNT, WAIT_TIME_BEFORE_FAILING) - _print_services("[BEFORE TEST]") - - yield { - "stacks": stacks_up, - "services": [service.name for service in docker_client.services.list()], - } - - _print_services("[AFTER TEST]") - - # clean up. Guarantees that all services are down before creating a new stack! - # - # WORKAROUND https://github.com/moby/moby/issues/30942#issue-207070098 - # - # docker stack rm services - - # until [ -z "$(docker service ls --filter label=com.docker.stack.namespace=services -q)" ] || [ "$limit" -lt 0 ]; do - # sleep 1; - # done - - # until [ -z "$(docker network ls --filter label=com.docker.stack.namespace=services -q)" ] || [ "$limit" -lt 0 ]; do - # sleep 1; - # done - - # make down - # NOTE: remove them in reverse order since stacks share common networks - WAIT_BEFORE_RETRY_SECS = 1 - stacks_up.reverse() - for stack in stacks_up: - subprocess.run(f"docker stack rm {stack}", shell=True, check=True) - - while docker_client.services.list( - filters={"label": f"com.docker.stack.namespace={stack}"} - ): - time.sleep(WAIT_BEFORE_RETRY_SECS) - - while docker_client.networks.list( - filters={"label": f"com.docker.stack.namespace={stack}"} - ): - time.sleep(WAIT_BEFORE_RETRY_SECS) - - _print_services("[AFTER REMOVED]") diff --git a/services/web/server/tests/integration/fixtures/postgres_service.py b/services/web/server/tests/integration/fixtures/postgres_service.py deleted file mode 100644 index 1e183463406..00000000000 --- a/services/web/server/tests/integration/fixtures/postgres_service.py +++ /dev/null @@ -1,48 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -from copy import deepcopy - -import pytest -import sqlalchemy as sa -import tenacity -from servicelib.aiopg_utils import DSN, PostgresRetryPolicyUponInitialization -from simcore_postgres_database.models.base import metadata -from sqlalchemy.orm import sessionmaker - - -@pytest.fixture(scope="module") -def postgres_db(_webserver_dev_config, webserver_environ, docker_stack): - cfg = deepcopy(_webserver_dev_config["db"]["postgres"]) - url = DSN.format(**cfg) - - # NOTE: Comment this to avoid postgres_service - assert wait_till_postgres_responsive(url) - - # Configures db and initializes tables - # Uses syncrounous engine for that - engine = sa.create_engine(url, isolation_level="AUTOCOMMIT") - metadata.create_all(bind=engine, checkfirst=True) - - yield engine - - metadata.drop_all(engine) - engine.dispose() - - -@pytest.fixture(scope="module") -def postgres_session(postgres_db): - Session = sessionmaker(postgres_db) - session = Session() - yield session - session.close() - - -@tenacity.retry(**PostgresRetryPolicyUponInitialization().kwargs) -def wait_till_postgres_responsive(url): - """Check if something responds to ``url`` """ - engine = sa.create_engine(url) - conn = engine.connect() - conn.close() - return True diff --git a/services/web/server/tests/integration/fixtures/rabbit_service.py b/services/web/server/tests/integration/fixtures/rabbit_service.py deleted file mode 100644 index 5ff28f6c3fe..00000000000 --- a/services/web/server/tests/integration/fixtures/rabbit_service.py +++ /dev/null @@ -1,29 +0,0 @@ -# pylint:disable=wildcard-import -# pylint:disable=unused-import -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -from copy import deepcopy -from typing import Dict - -import aio_pika -import pytest -import tenacity - - -@pytest.fixture(scope="function") -async def rabbit_service(_webserver_dev_config: Dict, docker_stack): - cfg = deepcopy(_webserver_dev_config["rabbit"]) - host = cfg["host"] - port = cfg["port"] - user = cfg["user"] - password = cfg["password"] - url = "amqp://{}:{}@{}:{}".format(user, password, host, port) - await wait_till_rabbit_responsive(url) - - -@tenacity.retry(wait=tenacity.wait_fixed(0.1), stop=tenacity.stop_after_delay(60)) -async def wait_till_rabbit_responsive(url: str): - await aio_pika.connect(url) - return True diff --git a/services/web/server/tests/integration/fixtures/redis_service.py b/services/web/server/tests/integration/fixtures/redis_service.py deleted file mode 100644 index 66dab115689..00000000000 --- a/services/web/server/tests/integration/fixtures/redis_service.py +++ /dev/null @@ -1,43 +0,0 @@ -# pylint:disable=wildcard-import -# pylint:disable=unused-import -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -from copy import deepcopy - -import aioredis -import pytest -import tenacity -from yarl import URL - - -@pytest.fixture(scope="module") -async def redis_service(loop, _webserver_dev_config, webserver_environ, docker_stack): - cfg = deepcopy(_webserver_dev_config["resource_manager"]["redis"]) - - host = cfg["host"] - port = cfg["port"] - url = URL(f"redis://{host}:{port}") - - assert await wait_till_redis_responsive(url) - - yield url - - -@tenacity.retry(wait=tenacity.wait_fixed(0.1), stop=tenacity.stop_after_delay(60)) -async def wait_till_redis_responsive(redis_url: URL) -> bool: - client = await aioredis.create_redis_pool(str(redis_url), encoding="utf-8") - client.close() - await client.wait_closed() - return True - - -@pytest.fixture(scope="module") -async def redis_client(loop, redis_service): - client = await aioredis.create_redis_pool(str(redis_service), encoding="utf-8") - yield client - - await client.flushall() - client.close() - await client.wait_closed() diff --git a/services/web/server/tests/integration/test_project_workflow.py b/services/web/server/tests/integration/test_project_workflow.py index 51261eb14aa..9b5a1357c8d 100644 --- a/services/web/server/tests/integration/test_project_workflow.py +++ b/services/web/server/tests/integration/test_project_workflow.py @@ -17,6 +17,9 @@ import pytest from aiohttp import web +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser +from pytest_simcore.helpers.utils_projects import delete_all_projects from servicelib.application import create_safe_application from simcore_service_webserver.db import setup_db from simcore_service_webserver.login import setup_login @@ -27,9 +30,6 @@ from simcore_service_webserver.security import setup_security from simcore_service_webserver.security_roles import UserRole from simcore_service_webserver.session import setup_session -from utils_assert import assert_status -from utils_login import LoggedUser -from utils_projects import delete_all_projects API_VERSION = "v0" diff --git a/services/web/server/tests/unit/conftest.py b/services/web/server/tests/unit/conftest.py index d68411e366a..92e81a90777 100644 --- a/services/web/server/tests/unit/conftest.py +++ b/services/web/server/tests/unit/conftest.py @@ -17,7 +17,6 @@ import pytest - from simcore_service_webserver.resources import resources from simcore_service_webserver.utils import now_str @@ -27,6 +26,8 @@ ## Log log = logging.getLogger(__name__) +pytest_plugins = ["pytest_simcore.environs"] + @pytest.fixture(scope="session") def here(): diff --git a/services/web/server/tests/unit/test_activity.py b/services/web/server/tests/unit/test_activity.py index 79068d489c3..c4dcaecaea4 100644 --- a/services/web/server/tests/unit/test_activity.py +++ b/services/web/server/tests/unit/test_activity.py @@ -6,17 +6,17 @@ from asyncio import Future from pathlib import Path -import yaml - import pytest +import yaml from aiohttp import web from aiohttp.client_exceptions import ClientConnectionError + +from pytest_simcore.helpers.utils_assert import assert_status from servicelib.application import create_safe_application from simcore_service_webserver.activity import handlers, setup_activity from simcore_service_webserver.rest import setup_rest from simcore_service_webserver.security import setup_security from simcore_service_webserver.session import setup_session -from utils_assert import assert_status def future_with_result(result): diff --git a/services/web/server/tests/unit/test_configs.py b/services/web/server/tests/unit/test_configs.py index e3fb1a7c363..39c48d5c7e0 100644 --- a/services/web/server/tests/unit/test_configs.py +++ b/services/web/server/tests/unit/test_configs.py @@ -12,8 +12,9 @@ import pytest import yaml - from aiohttp import web + +from pytest_simcore.helpers.utils_environs import eval_service_environ, load_env from servicelib.application_setup import is_setup_function from simcore_service_webserver.application_config import create_schema from simcore_service_webserver.cli import parse, setup_parser @@ -27,7 +28,6 @@ from simcore_service_webserver.login.cfg import DEFAULTS as CONFIG_DEFAULTS from simcore_service_webserver.login.cfg import Cfg from simcore_service_webserver.resources import resources -from utils_environs import eval_service_environ, load_env config_yaml_filenames = [str(name) for name in resources.listdir("config")] diff --git a/services/web/server/tests/unit/test_rest.py b/services/web/server/tests/unit/test_rest.py index 3ef4e9ea639..eec75c1c9bd 100644 --- a/services/web/server/tests/unit/test_rest.py +++ b/services/web/server/tests/unit/test_rest.py @@ -10,12 +10,12 @@ import yaml from aiohttp import web +from pytest_simcore.helpers.utils_assert import assert_status from servicelib.application import create_safe_application from servicelib.application_keys import APP_CONFIG_KEY from simcore_service_webserver.resources import resources from simcore_service_webserver.rest import setup_rest from simcore_service_webserver.security import setup_security -from utils_assert import assert_status # TODO: reduce log from openapi_core loggers diff --git a/services/web/server/tests/unit/with_dbs/test_access_to_studies.py b/services/web/server/tests/unit/with_dbs/test_access_to_studies.py index 2b2a5736cbe..5d71005afa3 100644 --- a/services/web/server/tests/unit/with_dbs/test_access_to_studies.py +++ b/services/web/server/tests/unit/with_dbs/test_access_to_studies.py @@ -17,6 +17,9 @@ from aiohttp import web import simcore_service_webserver.statics +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser, UserRole +from pytest_simcore.helpers.utils_projects import NewProject, delete_all_projects from servicelib.application import create_safe_application from servicelib.application_keys import APP_CONFIG_KEY from servicelib.rest_responses import unwrap_envelope @@ -30,9 +33,6 @@ from simcore_service_webserver.statics import setup_statics from simcore_service_webserver.studies_access import setup_studies_access from simcore_service_webserver.users import setup_users -from utils_assert import assert_status -from utils_login import LoggedUser, UserRole -from utils_projects import NewProject, delete_all_projects SHARED_STUDY_UUID = "e2e38eee-c569-4e55-b104-70d159e49c87" diff --git a/services/web/server/tests/unit/with_dbs/test_catalog_api.py b/services/web/server/tests/unit/with_dbs/test_catalog_api.py index 0a520c3b507..7c215c31c5a 100644 --- a/services/web/server/tests/unit/with_dbs/test_catalog_api.py +++ b/services/web/server/tests/unit/with_dbs/test_catalog_api.py @@ -8,6 +8,8 @@ import pytest from aiohttp import web +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser from simcore_service_webserver.application import ( create_safe_application, setup_catalog, @@ -18,8 +20,6 @@ setup_session, ) from simcore_service_webserver.db_models import UserRole -from utils_assert import assert_status -from utils_login import LoggedUser @pytest.fixture() diff --git a/services/web/server/tests/unit/with_dbs/test_change_email.py b/services/web/server/tests/unit/with_dbs/test_change_email.py index 373bcf23328..13075f47c45 100644 --- a/services/web/server/tests/unit/with_dbs/test_change_email.py +++ b/services/web/server/tests/unit/with_dbs/test_change_email.py @@ -7,11 +7,11 @@ from aiohttp import web from yarl import URL +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser, NewUser, parse_link from servicelib.rest_responses import unwrap_envelope from simcore_service_webserver.login import APP_LOGIN_CONFIG from simcore_service_webserver.statics import INDEX_RESOURCE_NAME -from utils_assert import assert_status -from utils_login import LoggedUser, NewUser, parse_link NEW_EMAIL = "new@mail.com" diff --git a/services/web/server/tests/unit/with_dbs/test_change_password.py b/services/web/server/tests/unit/with_dbs/test_change_password.py index 0de8f038a14..2c25ba31c13 100644 --- a/services/web/server/tests/unit/with_dbs/test_change_password.py +++ b/services/web/server/tests/unit/with_dbs/test_change_password.py @@ -7,10 +7,10 @@ from aiohttp import web from yarl import URL +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser, parse_link from servicelib.rest_responses import unwrap_envelope from simcore_service_webserver.login import APP_LOGIN_CONFIG -from utils_assert import assert_status -from utils_login import LoggedUser, parse_link NEW_PASSWORD = "NewPassword1*&^" diff --git a/services/web/server/tests/unit/with_dbs/test_guests_management.py b/services/web/server/tests/unit/with_dbs/test_guests_management.py index f75ecaf9050..aa901513e0c 100644 --- a/services/web/server/tests/unit/with_dbs/test_guests_management.py +++ b/services/web/server/tests/unit/with_dbs/test_guests_management.py @@ -9,9 +9,9 @@ import pytest from aiohttp import web +from pytest_simcore.helpers.utils_projects import create_project from servicelib.application import create_safe_application from simcore_service_webserver import application -from utils_projects import create_project @pytest.fixture diff --git a/services/web/server/tests/unit/with_dbs/test_login.py b/services/web/server/tests/unit/with_dbs/test_login.py index 8a5c3e96c7b..fe1db69f5c8 100644 --- a/services/web/server/tests/unit/with_dbs/test_login.py +++ b/services/web/server/tests/unit/with_dbs/test_login.py @@ -6,10 +6,10 @@ from aiohttp import web +from pytest_simcore.helpers.utils_login import NewUser from servicelib.rest_responses import unwrap_envelope from simcore_service_webserver.db_models import ConfirmationAction, UserStatus from simcore_service_webserver.login.cfg import cfg -from utils_login import NewUser EMAIL, PASSWORD = "tester@test.com", "password" diff --git a/services/web/server/tests/unit/with_dbs/test_logout.py b/services/web/server/tests/unit/with_dbs/test_logout.py index e8ed4e945bf..3c12cc82ed4 100644 --- a/services/web/server/tests/unit/with_dbs/test_logout.py +++ b/services/web/server/tests/unit/with_dbs/test_logout.py @@ -1,8 +1,8 @@ from aiohttp import web +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser from simcore_service_webserver.login.cfg import get_storage -from utils_assert import assert_status -from utils_login import LoggedUser async def test_logout(client): diff --git a/services/web/server/tests/unit/with_dbs/test_projects.py b/services/web/server/tests/unit/with_dbs/test_projects.py index 29a11190736..88ea68c5643 100644 --- a/services/web/server/tests/unit/with_dbs/test_projects.py +++ b/services/web/server/tests/unit/with_dbs/test_projects.py @@ -17,6 +17,10 @@ from mock import call from yarl import URL +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser +from pytest_simcore.helpers.utils_projects import (NewProject, + delete_all_projects) from servicelib.application import create_safe_application from servicelib.application_keys import APP_CONFIG_KEY from servicelib.rest_responses import unwrap_envelope @@ -32,9 +36,6 @@ from simcore_service_webserver.socketio import setup_sockets from simcore_service_webserver.tags import setup_tags from simcore_service_webserver.utils import now_str, to_datetime -from utils_assert import assert_status -from utils_login import LoggedUser -from utils_projects import NewProject, delete_all_projects API_VERSION = "v0" RESOURCE_NAME = "projects" @@ -703,7 +704,7 @@ async def test_close_project( "get_running_interactive_services" ].has_calls(calls) mocked_director_subsystem["get_running_interactive_services"].reset_mock() - + # close project url = client.app.router["close_project"].url_for(project_id=user_project["uuid"]) resp = await client.post(url, json=client_id) @@ -716,7 +717,7 @@ async def test_close_project( mocked_director_subsystem["get_running_interactive_services"].has_calls(calls) calls = [call(client.server.app, service["service_uuid"]) for service in fakes] mocked_director_subsystem["stop_service"].has_calls(calls) - + @pytest.mark.parametrize( diff --git a/services/web/server/tests/unit/with_dbs/test_registration.py b/services/web/server/tests/unit/with_dbs/test_registration.py index b301232a16f..0fbfd79e20d 100644 --- a/services/web/server/tests/unit/with_dbs/test_registration.py +++ b/services/web/server/tests/unit/with_dbs/test_registration.py @@ -6,12 +6,12 @@ import pytest from aiohttp import web +from pytest_simcore.helpers.utils_assert import assert_error, assert_status +from pytest_simcore.helpers.utils_login import NewInvitation, NewUser, parse_link from servicelib.rest_responses import unwrap_envelope from simcore_service_webserver.db_models import ConfirmationAction, UserStatus from simcore_service_webserver.login.cfg import cfg, get_storage from simcore_service_webserver.login.registration import get_confirmation_info -from utils_assert import assert_error, assert_status -from utils_login import NewInvitation, NewUser, parse_link EMAIL, PASSWORD = "tester@test.com", "password" diff --git a/services/web/server/tests/unit/with_dbs/test_reset_password.py b/services/web/server/tests/unit/with_dbs/test_reset_password.py index d324374d51f..8d8f80c500b 100644 --- a/services/web/server/tests/unit/with_dbs/test_reset_password.py +++ b/services/web/server/tests/unit/with_dbs/test_reset_password.py @@ -8,12 +8,12 @@ from aiohttp import web from yarl import URL +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import NewUser, parse_link, parse_test_marks from servicelib.rest_responses import unwrap_envelope from simcore_service_webserver.db_models import ConfirmationAction, UserStatus from simcore_service_webserver.login import APP_LOGIN_CONFIG from simcore_service_webserver.login.utils import get_random_string -from utils_assert import assert_status -from utils_login import NewUser, parse_link, parse_test_marks EMAIL, PASSWORD = "tester@test.com", "password" diff --git a/services/web/server/tests/unit/with_dbs/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/test_resource_manager.py index eb2551e52d2..86f75993b9b 100644 --- a/services/web/server/tests/unit/with_dbs/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/test_resource_manager.py @@ -16,6 +16,9 @@ from mock import call from yarl import URL +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser +from pytest_simcore.helpers.utils_projects import NewProject from servicelib.application import create_safe_application from servicelib.rest_responses import unwrap_envelope from simcore_service_webserver.db import setup_db @@ -35,9 +38,6 @@ from simcore_service_webserver.socketio import setup_sockets from simcore_service_webserver.users import setup_users from simcore_service_webserver.utils import now_str -from utils_assert import assert_status -from utils_login import LoggedUser -from utils_projects import NewProject API_VERSION = "v0" GARBAGE_COLLECTOR_INTERVAL = 1 diff --git a/services/web/server/tests/unit/with_dbs/test_storage.py b/services/web/server/tests/unit/with_dbs/test_storage.py index 50652df1aff..59f94a724a2 100644 --- a/services/web/server/tests/unit/with_dbs/test_storage.py +++ b/services/web/server/tests/unit/with_dbs/test_storage.py @@ -9,12 +9,12 @@ import pytest from aiohttp import web +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser from servicelib.application import create_safe_application from servicelib.rest_responses import unwrap_envelope from servicelib.rest_utils import extract_and_validate from simcore_service_webserver.security_roles import UserRole -from utils_assert import assert_status -from utils_login import LoggedUser API_VERSION = "v0" diff --git a/services/web/server/tests/unit/with_dbs/test_users.py b/services/web/server/tests/unit/with_dbs/test_users.py index 738092f756e..d46b6e57033 100644 --- a/services/web/server/tests/unit/with_dbs/test_users.py +++ b/services/web/server/tests/unit/with_dbs/test_users.py @@ -14,6 +14,13 @@ from aiopg.sa.connection import SAConnection from psycopg2 import OperationalError +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser +from pytest_simcore.helpers.utils_tokens import ( + create_token_in_db, + delete_all_tokens_from_db, + get_token_from_db, +) from servicelib.application import create_safe_application from simcore_service_webserver.db import APP_DB_ENGINE_KEY, setup_db from simcore_service_webserver.login import setup_login @@ -22,13 +29,6 @@ from simcore_service_webserver.security_roles import UserRole from simcore_service_webserver.session import setup_session from simcore_service_webserver.users import setup_users -from utils_assert import assert_status -from utils_login import LoggedUser -from utils_tokens import ( - create_token_in_db, - delete_all_tokens_from_db, - get_token_from_db, -) API_VERSION = "v0"