Is/auto add to group (#1745)

Author: sanderegg

packages/postgres-database/src/simcore_postgres_database/migration/versions/43766dcf0fc2_add_groups_inclusion_rules_column.py

@@ -0,0 +1,28 @@
+"""add groups inclusion_rules column
+
+Revision ID: 43766dcf0fc2
+Revises: 01073b43fc74
+Create Date: 2020-08-25 08:54:55.441452+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '43766dcf0fc2'
+down_revision = '01073b43fc74'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('groups', sa.Column('inclusion_rules', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('groups', 'inclusion_rules')
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/groups.py

@@ -34,6 +34,7 @@ class GroupType(enum.Enum):
     sa.Column("description", sa.String, nullable=False),
     sa.Column("type", sa.Enum(GroupType), nullable=False, server_default="STANDARD"),
     sa.Column("thumbnail", sa.String, nullable=True),
+    sa.Column("inclusion_rules", JSONB, nullable=False, server_default=sa.text("'{}'::jsonb")),
     sa.Column("created", sa.DateTime(), nullable=False, server_default=func.now()),
     sa.Column(
         "modified",

packages/simcore-sdk/tests/integration/test_dbmanager.py

@@ -13,7 +13,7 @@
     "postgres",
 ]
 
-ops_services = ["minio", "adminer"]
+ops_services = ["minio"]
 
 
 async def test_db_manager_read_config(

services/director/tests/fixtures/fake_services.py

@@ -5,14 +5,18 @@
 import json
 import logging
 import random
+from io import BytesIO
 from pathlib import Path
 
-import docker
 import pytest
 import requests
+from aiodocker.docker import Docker
+from aiodocker.exceptions import DockerError
 
 _logger = logging.getLogger(__name__)
 
+import asyncio
+
 
 @pytest.fixture(scope="function")
 def push_services(loop, docker_registry, tmpdir):
@@ -22,7 +26,7 @@ def push_services(loop, docker_registry, tmpdir):
     list_of_pushed_images_tags = []
     dependent_images = []
 
-    def build_push_images(
+    async def build_push_images(
         number_of_computational_services,
         number_of_interactive_services,
         inter_dependent_services=False,
@@ -32,7 +36,7 @@ def build_push_images(
         try:
             dependent_image = None
             if inter_dependent_services:
-                dependent_image = _build_push_image(
+                dependent_image = await _build_push_image(
                     tmp_dir,
                     registry_url,
                     "computational",
@@ -43,30 +47,36 @@ def build_push_images(
                 )
                 dependent_images.append(dependent_image)
 
+            images_to_build = []
+
             for image_index in range(0, number_of_computational_services):
-                image = _build_push_image(
-                    tmp_dir,
-                    registry_url,
-                    "computational",
-                    "test",
-                    version + str(image_index),
-                    dependent_image,
-                    bad_json_format=bad_json_format,
+                images_to_build.append(
+                    _build_push_image(
+                        tmp_dir,
+                        registry_url,
+                        "computational",
+                        "test",
+                        version + str(image_index),
+                        dependent_image,
+                        bad_json_format=bad_json_format,
+                    )
                 )
-                list_of_pushed_images_tags.append(image)
 
             for image_index in range(0, number_of_interactive_services):
-                image = _build_push_image(
-                    tmp_dir,
-                    registry_url,
-                    "dynamic",
-                    "test",
-                    version + str(image_index),
-                    dependent_image,
-                    bad_json_format=bad_json_format,
+                images_to_build.append(
+                    _build_push_image(
+                        tmp_dir,
+                        registry_url,
+                        "dynamic",
+                        "test",
+                        version + str(image_index),
+                        dependent_image,
+                        bad_json_format=bad_json_format,
+                    )
                 )
-                list_of_pushed_images_tags.append(image)
-        except docker.errors.APIError:
+            results = await asyncio.gather(*images_to_build)
+            list_of_pushed_images_tags.extend(results)
+        except DockerError:
             _logger.exception("Unexpected docker API error")
             raise
 
@@ -78,17 +88,17 @@ def build_push_images(
     _clean_registry(registry_url, dependent_images)
 
 
-def _build_push_image(
+async def _build_push_image(
     docker_dir,
     registry_url,
     service_type,
     name,
     tag,
     dependent_image=None,
     *,
-    bad_json_format=False
+    bad_json_format=False,
 ):  # pylint: disable=R0913
-    docker_client = docker.from_env()
+    docker = Docker()
     # crate image
     service_description = _create_service_description(service_type, name, tag)
     docker_labels = _create_docker_labels(service_description, bad_json_format)
@@ -141,16 +151,15 @@ def _build_push_image(
     docker_labels["org.label-schema.vcs-ref"] = service_extras["vcs_ref"]
     docker_labels["org.label-schema.vcs-url"] = service_extras["vcs_url"]
 
-    image = _create_base_image(docker_dir, docker_labels)
-    # tag image
     image_tag = registry_url + "/{key}:{version}".format(
         key=service_description["key"], version=tag
     )
-    assert image.tag(image_tag) is True
+    await _create_base_image(docker_labels, image_tag)
+
     # push image to registry
-    docker_client.images.push(image_tag)
+    await docker.images.push(image_tag)
     # remove image from host
-    docker_client.images.remove(image_tag)
+    # docker.images.remove(image_tag)
     return {
         "service_description": service_description,
         "docker_labels": docker_labels,
@@ -181,16 +190,21 @@ def _clean_registry(registry_url, list_of_images):
         response = requests.delete(url, headers=request_headers)
 
 
-def _create_base_image(base_dir, labels):
-    # create a basic dockerfile
-    docker_file = base_dir / "Dockerfile"
-    with docker_file.open("w") as file_pointer:
-        file_pointer.write("FROM alpine\nCMD while true; do sleep 10; done\n")
-    assert docker_file.exists() == True
+from aiodocker import utils
+
+
+async def _create_base_image(labels, tag):
+    dockerfile = """
+FROM alpine
+CMD while true; do sleep 10; done
+    """
+    f = BytesIO(dockerfile.encode("utf-8"))
+    tar_obj = utils.mktar_from_dockerfile(f)
+
     # build docker base image
-    docker_client = docker.from_env()
-    base_docker_image = docker_client.images.build(
-        path=str(base_dir), rm=True, labels=labels
+    docker = Docker()
+    base_docker_image = await docker.images.build(
+        fileobj=tar_obj, encoding="gzip", rm=True, labels=labels, tag=tag
     )
     return base_docker_image[0]
 

services/director/tests/test_docker_utils.py

@@ -56,3 +56,11 @@ async def test_swarm_has_manager_nodes(loop, docker_swarm):
 
 async def test_swarm_has_worker_nodes(loop, docker_swarm):
     assert (await docker_utils.swarm_has_worker_nodes()) == False
+
+
+async def test_push_services(
+    push_services, configure_registry_access, configure_schemas_location,
+):
+    images = await push_services(
+        number_of_computational_services=3, number_of_interactive_services=3
+    )

services/director/tests/test_dummy_services.py

@@ -11,10 +11,10 @@
 log = logging.getLogger(__name__)
 
 
-def test_services_conformity(configure_schemas_location, push_services):
+async def test_services_conformity(configure_schemas_location, push_services):
     from simcore_service_director import resources
 
-    services = push_services(1, 1)
+    services = await push_services(1, 1)
     with resources.stream(resources.RESOURCE_NODE_SCHEMA) as file_pt:
         service_schema = json.load(file_pt)
     for service in services:

services/director/tests/test_handlers.py

@@ -74,7 +74,7 @@ async def test_services_get(docker_registry, client, push_services, api_version_
     _check_services([], services)
 
     # some services
-    created_services = push_services(3, 2)
+    created_services = await push_services(3, 2)
     web_response = await client.get(f"/{api_version_prefix}/services")
     assert web_response.status == 200
     assert web_response.content_type == "application/json"
@@ -129,7 +129,7 @@ async def test_services_by_key_version_get(
     )
     assert web_response.status == 404
 
-    created_services = push_services(3, 2)
+    created_services = await push_services(3, 2)
     assert len(created_services) == 5
 
     retrieved_services = []
@@ -171,7 +171,7 @@ async def test_services_extras_by_key_version_get(
     )
     assert web_response.status == 404
 
-    created_services = push_services(3, 2)
+    created_services = await push_services(3, 2)
     assert len(created_services) == 5
 
     for created_service in created_services:
@@ -225,7 +225,7 @@ async def _start_get_stop_services(
     data = await web_response.json()
     assert web_response.status == 404, data
 
-    created_services = push_services(0, 2)
+    created_services = await push_services(0, 2)
     assert len(created_services) == 2
     for created_service in created_services:
         service_description = created_service["service_description"]
@@ -372,7 +372,7 @@ async def test_running_interactive_services_list_get(
     project_ids = ["first_project_id", "second_project_id", "third_project_id"]
     # prepare services
     NUM_SERVICES = 1
-    created_services = push_services(0, NUM_SERVICES)
+    created_services = await push_services(0, NUM_SERVICES)
     assert len(created_services) == NUM_SERVICES
     # start the services
     for user_id in user_ids:

services/director/tests/test_producer.py

@@ -27,7 +27,7 @@ async def run_services(
     started_services = []
 
     async def push_start_services(number_comp, number_dyn, dependant=False):
-        pushed_services = push_services(
+        pushed_services = await push_services(
             number_comp, number_dyn, inter_dependent_services=dependant
         )
         assert len(pushed_services) == (number_comp + number_dyn)

services/director/tests/test_registry_cache_task.py

@@ -42,7 +42,7 @@ async def test_registry_caching_task(loop, client, push_services):
     assert not list_of_services
     assert app[registry_cache_task.APP_REGISTRY_CACHE_DATA_KEY] != {}
     # create services in the registry
-    pushed_services = push_services(
+    pushed_services = await push_services(
         number_of_computational_services=1, number_of_interactive_services=1
     )
     # the services shall be updated
@@ -54,7 +54,7 @@ async def test_registry_caching_task(loop, client, push_services):
     )
     assert len(list_of_services) == 2
     # add more
-    pushed_services = push_services(
+    pushed_services = await push_services(
         number_of_computational_services=2,
         number_of_interactive_services=2,
         version="2.0.",

services/director/tests/test_registry_proxy.py

@@ -38,7 +38,7 @@ async def test_list_services_with_bad_json_formatting(
     push_services,
 ):
     # some services
-    created_services = push_services(
+    created_services = await push_services(
         number_of_computational_services=3,
         number_of_interactive_services=2,
         bad_json_format=True,
@@ -65,7 +65,9 @@ async def test_list_computational_services(
     configure_registry_access,
     configure_schemas_location,
 ):
-    push_services(number_of_computational_services=6, number_of_interactive_services=3)
+    await push_services(
+        number_of_computational_services=6, number_of_interactive_services=3
+    )
 
     computational_services = await registry_proxy.list_services(
         aiohttp_mock_app, registry_proxy.ServiceType.COMPUTATIONAL
@@ -80,7 +82,9 @@ async def test_list_interactive_services(
     configure_registry_access,
     configure_schemas_location,
 ):
-    push_services(number_of_computational_services=5, number_of_interactive_services=4)
+    await push_services(
+        number_of_computational_services=5, number_of_interactive_services=4
+    )
     interactive_services = await registry_proxy.list_services(
         aiohttp_mock_app, registry_proxy.ServiceType.DYNAMIC
     )
@@ -94,7 +98,7 @@ async def test_list_of_image_tags(
     configure_registry_access,
     configure_schemas_location,
 ):
-    images = push_services(
+    images = await push_services(
         number_of_computational_services=5, number_of_interactive_services=3
     )
     image_number = {}
@@ -117,7 +121,7 @@ async def test_list_interactive_service_dependencies(
     configure_registry_access,
     configure_schemas_location,
 ):
-    images = push_services(
+    images = await push_services(
         number_of_computational_services=2,
         number_of_interactive_services=2,
         inter_dependent_services=True,
@@ -147,7 +151,7 @@ async def test_get_image_labels(
     configure_registry_access,
     configure_schemas_location,
 ):
-    images = push_services(
+    images = await push_services(
         number_of_computational_services=1, number_of_interactive_services=1
     )
     for image in images:
@@ -213,7 +217,7 @@ async def test_get_image_details(
     configure_registry_access,
     configure_schemas_location,
 ):
-    images = push_services(
+    images = await push_services(
         number_of_computational_services=1, number_of_interactive_services=1
     )
     for image in images:
@@ -231,7 +235,7 @@ async def test_registry_caching(
     configure_registry_access,
     configure_schemas_location,
 ):
-    images = push_services(
+    images = await push_services(
         number_of_computational_services=1, number_of_interactive_services=1
     )
     config.DIRECTOR_REGISTRY_CACHING = True
@@ -274,7 +278,7 @@ async def test_generate_service_extras(
     configure_registry_access,
     configure_schemas_location,
 ):
-    images = push_services(
+    images = await push_services(
         number_of_computational_services=1, number_of_interactive_services=1
     )