Fixes pytest-asyncio loop issue in storage

Copied changes in storage files from PR ITISFoundation#355 - commit ff814d9

Author: Pedro Crespo

services/storage/src/simcore_service_storage/handlers.py

@@ -68,6 +68,7 @@ async def check_action(request: web.Request):
     }
     return output
 
+
 async def get_storage_locations(request: web.Request):
     log.info("CHECK LOCATION PATH %s %s",request.path, request.url)
 

services/storage/tests/conftest.py

@@ -1,9 +1,9 @@
-# TODO: W0611:Unused import ...
-# pylint: disable=W0611
-# TODO: W0613:Unused argument ...
-# pylint: disable=W0613
-#
-# pylint: disable=W0621
+# pylint:disable=wildcard-import
+# pylint:disable=unused-import
+# pylint:disable=unused-variable
+# pylint:disable=unused-argument
+# pylint:disable=redefined-outer-name
+
 import asyncio
 import os
 import subprocess
@@ -26,24 +26,27 @@
                                         SIMCORE_S3_STR)
 from utils import ACCESS_KEY, BUCKET_NAME, DATABASE, PASS, SECRET_KEY, USER
 
-# fixtures -------------------------------------------------------
+
 
 @pytest.fixture(scope='session')
 def here():
     return Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
 
+
 @pytest.fixture(scope='session')
 def package_dir(here):
     dirpath = Path(simcore_service_storage.__file__).parent
     assert dirpath.exists()
     return dirpath
 
+
 @pytest.fixture(scope='session')
 def osparc_simcore_root_dir(here):
     root_dir = here.parent.parent.parent
     assert root_dir.exists(), "Is this service within osparc-simcore repo?"
     return root_dir
 
+
 @pytest.fixture(scope='session')
 def python27_exec(osparc_simcore_root_dir, tmpdir_factory, here):
     # Assumes already created with make .venv27
@@ -52,16 +55,18 @@ def python27_exec(osparc_simcore_root_dir, tmpdir_factory, here):
     if not venv27.exists():
         # create its own virtualenv
         venv27 = tmpdir_factory.mktemp("virtualenv") / ".venv27"
-        cmd = "virtualenv --python=python2 %s"%(venv27) # TODO: how to split in command safely?
-        assert subprocess.check_call(cmd.split()) == 0, "Unable to run %s" %cmd
+        # TODO: how to split in command safely?
+        cmd = "virtualenv --python=python2 %s" % (venv27)
+        assert subprocess.check_call(
+            cmd.split()) == 0, "Unable to run %s" % cmd
 
         # installs python2 requirements
         pip_exec = venv27 / "bin" / "pip"
         assert pip_exec.exists()
         requirements_py2 = here.parent / "requirements/py27.txt"
         cmd = "{} install -r {}".format(pip_exec, requirements_py2)
-        assert subprocess.check_call(cmd.split()) == 0, "Unable to run %s" %cmd
-
+        assert subprocess.check_call(
+            cmd.split()) == 0, "Unable to run %s" % cmd
 
     python27_exec = venv27 / "bin" / "python2.7"
     assert python27_exec.exists()
@@ -73,19 +78,20 @@ def python27_path(python27_exec):
     return Path(python27_exec).parent.parent
     # Assumes already created with make .venv27
 
+
 @pytest.fixture(scope='session')
 def docker_compose_file(here):
     """ Overrides pytest-docker fixture
     """
     old = os.environ.copy()
 
     # docker-compose reads these environs
-    os.environ['POSTGRES_DB']=DATABASE
-    os.environ['POSTGRES_USER']=USER
-    os.environ['POSTGRES_PASSWORD']=PASS
-    os.environ['POSTGRES_ENDPOINT']="FOO" # TODO: update config schema!!
-    os.environ['MINIO_ACCESS_KEY']=ACCESS_KEY
-    os.environ['MINIO_SECRET_KEY']=SECRET_KEY
+    os.environ['POSTGRES_DB'] = DATABASE
+    os.environ['POSTGRES_USER'] = USER
+    os.environ['POSTGRES_PASSWORD'] = PASS
+    os.environ['POSTGRES_ENDPOINT'] = "FOO"  # TODO: update config schema!!
+    os.environ['MINIO_ACCESS_KEY'] = ACCESS_KEY
+    os.environ['MINIO_SECRET_KEY'] = SECRET_KEY
 
     dc_path = here / 'docker-compose.yml'
 
@@ -94,12 +100,13 @@ def docker_compose_file(here):
 
     os.environ = old
 
+
 @pytest.fixture(scope='session')
 def postgres_service(docker_services, docker_ip):
     url = 'postgresql://{user}:{password}@{host}:{port}/{database}'.format(
-        user = USER,
-        password = PASS,
-        database = DATABASE,
+        user=USER,
+        password=PASS,
+        database=DATABASE,
         host=docker_ip,
         port=docker_services.port_for('postgres', 5432),
     )
@@ -112,27 +119,29 @@ def postgres_service(docker_services, docker_ip):
     )
 
     postgres_service = {
-        'user' : USER,
-        'password' : PASS,
-        'database' : DATABASE,
-        'host' : docker_ip,
-        'port' : docker_services.port_for('postgres', 5432)
+        'user': USER,
+        'password': PASS,
+        'database': DATABASE,
+        'host': docker_ip,
+        'port': docker_services.port_for('postgres', 5432)
     }
 
     return postgres_service
 
+
 @pytest.fixture(scope='session')
 def postgres_service_url(postgres_service, docker_services, docker_ip):
     postgres_service_url = 'postgresql://{user}:{password}@{host}:{port}/{database}'.format(
-        user = USER,
-        password = PASS,
-        database = DATABASE,
+        user=USER,
+        password=PASS,
+        database=DATABASE,
         host=docker_ip,
         port=docker_services.port_for('postgres', 5432),
     )
 
     return postgres_service_url
 
+
 @pytest.fixture(scope='function')
 async def postgres_engine(loop, postgres_service_url):
     postgres_engine = await create_engine(postgres_service_url)
@@ -163,24 +172,28 @@ def minio_service(docker_services, docker_ip):
     return {
         'endpoint': '{ip}:{port}'.format(ip=docker_ip, port=docker_services.port_for('minio', 9000)),
         'access_key': ACCESS_KEY,
-        'secret_key' : SECRET_KEY,
-        'bucket_name' : BUCKET_NAME,
-        }
+        'secret_key': SECRET_KEY,
+        'bucket_name': BUCKET_NAME,
+    }
+
 
 @pytest.fixture(scope="module")
 def s3_client(minio_service):
     from s3wrapper.s3_client import S3Client
 
-    s3_client = S3Client(endpoint=minio_service['endpoint'],access_key=minio_service["access_key"], secret_key=minio_service["secret_key"])
+    s3_client = S3Client(
+        endpoint=minio_service['endpoint'], access_key=minio_service["access_key"], secret_key=minio_service["secret_key"])
     return s3_client
 
+
 @pytest.fixture(scope="function")
 def mock_files_factory(tmpdir_factory):
     def _create_files(count):
         filepaths = []
         for _i in range(count):
             name = str(uuid.uuid4())
-            filepath = os.path.normpath(str(tmpdir_factory.mktemp('data').join(name + ".txt")))
+            filepath = os.path.normpath(
+                str(tmpdir_factory.mktemp('data').join(name + ".txt")))
             with open(filepath, 'w') as fout:
                 fout.write("Hello world\n")
             filepaths.append(filepath)
@@ -198,10 +211,11 @@ def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory):
     bucket_name = BUCKET_NAME
     s3_client.create_bucket(bucket_name, delete_contents_if_exists=True)
 
-    #TODO: use pip install Faker
-    users = [ 'alice', 'bob', 'chuck', 'dennis']
+    # TODO: use pip install Faker
+    users = ['alice', 'bob', 'chuck', 'dennis']
 
-    projects = ['astronomy', 'biology', 'chemistry', 'dermatology', 'economics', 'futurology', 'geology']
+    projects = ['astronomy', 'biology', 'chemistry',
+                'dermatology', 'economics', 'futurology', 'geology']
     location = SIMCORE_S3_STR
 
     nodes = ['alpha', 'beta', 'gamma', 'delta']
@@ -214,41 +228,43 @@ def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory):
         idx = randrange(len(users))
         user_name = users[idx]
         user_id = idx + 10
-        idx =  randrange(len(projects))
+        idx = randrange(len(projects))
         project_name = projects[idx]
         project_id = idx + 100
-        idx =  randrange(len(nodes))
+        idx = randrange(len(nodes))
         node = nodes[idx]
         node_id = idx + 10000
         file_name = str(counter)
-        object_name = Path(str(project_id), str(node_id), str(counter)).as_posix()
+        object_name = Path(str(project_id), str(
+            node_id), str(counter)).as_posix()
         file_uuid = Path(object_name).as_posix()
 
         assert s3_client.upload_file(bucket_name, object_name, _file)
 
-        d = { 'file_uuid' : file_uuid,
-              'location_id' : "0",
-              'location' : location,
-              'bucket_name' : bucket_name,
-              'object_name' : object_name,
-              'project_id' : str(project_id),
-              'project_name' : project_name,
-              'node_id' : str(node_id),
-              'node_name' : node,
-              'file_name' : file_name,
-              'user_id' : str(user_id),
-              'user_name' : user_name
-            }
+        d = {'file_uuid': file_uuid,
+             'location_id': "0",
+             'location': location,
+             'bucket_name': bucket_name,
+             'object_name': object_name,
+             'project_id': str(project_id),
+             'project_name': project_name,
+             'node_id': str(node_id),
+             'node_name': node,
+             'file_name': file_name,
+             'user_id': str(user_id),
+             'user_name': user_name
+             }
 
         counter = counter + 1
 
         data[object_name] = FileMetaData(**d)
 
-        utils.insert_metadata(postgres_service_url, data[object_name]) #pylint: disable=no-member
-
+        # pylint: disable=no-member
+        utils.insert_metadata(postgres_service_url,
+                              data[object_name])
 
     total_count = 0
-    for _obj in s3_client.list_objects_v2(bucket_name, recursive = True):
+    for _obj in s3_client.list_objects_v2(bucket_name, recursive=True):
         total_count = total_count + 1
 
     assert total_count == N
@@ -260,10 +276,6 @@ def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory):
     # db
     utils.drop_tables(url=postgres_service_url)
 
-# This is weird, somehow the default loop gives problems with pytest asyncio, so lets override it
-@pytest.fixture
-def loop(event_loop):
-    return event_loop
 
 @pytest.fixture(scope="function")
 async def datcore_testbucket(loop, python27_exec, mock_files_factory):
@@ -282,19 +294,20 @@ async def datcore_testbucket(loop, python27_exec, mock_files_factory):
 
     ready = False
     counter = 0
-    while not ready and counter<5:
+    while not ready and counter < 5:
         data = await dcw.list_files()
         ready = len(data) == 2
         await asyncio.sleep(10)
         counter = counter + 1
 
-
     yield BUCKET_NAME
 
     await dcw.delete_test_dataset(BUCKET_NAME)
 
+
 @pytest.fixture(scope="function")
 def dsm_fixture(s3_client, python27_exec, postgres_engine, loop):
     pool = ThreadPoolExecutor(3)
-    dsm_fixture = DataStorageManager(s3_client, python27_exec, postgres_engine, loop, pool, BUCKET_NAME)
+    dsm_fixture = DataStorageManager(
+        s3_client, python27_exec, postgres_engine, loop, pool, BUCKET_NAME)
     return dsm_fixture

services/storage/tests/requirements.txt

@@ -5,9 +5,11 @@
 coveralls
 pytest
 pytest-aiohttp
-pytest-asyncio
 pytest-cov
 pytest-docker
 openapi_spec_validator
 pyyaml
 virtualenv
+
+# NOTE: pytest-aiohttp and pytest-asyncio incompatible
+# https://github.com/pytest-dev/pytest-asyncio/issues/76

services/storage/tests/test_dsm.py

@@ -1,7 +1,9 @@
-# TODO: W0611:Unused import ...
-# pylint: disable=W0611
-# TODO: W0613:Unused argument ...
-# pylint: disable=W0613
+# pylint:disable=wildcard-import
+# pylint:disable=unused-import
+# pylint:disable=unused-variable
+# pylint:disable=unused-argument
+# pylint:disable=redefined-outer-name
+# pylint: disable=too-many-arguments
 
 import filecmp
 import io
@@ -10,17 +12,17 @@
 import pdb
 import urllib
 import uuid
-from pprint import pprint
-
 from pathlib import Path
+from pprint import pprint
 
 import attr
 import pytest
 
 import utils
 from simcore_service_storage.dsm import DataStorageManager
 from simcore_service_storage.models import FileMetaData
-from simcore_service_storage.s3 import DATCORE_STR, SIMCORE_S3_STR, SIMCORE_S3_ID
+from simcore_service_storage.s3 import (DATCORE_STR, SIMCORE_S3_ID,
+                                        SIMCORE_S3_STR)
 from utils import BUCKET_NAME
 
 
@@ -176,7 +178,6 @@ async def test_copy_s3_s3(postgres_service_url, s3_client, mock_files_factory, d
     assert len(data) == 2
 
 #NOTE: Below tests directly access the datcore platform, use with care!
-
 @pytest.mark.travis
 def test_datcore_fixture(datcore_testbucket):
     print(datcore_testbucket)
@@ -198,8 +199,6 @@ async def test_dsm_datcore(postgres_service_url, dsm_fixture, datcore_testbucket
     data = await dsm.list_files(user_id=user_id, location=DATCORE_STR)
     assert len(data) == 1
 
-# pylint: disable=R0913
-# Too many arguments
 @pytest.mark.travis
 async def test_dsm_s3_to_datcore(postgres_service_url, s3_client, mock_files_factory, dsm_fixture, datcore_testbucket):
     utils.create_tables(url=postgres_service_url)
@@ -230,8 +229,6 @@ async def test_dsm_s3_to_datcore(postgres_service_url, s3_client, mock_files_fac
     # there should now be 3 files
     assert len(data) == 3
 
-# pylint: disable=R0913
-# Too many arguments
 @pytest.mark.travis
 async def test_dsm_datcore_to_local(postgres_service_url, dsm_fixture, mock_files_factory, datcore_testbucket):
     utils.create_tables(url=postgres_service_url)
@@ -250,8 +247,6 @@ async def test_dsm_datcore_to_local(postgres_service_url, dsm_fixture, mock_file
 
     assert filecmp.cmp(tmp_file2, tmp_file)
 
-# pylint: disable=R0913
-# Too many arguments
 @pytest.mark.travis
 async def test_dsm_datcore_to_S3(postgres_service_url, s3_client, dsm_fixture, mock_files_factory, datcore_testbucket):
     utils.create_tables(url=postgres_service_url)
@@ -287,10 +282,6 @@ async def test_dsm_datcore_to_S3(postgres_service_url, s3_client, dsm_fixture, m
 
     assert filecmp.cmp(tmp_file1, tmp_file2)
 
-
-
-# pylint: disable=R0913
-# Too many arguments
 @pytest.mark.travis
 async def test_copy_datcore(postgres_service_url, s3_client, dsm_fixture, mock_files_factory, datcore_testbucket):
     utils.create_tables(url=postgres_service_url)