Is369/storage tokens (#380)

* storage can access the user's datcore tokens info
 -  storage service retrieves user token's key and secret when it needs to operate on external services
 -  finishes implementation calls to list datcore from wrapper
 -  fixes on front-end (@odeimaiz )

* closes #369

* merged squashed is309/storage-api-spec from PR #366 (pending review but tests passing)

* Removed session from storage. not needed.

* Cleanup storage submodule in web-server
 - adds body in request

* Injected datcore tokens into dsm

* renamed:    db_helpers.py -> db_tokens.py


* Fixed dev-config in storage (does not have entire root folder inside of the container!)
Added acces stdin_open and tty to attach for debuggin


* Fixes pytest warning
 "cannot collect test class 'TestClient' because it has a __init__ constructor"

Pytest tries to collect TestClient as its name matches the naming conventions for test classes. However it finds a __init__ method, which it cannot understand.

* Modified storage config schemas on datcore test tokens

* Does not update dsm.datcore_tokens if not not available

* Fixes case of empty token match

* Injects dummy tokens table to init postgres fixture
userid is an integer but not a string
removed deprecated @pytest.mark.asyncio

Author: pcrespov

packages/simcore-sdk/tests/fixtures/postgres.py

@@ -2,11 +2,22 @@
 import os
 
 import pytest
-from pytest_docker import docker_ip, docker_services  # pylint:disable=W0611
 import sqlalchemy as sa
+from pytest_docker import docker_ip, docker_services  # pylint:disable=W0611
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
+#
+# FIXME: this should be in sync with the original
+# which is owned by the webserver???
+_metadata = sa.MetaData()
+_tokens = sa.Table("tokens", _metadata,
+    sa.Column("token_id", sa.BigInteger, nullable=False, primary_key=True),
+    sa.Column("user_id", sa.BigInteger, nullable=False),
+    sa.Column("token_service", sa.String, nullable=False),
+    sa.Column("token_data", sa.JSON, nullable=False),
+)
+
 
 def is_responsive(url):
     """Check if there is a db"""
@@ -22,7 +33,7 @@ def is_responsive(url):
 
 # pylint:disable=redefined-outer-name
 @pytest.fixture(scope="module")
-def engine(docker_ip, docker_services): 
+def engine(docker_ip, docker_services):
     dbname = 'test'
     user = 'user'
     password = 'pwd'
@@ -42,7 +53,10 @@ def engine(docker_ip, docker_services):
         pause=1.0,
     )
 
+    # Configures db and initializes tables
+    # Uses syncrounous engine for that
     engine = create_engine(url, client_encoding='utf8')
+    _metadata.create_all(bind=engine, tables=[_tokens, ], checkfirst=True)
 
     os.environ["POSTGRES_ENDPOINT"]="{host}:{port}".format(host=host, port=port)
     os.environ["POSTGRES_USER"]="user"
@@ -51,7 +65,8 @@ def engine(docker_ip, docker_services):
     yield engine
     # cleanup
     engine.dispose()
-    
+
+
 
 # pylint:disable=redefined-outer-name
 @pytest.fixture(scope="module")
@@ -62,4 +77,3 @@ def session(engine):
     yield session
     #cleanup
     session.close()
-    

packages/simcore-sdk/tests/node_ports/conftest.py

@@ -14,8 +14,9 @@
 
 
 @pytest.fixture
-def user_id()->str:
-    yield "testuser"
+def user_id()->int:
+    # see fixtures/postgres.py
+    yield 1258
 
 @pytest.fixture
 def s3_simcore_location() ->str:
@@ -25,8 +26,8 @@ def s3_simcore_location() ->str:
 def filemanager_cfg(storage, user_id, bucket):
     storage_endpoint = yarl.URL(storage)
     node_config.STORAGE_ENDPOINT = "{}:{}".format(storage_endpoint.host, storage_endpoint.port)
-    node_config.USER_ID = user_id    
-    node_config.BUCKET = bucket    
+    node_config.USER_ID = user_id
+    node_config.BUCKET = bucket
     yield
 
 @pytest.fixture
@@ -39,12 +40,12 @@ def node_uuid()->str:
 
 @pytest.fixture
 def file_uuid(project_id, node_uuid)->str:
-    def create(file_path:Path, project:str=None, node:str=None):  
+    def create(file_path:Path, project:str=None, node:str=None):
         if project is None:
             project = project_id
         if node is None:
-            node = node_uuid            
-        return helpers.file_uuid(file_path, project, node)              
+            node = node_uuid
+        return helpers.file_uuid(file_path, project, node)
     yield create
 
 @pytest.fixture(scope='session')
@@ -78,7 +79,7 @@ def postgres(engine, session):
 def default_configuration(postgres, default_configuration_file, project_id, node_uuid):
     # prepare database with default configuration
     json_configuration = default_configuration_file.read_text()
-    
+
     _create_new_pipeline(postgres, project_id)
     _set_configuration(postgres, project_id, node_uuid, json_configuration)
     config_dict = json.loads(json_configuration)
@@ -129,7 +130,7 @@ def create_config(inputs: List[Tuple[str, str, Any]] =None, outputs: List[Tuple[
 @pytest.fixture(scope="function")
 def special_2nodes_configuration(postgres, empty_configuration_file: Path, project_id, node_uuid):
     def create_config(prev_node_inputs: List[Tuple[str, str, Any]] =None, prev_node_outputs: List[Tuple[str, str, Any]] =None,
-                    inputs: List[Tuple[str, str, Any]] =None, outputs: List[Tuple[str, str, Any]] =None, 
+                    inputs: List[Tuple[str, str, Any]] =None, outputs: List[Tuple[str, str, Any]] =None,
                     project_id:str =project_id, previous_node_id:str = node_uuid, node_id:str = "asdasdadsa"):
         _create_new_pipeline(postgres, project_id)
 
@@ -157,7 +158,7 @@ def create_config(prev_node_inputs: List[Tuple[str, str, Any]] =None, prev_node_
     postgres.query(ComputationalPipeline).delete()
     postgres.commit()
 
-def _create_new_pipeline(session, project:str)->str:    
+def _create_new_pipeline(session, project:str)->str:
     new_Pipeline = ComputationalPipeline(project_id=project)
     session.add(new_Pipeline)
     session.commit()
@@ -170,7 +171,7 @@ def _set_configuration(session, project_id: str, node_id:str, json_configuration
 
     new_Node = ComputationalTask(project_id=project_id, node_id=node_uuid, schema=configuration["schema"], inputs=configuration["inputs"], outputs=configuration["outputs"])
     session.add(new_Node)
-    session.commit()    
+    session.commit()
     return node_uuid
 
 def _assign_config(config_dict:dict, port_type:str, entries: List[Tuple[str, str, Any]]):

packages/simcore-sdk/tests/node_ports/test_filemanager.py

@@ -7,12 +7,12 @@
 from simcore_sdk.node_ports import exceptions, filemanager
 
 
-@pytest.mark.asyncio
+
 async def test_valid_upload_download(tmpdir, bucket, storage, filemanager_cfg, user_id, file_uuid, s3_simcore_location):
     file_path = Path(tmpdir) / "test.test"
     file_path.write_text("I am a test file")
     assert file_path.exists()
-    
+
     file_id = file_uuid(file_path)
     store = s3_simcore_location
     await filemanager.upload_file(store_id=store, s3_object=file_id, local_file_path=file_path)
@@ -22,14 +22,14 @@ async def test_valid_upload_download(tmpdir, bucket, storage, filemanager_cfg, u
     assert download_file_path.exists()
 
     assert filecmp.cmp(download_file_path, file_path)
-    
-@pytest.mark.asyncio
+
+
 async def test_invalid_file_path(tmpdir, bucket, storage, filemanager_cfg, user_id, file_uuid, s3_simcore_location):
     file_path = Path(tmpdir) / "test.test"
     file_path.write_text("I am a test file")
     assert file_path.exists()
-    
-    
+
+
     file_id = file_uuid(file_path)
     store = s3_simcore_location
     with pytest.raises(FileNotFoundError):
@@ -39,12 +39,12 @@ async def test_invalid_file_path(tmpdir, bucket, storage, filemanager_cfg, user_
     with pytest.raises(exceptions.S3InvalidPathError):
         await filemanager.download_file(store_id=store, s3_object=file_id, local_file_path=download_file_path)
 
-@pytest.mark.asyncio
+
 async def test_invalid_fileid(tmpdir, bucket, storage, filemanager_cfg, user_id, s3_simcore_location):
     file_path = Path(tmpdir) / "test.test"
     file_path.write_text("I am a test file")
     assert file_path.exists()
-    
+
     store = s3_simcore_location
     with pytest.raises(exceptions.StorageInvalidCall):
         await filemanager.upload_file(store_id=store, s3_object="", local_file_path=file_path)
@@ -56,8 +56,8 @@ async def test_invalid_fileid(tmpdir, bucket, storage, filemanager_cfg, user_id,
         await filemanager.download_file(store_id=store, s3_object="", local_file_path=download_file_path)
     with pytest.raises(exceptions.S3InvalidPathError):
         await filemanager.download_file(store_id=store, s3_object="file_id", local_file_path=download_file_path)
-    
-@pytest.mark.asyncio
+
+
 async def test_invalid_store(tmpdir, bucket, storage, filemanager_cfg, user_id, file_uuid, s3_simcore_location):
     file_path = Path(tmpdir) / "test.test"
     file_path.write_text("I am a test file")
@@ -72,6 +72,6 @@ async def test_invalid_store(tmpdir, bucket, storage, filemanager_cfg, user_id,
     with pytest.raises(exceptions.S3InvalidStore):
         await filemanager.download_file(store_name=store, s3_object=file_id, local_file_path=download_file_path)
 
-@pytest.mark.asyncio
+
 async def test_storage_sdk_client(storage):
-    pass
+    pass

services/director/src/simcore_service_director/system_utils.py

@@ -5,7 +5,7 @@
 def get_system_extra_hosts(extra_host_domain: str) -> Dict:
     extra_hosts = {}
     hosts_path = Path("/etc/hosts")
-    if hosts_path.exists() and extra_host_domain != "undefined":        
+    if hosts_path.exists() and extra_host_domain != "undefined":
         with hosts_path.open() as hosts:
             for line in hosts:
                 if extra_host_domain in line:

services/storage/src/simcore_service_storage/application.py

@@ -9,7 +9,6 @@
 from .s3 import setup_s3
 from .db import setup_db
 from .rest import setup_rest
-from .session import setup_session
 from .settings import APP_CONFIG_KEY
 from .dsm import setup_dsm
 
@@ -22,7 +21,6 @@ def create(config):
     app[APP_CONFIG_KEY] = config
 
     setup_db(app)
-    setup_session(app)
     setup_rest(app)
     setup_s3(app)
     setup_dsm(app)

services/storage/src/simcore_service_storage/cli.py

@@ -40,7 +40,6 @@ def create_environ(skip_system_environ=False):
     if rootdir is not None:
         environ['OSPARC_SIMCORE_REPO_ROOTDIR'] = str(rootdir)
 
-
     return environ
 
 
@@ -72,7 +71,6 @@ def main(args=None):
 
     log_level = config["main"]["log_level"]
     logging.basicConfig(level=getattr(logging, log_level))
-    print("hello, iam")
 
     application.run(config)
 

services/storage/src/simcore_service_storage/config_schema.py

@@ -12,8 +12,8 @@
     "python2": T.String(),
     T.Key("max_workers", default=8, optional=True) : T.Int(),
     T.Key("test_datcore", optional=True): T.Dict({
-        "api_token": T.String(),
-        "api_secret": T.String()
+        "token_key": T.String(),
+        "token_secret": T.String()
     }),
     T.Key("disable_services", default=[], optional=True): T.List(T.String())
 })

services/storage/src/simcore_service_storage/data/docker-dev-config.yaml

@@ -7,8 +7,8 @@ main:
   max_workers: 8
   python2: ${VENV2}
   test_datcore:
-     api_token: ${BF_API_KEY}
-     api_secret: ${BF_API_SECRET}
+     token_key: ${BF_API_KEY}
+     token_secret: ${BF_API_SECRET}
 postgres:
   database: ${POSTGRES_DB}
   endpoint: ${POSTGRES_ENDPOINT}

services/storage/src/simcore_service_storage/data/docker-prod-config.yaml

@@ -6,8 +6,8 @@ main:
   testing: True
   python2: ${VENV2}
   test_datcore:
-     api_token: ${BF_API_KEY}
-     api_secret: ${BF_API_SECRET}
+     token_key: ${BF_API_KEY}
+     token_secret: ${BF_API_SECRET}
 postgres:
   database: ${POSTGRES_DB}
   endpoint: ${POSTGRES_ENDPOINT}

services/storage/src/simcore_service_storage/data/host-dev-config.yaml

@@ -7,8 +7,8 @@ main:
   testing: true
   python2: ${VENV2}
   test_datcore:
-     api_token: ${BF_API_KEY}
-     api_secret: ${BF_API_SECRET}
+     token_key: ${BF_API_KEY}
+     token_secret: ${BF_API_SECRET}
 postgres:
   database: simcoredb
   endpoint: postgres:5432

services/storage/src/simcore_service_storage/db_tokens.py

@@ -0,0 +1,37 @@
+
+from typing import Tuple
+
+import sqlalchemy as sa
+from aiohttp import web
+
+from .settings import APP_DB_ENGINE_KEY, APP_CONFIG_KEY
+
+# FIXME: this is a temporary solution DO NOT USE. This table needs to be in sync
+# with services/web/server/src/simcore_service_webserver/db_models.py
+_metadata = sa.MetaData()
+_tokens = sa.Table("tokens", _metadata,
+    sa.Column("token_id", sa.BigInteger, nullable=False, primary_key=True),
+    sa.Column("user_id", sa.BigInteger, nullable=False),
+    sa.Column("token_service", sa.String, nullable=False),
+    sa.Column("token_data", sa.JSON, nullable=False),
+)
+
+
+async def get_api_token_and_secret(request: web.Request, userid) -> Tuple[str, str]:
+    # FIXME: this is a temporary solution. This information should be sent in some form
+    # from the client side together with the userid?
+    engine = request.app.get(APP_DB_ENGINE_KEY, None)
+    data = {}
+    if engine:
+        async with engine.acquire() as conn:
+            stmt = sa.select([_tokens, ]).where(_tokens.c.user_id==userid)
+            result = await conn.execute(stmt)
+            row = await result.first()
+            data =  dict(row) if row else {}
+
+    defaults = request.app[APP_CONFIG_KEY]["main"].get("test_datacore", {})
+
+    api_token = data.get('token_data', {}).get('token_key', defaults.get('api_token'))
+    api_secret = data.get('token_data',{}).get('token_secret', defaults.get('api_secret'))
+
+    return api_token, api_secret