From 021a89c40f38c387d6739ce663337402828468ba Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 16:22:59 +0200 Subject: [PATCH 01/33] Infra fixes --- requirements.txt | 1 - services/api-gateway/Makefile | 11 +++++++---- services/api-gateway/requirements/dev.txt | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 326949ec9ef..6fd16078a84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ # # $ make devenv # -pylint black pip-tools rope diff --git a/services/api-gateway/Makefile b/services/api-gateway/Makefile index 3540dc0536d..c2f63c6fa64 100644 --- a/services/api-gateway/Makefile +++ b/services/api-gateway/Makefile @@ -13,8 +13,9 @@ export APP_VERSION = $(shell cat VERSION) reqs: ## compiles pip requirements (.in -> .txt) @$(MAKE) --directory requirements reqs + .PHONY: install-dev install-prod install-ci -install-dev install-prod install-ci: _check-venv-active ## install app in development/production or CI mode +install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode # installing in $(subst install-,,$@) mode pip-sync requirements/$(subst install-,,$@).txt @@ -34,16 +35,17 @@ tests-integration: ## runs integration tests against local+production images .PHONY: run-devel down -run-devel: down ## runs app with pg fixture for development +run-devel: down ## runs app on host with pg fixture for development # starting pg database ... - docker-compose -f $(CURDIR)/tests/unit/with_dbs/docker-compose.yml up --detach + docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml up --detach # starting service export SECRET_KEY="d0d0397de2c85ad26ffd4a0f9643dfe3a0ca3937f99cf3c2e174e11b5ef79880"; \ uvicorn simcore_service_api_gateway.main:the_app --reload --port=8001 --host=0.0.0.0 # stop down: ## stops pg fixture - docker-compose -f $(CURDIR)/tests/unit/with_dbs/docker-compose.yml down + docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml down + .PHONY: build build: ## builds docker image (using main services/docker-compose-build.yml) @@ -51,6 +53,7 @@ build: ## builds docker image (using main services/docker-compose-build.yml) .PHONY: replay +# TODO: replay shall point to online cookiecutter replay: .cookiecutterrc ## re-applies cookiecutter # Replaying /home/crespo/devp/osparc-simcore/services/api-gateway/../../../cookiecutter-simcore-py-fastapi ... @cookiecutter --no-input --overwrite-if-exists \ diff --git a/services/api-gateway/requirements/dev.txt b/services/api-gateway/requirements/dev.txt index 2549941088c..41c6eaf4cd5 100644 --- a/services/api-gateway/requirements/dev.txt +++ b/services/api-gateway/requirements/dev.txt @@ -11,3 +11,6 @@ # installs current package -e . + +# common dev-tools as well +-r ../../../requirements.txt From 50f268734615aeb9a4c67aafd743b0292b8c6583 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 16:43:56 +0200 Subject: [PATCH 02/33] Fixes make run-devel - sets up a system with environs for devel - --- services/api-gateway/.env-devel | 11 ++++++++ services/api-gateway/Makefile | 10 +++---- .../simcore_service_api_gateway/settings.py | 2 ++ .../tests/utils/docker-compose.yml | 27 +++++++++++++++++++ 4 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 services/api-gateway/.env-devel create mode 100644 services/api-gateway/tests/utils/docker-compose.yml diff --git a/services/api-gateway/.env-devel b/services/api-gateway/.env-devel new file mode 100644 index 00000000000..6fe85f17c53 --- /dev/null +++ b/services/api-gateway/.env-devel @@ -0,0 +1,11 @@ +# +# Environment variables used to configure this service +# + +# SEE services/api-gateway/src/simcore_service_api_gateway/auth_security.py +SECRET_KEY=d0d0397de2c85ad26ffd4a0f9643dfe3a0ca3937f99cf3c2e174e11b5ef79880 + +# SEE services/api-gateway/src/simcore_service_api_gateway/settings.py +POSTGRES_USER=test +POSTGRES_PASSWORD=test +POSTGRES_DB=test diff --git a/services/api-gateway/Makefile b/services/api-gateway/Makefile index c2f63c6fa64..fcf54ffc964 100644 --- a/services/api-gateway/Makefile +++ b/services/api-gateway/Makefile @@ -35,13 +35,11 @@ tests-integration: ## runs integration tests against local+production images .PHONY: run-devel down -run-devel: down ## runs app on host with pg fixture for development - # starting pg database ... - docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml up --detach - # starting service - export SECRET_KEY="d0d0397de2c85ad26ffd4a0f9643dfe3a0ca3937f99cf3c2e174e11b5ef79880"; \ +run-devel: .env-devel down ## runs app on host with pg fixture for development + export $(shell grep -v '^#' $< | xargs -d '\n'); \ + docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml up --detach; \ uvicorn simcore_service_api_gateway.main:the_app --reload --port=8001 --host=0.0.0.0 - # stop + down: ## stops pg fixture docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml down diff --git a/services/api-gateway/src/simcore_service_api_gateway/settings.py b/services/api-gateway/src/simcore_service_api_gateway/settings.py index 79b49666046..0890b750fa3 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/settings.py +++ b/services/api-gateway/src/simcore_service_api_gateway/settings.py @@ -1,5 +1,7 @@ # pylint: disable=no-name-in-module +# NOTE: SEE https://pydantic-docs.helpmanual.io/usage/settings/ for usage + from pydantic import BaseSettings, Field, SecretStr, validator from enum import Enum from typing import Optional diff --git a/services/api-gateway/tests/utils/docker-compose.yml b/services/api-gateway/tests/utils/docker-compose.yml new file mode 100644 index 00000000000..a5afea1eca1 --- /dev/null +++ b/services/api-gateway/tests/utils/docker-compose.yml @@ -0,0 +1,27 @@ +version: '3.4' +services: + postgres: + image: postgres:10 + environment: + - POSTGRES_USER=${POSTGRES_USER:-test} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test} + - POSTGRES_DB=${POSTGRES_PASSWORD:-test} + - POSTGRES_HOST=${POSTGRES_HOST:-localhost} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + ports: + - "5432:5432" + # https://www.postgresql.org/docs/10/runtime-config-logging.html#GUC-LOG-STATEMENT + command: + [ + "postgres", + "-c", "log_connections=true", + "-c", "log_disconnections=true", + "-c", "log_duration=true", + "-c", "log_line_prefix=[%p] [%a] [%c] [%x] " + ] + adminer: + image: adminer + ports: + - 18080:8080 + depends_on: + - postgres From 29c4d358fcb5a99159409ea0cdf197c013260862 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 17:24:49 +0200 Subject: [PATCH 03/33] api_version_prefix by api_vtag --- .../simcore_service_api_gateway/__version__.py | 2 +- .../simcore_service_api_gateway/application.py | 5 +++-- .../src/simcore_service_api_gateway/auth.py | 4 ++-- .../endpoints_check.py | 18 ++++++++++++++---- .../src/simcore_service_api_gateway/main.py | 6 +++--- .../tools/templates/test_endpoints.py.jinja2 | 14 +++++++------- 6 files changed, 30 insertions(+), 19 deletions(-) diff --git a/services/api-gateway/src/simcore_service_api_gateway/__version__.py b/services/api-gateway/src/simcore_service_api_gateway/__version__.py index 2b880c7eec4..4c9375a1028 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/__version__.py +++ b/services/api-gateway/src/simcore_service_api_gateway/__version__.py @@ -7,4 +7,4 @@ major, minor, patch = __version__.split(".") api_version = __version__ -api_version_prefix: str = f"v{major}" +api_vtag: str = f"v{major}" diff --git a/services/api-gateway/src/simcore_service_api_gateway/application.py b/services/api-gateway/src/simcore_service_api_gateway/application.py index c0d0a7ecc8b..0268bd71aa9 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/application.py +++ b/services/api-gateway/src/simcore_service_api_gateway/application.py @@ -1,5 +1,6 @@ """ Helpers wrapping or producing FastAPI's app + These helpers are typically used with main.the_app singleton instance """ import json from pathlib import Path @@ -8,7 +9,7 @@ import yaml from fastapi import FastAPI -from .__version__ import api_version, api_version_prefix +from .__version__ import api_version, api_vtag from .settings import AppSettings @@ -19,7 +20,7 @@ def create(settings: AppSettings) -> FastAPI: title="Public API Gateway", description="Platform's API Gateway for external clients", version=api_version, - openapi_url=f"/api/{api_version_prefix}/openapi.json", + openapi_url=f"/api/{api_vtag}/openapi.json", ) app.state.settings = settings diff --git a/services/api-gateway/src/simcore_service_api_gateway/auth.py b/services/api-gateway/src/simcore_service_api_gateway/auth.py index 5aabd005453..f8e72cf5a7a 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/auth.py +++ b/services/api-gateway/src/simcore_service_api_gateway/auth.py @@ -5,7 +5,7 @@ from fastapi.security import OAuth2PasswordBearer, SecurityScopes from . import crud_users as crud -from .__version__ import api_version_prefix +from .__version__ import api_vtag from .auth_security import get_access_token_data from .schemas import TokenData, User, UserInDB @@ -37,7 +37,7 @@ # callable with request as argument -> extracts token from Authentication header oauth2_scheme = OAuth2PasswordBearer( - tokenUrl=f"{api_version_prefix}/token", + tokenUrl=f"{api_vtag}/token", scopes={ "me": "Read information about the current user.", "projects": "Read projects.", diff --git a/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py b/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py index 3a65c423c87..3049e267a4c 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py +++ b/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py @@ -1,14 +1,24 @@ from fastapi import APIRouter -from .__version__ import __version__, api_version +from .__version__ import __version__, api_version, api_vtag router = APIRouter() @router.get("/") -async def healthcheck(): +async def service_info(): return { "name": __name__.split(".")[0], - "version": __version__, - "api_version": api_version, + "version": api_version, + # TODO: a way to get first part of the url?? "version_prefix": f"/{api_vtag}", + # TODO: sync this info + "released": { + api_vtag: api_version + } } + + +@router.get("/health") +async def health_check(): + # TODO: if not, raise ServiceUnavailable (use diagnostic concept as in webserver) + return True diff --git a/services/api-gateway/src/simcore_service_api_gateway/main.py b/services/api-gateway/src/simcore_service_api_gateway/main.py index cdb23024158..375c06b540d 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/main.py +++ b/services/api-gateway/src/simcore_service_api_gateway/main.py @@ -5,7 +5,7 @@ from fastapi import FastAPI from . import application, endpoints_auth, endpoints_check, endpoints_user -from .__version__ import api_version_prefix +from .__version__ import api_vtag from .db import setup_db from .utils.remote_debug import setup_remote_debugging from .settings import AppSettings @@ -32,10 +32,10 @@ def startup_event(): # pylint: disable=unused-variable app.include_router(endpoints_check.router, tags=["check"]) app.include_router( - endpoints_auth.router, tags=["auth"], prefix=f"/{api_version_prefix}" + endpoints_auth.router, tags=["auth"], prefix=f"/{api_vtag}" ) app.include_router( - endpoints_user.router, tags=["users"], prefix=f"/{api_version_prefix}" + endpoints_user.router, tags=["users"], prefix=f"/{api_vtag}" ) # SUBMODULES setups diff --git a/services/api-gateway/tools/templates/test_endpoints.py.jinja2 b/services/api-gateway/tools/templates/test_endpoints.py.jinja2 index a22a229329d..c1cc99fed48 100644 --- a/services/api-gateway/tools/templates/test_endpoints.py.jinja2 +++ b/services/api-gateway/tools/templates/test_endpoints.py.jinja2 @@ -10,7 +10,7 @@ from starlette import status # TODO: app is init globally ... which is bad! -from simcore_service_api_gateway.main import api_version, app, api_version_prefix +from simcore_service_api_gateway.main import api_version, app, api_vtag @pytest.fixture @@ -21,12 +21,12 @@ def client(environ_context, postgres_service): def test_list_{{ rnp }}(client): - response = client.get(f"/{api_version_prefix}/{{ rnp }}") + response = client.get(f"/{api_vtag}/{{ rnp }}") assert response.status_code == status.HTTP_200_OK assert response.json() == [] # inject three dagin - response = client.get(f"/{api_version_prefix}/{{ rnp }}") + response = client.get(f"/{api_vtag}/{{ rnp }}") assert response.status_code == status.HTTP_200_OK # TODO: assert i can list them as dagouts @@ -35,12 +35,12 @@ def test_list_{{ rnp }}(client): def test_standard_operations_on_resource(client, fake_data_dag_in): - response = client.post(f"/{api_version_prefix}/{{ rnp }}", json=fake_data_dag_in) + response = client.post(f"/{api_vtag}/{{ rnp }}", json=fake_data_dag_in) assert response.status_code == status.HTTP_201_CREATED assert response.json() == 1 # list - response = client.get(f"/{api_version_prefix}/{{ rnp }}") + response = client.get(f"/{api_vtag}/{{ rnp }}") assert response.status_code == status.HTTP_200_OK got = response.json() @@ -52,10 +52,10 @@ def test_standard_operations_on_resource(client, fake_data_dag_in): assert data_out["id"] == 1 # extra key, once in db # get - response = client.get(f"/{api_version_prefix}/{{ rnp }}/1") + response = client.get(f"/{api_vtag}/{{ rnp }}/1") assert response.status_code == status.HTTP_200_OK assert response.json() == data_out # delete - response = client.delete(f"/{api_version_prefix}/{{ rnp }}/1") + response = client.delete(f"/{api_vtag}/{{ rnp }}/1") assert response.status_code == status.HTTP_204_NO_CONTENT From 3a80a36210df5ab82cf72978c09f58e5ba2ba9f2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 17:39:57 +0200 Subject: [PATCH 04/33] Fixes start_db not called --- .../api-gateway/src/simcore_service_api_gateway/db.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/api-gateway/src/simcore_service_api_gateway/db.py b/services/api-gateway/src/simcore_service_api_gateway/db.py index 5fc1bee86c2..96f920f4d77 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/db.py +++ b/services/api-gateway/src/simcore_service_api_gateway/db.py @@ -1,6 +1,7 @@ """ Access to postgres service DUMMY! """ +import asyncio import logging from functools import partial from typing import Dict, Optional @@ -78,19 +79,18 @@ async def start_db(app: FastAPI): async def _go(): await setup_engine(app) - #if False: + # if False: # log.info("Creating db tables (testing mode)") # create_tables() - -async def shutdown_db(app: FastAPI): +def shutdown_db(app: FastAPI): # TODO: tmp disabled log.debug("DUMMY: Shutting down db in %s", app) # await teardown_engine(app) def setup_db(app: FastAPI): - app.router.add_event_handler("startup", partial(start_db, app)) + app.router.add_event_handler("startup", asyncio.coroutine(partial(start_db, app))) app.router.add_event_handler("shutdown", partial(shutdown_db, app)) From 6547e7ac3daa942c96465cc6e5ea07a84b9d4d12 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 17:51:42 +0200 Subject: [PATCH 05/33] Add helpers for fastapi --- .../utils/fastapi_shortcuts.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 services/api-gateway/src/simcore_service_api_gateway/utils/fastapi_shortcuts.py diff --git a/services/api-gateway/src/simcore_service_api_gateway/utils/fastapi_shortcuts.py b/services/api-gateway/src/simcore_service_api_gateway/utils/fastapi_shortcuts.py new file mode 100644 index 00000000000..6d28cfade12 --- /dev/null +++ b/services/api-gateway/src/simcore_service_api_gateway/utils/fastapi_shortcuts.py @@ -0,0 +1,31 @@ +""" Thin wrappers around fastapi interface for convenience + + When to add here a function? These are the goals: + - overcome common mistakes + - shortcuts + + And these are the non-goals: + - replace FastAPI interface + +""" +import asyncio +from functools import partial +from typing import Callable + +from fastapi import FastAPI + + +def _wrap_partial(func: Callable, app: FastAPI) -> Callable: + if asyncio.iscoroutinefunction(func): + return asyncio.coroutine(partial(func, app)) + return partial(func, app) + + +def add_event_on_startup(app: FastAPI, func: Callable) -> None: + callback = _wrap_partial(func, app) + app.router.add_event_handler("startup", callback) + + +def add_event_on_shutdown(app: FastAPI, func: Callable) -> None: + callback = _wrap_partial(func, app) + app.router.add_event_handler("shutdown", callback) From e976ef58320d7720ecfc6f2590ea7d4421c75ba9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 17:55:36 +0200 Subject: [PATCH 06/33] Using new shortcuts --- services/api-gateway/src/simcore_service_api_gateway/db.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/api-gateway/src/simcore_service_api_gateway/db.py b/services/api-gateway/src/simcore_service_api_gateway/db.py index 96f920f4d77..5f7fad82bb1 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/db.py +++ b/services/api-gateway/src/simcore_service_api_gateway/db.py @@ -16,6 +16,7 @@ from .application import FastAPI, get_settings from .settings import AppSettings +from .utils.fastapi_shortcuts import add_event_on_shutdown, add_event_on_startup ## from .orm.base import Base @@ -83,6 +84,7 @@ async def _go(): # log.info("Creating db tables (testing mode)") # create_tables() + def shutdown_db(app: FastAPI): # TODO: tmp disabled log.debug("DUMMY: Shutting down db in %s", app) @@ -90,8 +92,8 @@ def shutdown_db(app: FastAPI): def setup_db(app: FastAPI): - app.router.add_event_handler("startup", asyncio.coroutine(partial(start_db, app))) - app.router.add_event_handler("shutdown", partial(shutdown_db, app)) + add_event_on_startup(app, start_db) + add_event_on_shutdown(app, shutdown_db) __all__ = ("Engine", "ResultProxy", "RowProxy", "SAConnection") From 31cffe8c6806fa957835da0ea0efdb5ce7f28dd6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 17:59:47 +0200 Subject: [PATCH 07/33] Fixes logging --- .../api-gateway/src/simcore_service_api_gateway/main.py | 6 +++++- .../simcore_service_api_gateway/utils/fastapi_shortcuts.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/services/api-gateway/src/simcore_service_api_gateway/main.py b/services/api-gateway/src/simcore_service_api_gateway/main.py index 375c06b540d..b7f721a8cdc 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/main.py +++ b/services/api-gateway/src/simcore_service_api_gateway/main.py @@ -21,6 +21,8 @@ def build_app() -> FastAPI: """ app_settings = AppSettings() + logging.root.setLevel(app_settings.loglevel) + app: FastAPI = application.create(settings=app_settings) @app.on_event("startup") @@ -40,7 +42,9 @@ def startup_event(): # pylint: disable=unused-variable # SUBMODULES setups setup_db(app) - # add new here! + # NOTE: add new here! + # ... + @app.on_event("shutdown") def shutdown_event(): # pylint: disable=unused-variable diff --git a/services/api-gateway/src/simcore_service_api_gateway/utils/fastapi_shortcuts.py b/services/api-gateway/src/simcore_service_api_gateway/utils/fastapi_shortcuts.py index 6d28cfade12..450d979745e 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/utils/fastapi_shortcuts.py +++ b/services/api-gateway/src/simcore_service_api_gateway/utils/fastapi_shortcuts.py @@ -2,7 +2,8 @@ When to add here a function? These are the goals: - overcome common mistakes - - shortcuts + - shortcuts to code faster + - replicates rationale in aiohttp And these are the non-goals: - replace FastAPI interface From ef9aa0aeb2a63af7c8a47c237db19cc1a8e0348d Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 19:15:43 +0200 Subject: [PATCH 08/33] Doc and minor cleanup - logging - fixes tests - / and /health untagged --- services/api-gateway/.env-devel | 2 + services/api-gateway/Makefile | 1 - .../application.py | 3 ++ .../src/simcore_service_api_gateway/auth.py | 50 +++++++++---------- .../auth_security.py | 1 + .../src/simcore_service_api_gateway/db.py | 3 +- .../endpoints_check.py | 2 +- .../endpoints_user.py | 8 +-- .../src/simcore_service_api_gateway/main.py | 2 +- .../simcore_service_api_gateway/schemas.py | 3 +- .../tests/unit/test_endpoints_check.py | 5 +- 11 files changed, 38 insertions(+), 42 deletions(-) diff --git a/services/api-gateway/.env-devel b/services/api-gateway/.env-devel index 6fe85f17c53..4f55c26b4b4 100644 --- a/services/api-gateway/.env-devel +++ b/services/api-gateway/.env-devel @@ -6,6 +6,8 @@ SECRET_KEY=d0d0397de2c85ad26ffd4a0f9643dfe3a0ca3937f99cf3c2e174e11b5ef79880 # SEE services/api-gateway/src/simcore_service_api_gateway/settings.py +LOGLEVEL=DEBUG + POSTGRES_USER=test POSTGRES_PASSWORD=test POSTGRES_DB=test diff --git a/services/api-gateway/Makefile b/services/api-gateway/Makefile index fcf54ffc964..b525f54c514 100644 --- a/services/api-gateway/Makefile +++ b/services/api-gateway/Makefile @@ -40,7 +40,6 @@ run-devel: .env-devel down ## runs app on host with pg fixture for development docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml up --detach; \ uvicorn simcore_service_api_gateway.main:the_app --reload --port=8001 --host=0.0.0.0 - down: ## stops pg fixture docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml down diff --git a/services/api-gateway/src/simcore_service_api_gateway/application.py b/services/api-gateway/src/simcore_service_api_gateway/application.py index 0268bd71aa9..02ea9381973 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/application.py +++ b/services/api-gateway/src/simcore_service_api_gateway/application.py @@ -13,6 +13,7 @@ from .settings import AppSettings + def create(settings: AppSettings) -> FastAPI: # factory app = FastAPI( @@ -33,6 +34,8 @@ def get_settings(app: FastAPI) -> AppSettings: def add_startup_handler(app: FastAPI, startup_event: Callable): + # TODO: this is different from fastapi_shortcuts + # Add Callable with and w/o arguments? app.router.add_event_handler("startup", startup_event) diff --git a/services/api-gateway/src/simcore_service_api_gateway/auth.py b/services/api-gateway/src/simcore_service_api_gateway/auth.py index f8e72cf5a7a..3ed8c670a20 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/auth.py +++ b/services/api-gateway/src/simcore_service_api_gateway/auth.py @@ -1,3 +1,27 @@ +""" This submodule includes responsibilities from authorization server + + +--------+ +---------------+ + | |--(A)- Authorization Request ->| Resource | + | | | Owner | Authorization request + | |<-(B)-- Authorization Grant ---| | + | | +---------------+ + | | + | | +---------------+ + | |--(C)-- Authorization Grant -->| Authorization | + | Client | | Server | Token request + | |<-(D)----- Access Token -------| | + | | +---------------+ + | | + | | +---------------+ + | |--(E)----- Access Token ------>| Resource | + | | | Server | + | |<-(F)--- Protected Resource ---| | + +--------+ +---------------+ + + Figure 1: Abstract Protocol Flow +""" +# TODO: this module shall delegate the auth functionality to a separate service + import logging from typing import Optional @@ -11,30 +35,6 @@ log = logging.getLogger(__name__) - -# Resource SERVER ---------------------------------------------- -# -# +--------+ +---------------+ -# | |--(A)- Authorization Request ->| Resource | -# | | | Owner | Authorization request -# | |<-(B)-- Authorization Grant ---| | -# | | +---------------+ -# | | -# | | +---------------+ -# | |--(C)-- Authorization Grant -->| Authorization | -# | Client | | Server | Token request -# | |<-(D)----- Access Token -------| | -# | | +---------------+ -# | | -# | | +---------------+ -# | |--(E)----- Access Token ------>| Resource | -# | | | Server | -# | |<-(F)--- Protected Resource ---| | -# +--------+ +---------------+ -# -# Figure 1: Abstract Protocol Flow - - # callable with request as argument -> extracts token from Authentication header oauth2_scheme = OAuth2PasswordBearer( tokenUrl=f"{api_vtag}/token", @@ -60,7 +60,7 @@ async def get_current_user( headers={"WWW-Authenticate": authenticate_value}, ) - # validates and decode jwt-based access token + # decodes and validates jwt-based access token token_data: Optional[TokenData] = get_access_token_data(access_token) if token_data is None: raise credentials_exception diff --git a/services/api-gateway/src/simcore_service_api_gateway/auth_security.py b/services/api-gateway/src/simcore_service_api_gateway/auth_security.py index f5f4c22f486..51fa730d366 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/auth_security.py +++ b/services/api-gateway/src/simcore_service_api_gateway/auth_security.py @@ -68,6 +68,7 @@ def decode_token(encoded_jwt: str) -> Dict: def get_access_token_data(encoded_jwt: str) -> Optional[TokenData]: """ Decodes and validates JWT and returns TokenData + Returns None, if invalid token """ try: # decode JWT [header.payload.signature] and get payload: diff --git a/services/api-gateway/src/simcore_service_api_gateway/db.py b/services/api-gateway/src/simcore_service_api_gateway/db.py index 5f7fad82bb1..1cc21e3cdfc 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/db.py +++ b/services/api-gateway/src/simcore_service_api_gateway/db.py @@ -1,9 +1,8 @@ """ Access to postgres service DUMMY! """ -import asyncio + import logging -from functools import partial from typing import Dict, Optional import aiopg.sa diff --git a/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py b/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py index 3049e267a4c..380a75733c2 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py +++ b/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py @@ -21,4 +21,4 @@ async def service_info(): @router.get("/health") async def health_check(): # TODO: if not, raise ServiceUnavailable (use diagnostic concept as in webserver) - return True + return diff --git a/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py b/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py index 42cd4c41b94..7634bf972d3 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py +++ b/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, Depends, Security -from .auth import get_current_active_user, get_current_user +from .auth import get_current_active_user from .schemas import User router = APIRouter() @@ -16,9 +16,3 @@ async def list_own_projects( current_user: User = Security(get_current_active_user, scopes=["projects"]) ): return [{"project_id": "Foo", "owner": current_user.username}] - - -@router.get("/status/") -async def read_system_status(current_user: User = Depends(get_current_user)): - print(current_user) - return {"status": "ok"} diff --git a/services/api-gateway/src/simcore_service_api_gateway/main.py b/services/api-gateway/src/simcore_service_api_gateway/main.py index b7f721a8cdc..3dbe7bb81eb 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/main.py +++ b/services/api-gateway/src/simcore_service_api_gateway/main.py @@ -31,7 +31,7 @@ def startup_event(): # pylint: disable=unused-variable setup_remote_debugging() # ROUTES - app.include_router(endpoints_check.router, tags=["check"]) + app.include_router(endpoints_check.router) app.include_router( endpoints_auth.router, tags=["auth"], prefix=f"/{api_vtag}" diff --git a/services/api-gateway/src/simcore_service_api_gateway/schemas.py b/services/api-gateway/src/simcore_service_api_gateway/schemas.py index 53e394b4be4..04801de87e4 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/schemas.py +++ b/services/api-gateway/src/simcore_service_api_gateway/schemas.py @@ -17,8 +17,7 @@ class User(BaseModel): username: str email: str = None full_name: str = None - disabled: bool = None - class UserInDB(User): hashed_password: str + disabled: bool = None diff --git a/services/api-gateway/tests/unit/test_endpoints_check.py b/services/api-gateway/tests/unit/test_endpoints_check.py index e9210c5a595..fe680f38ef6 100644 --- a/services/api-gateway/tests/unit/test_endpoints_check.py +++ b/services/api-gateway/tests/unit/test_endpoints_check.py @@ -30,8 +30,7 @@ def client(monkeypatch) -> TestClient: yield cli -def test_read_healthcheck(client: TestClient): +def test_read_service_info(client: TestClient): response = client.get("/") assert response.status_code == 200 - assert "api_version" in response.json() - assert response.json()["api_version"] == api_version + assert response.json()["version"] == api_version From 59487c336004fc83d10e2fc43952c74203449d43 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 19:16:16 +0200 Subject: [PATCH 09/33] =?UTF-8?q?Bump=20version:=200.1.0=20=E2=86=92=200.1?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/api-gateway/.cookiecutterrc | 2 +- services/api-gateway/VERSION | 2 +- services/api-gateway/setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/api-gateway/.cookiecutterrc b/services/api-gateway/.cookiecutterrc index 4020c82c992..980023f6e13 100644 --- a/services/api-gateway/.cookiecutterrc +++ b/services/api-gateway/.cookiecutterrc @@ -16,5 +16,5 @@ default_context: project_name: 'Public API Gateway' project_short_description: "Platform's API Gateway for external clients" project_slug: 'api-gateway' - version: '0.1.0' + version: '0.1.1' year: '2020' diff --git a/services/api-gateway/VERSION b/services/api-gateway/VERSION index 6c6aa7cb091..6da28dde76d 100644 --- a/services/api-gateway/VERSION +++ b/services/api-gateway/VERSION @@ -1 +1 @@ -0.1.0 \ No newline at end of file +0.1.1 \ No newline at end of file diff --git a/services/api-gateway/setup.cfg b/services/api-gateway/setup.cfg index 9ca92ff7130..173cc1d88d4 100644 --- a/services/api-gateway/setup.cfg +++ b/services/api-gateway/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.0 +current_version = 0.1.1 commit = True tag = True From 268451141c9d56ca2223ad33ba7eb4402788d944 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 19:27:43 +0200 Subject: [PATCH 10/33] improves make info and formatting --- scripts/common.Makefile | 9 +++++---- .../src/simcore_service_api_gateway/application.py | 1 - .../src/simcore_service_api_gateway/endpoints_check.py | 6 ++---- .../api-gateway/src/simcore_service_api_gateway/main.py | 9 ++------- .../src/simcore_service_api_gateway/schemas.py | 1 + 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/scripts/common.Makefile b/scripts/common.Makefile index 3c525204c57..ad37312f937 100644 --- a/scripts/common.Makefile +++ b/scripts/common.Makefile @@ -76,11 +76,12 @@ info: ## displays basic info @echo ' NOW_TIMESTAMP : ${NOW_TIMESTAMP}' @echo ' VCS_URL : ${VCS_URL}' @echo ' VCS_REF : ${VCS_REF}' - # installed + # installed in .venv @pip list - # version - @cat setup.py | grep name= - @cat setup.py | grep version= + # package + -@echo ' name : ' $(shell python ${CURDIR}/setup.py --name) + -@echo ' version : ' $(shell python ${CURDIR}/setup.py --version) + .PHONY: autoformat diff --git a/services/api-gateway/src/simcore_service_api_gateway/application.py b/services/api-gateway/src/simcore_service_api_gateway/application.py index 02ea9381973..ce6e256eb94 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/application.py +++ b/services/api-gateway/src/simcore_service_api_gateway/application.py @@ -13,7 +13,6 @@ from .settings import AppSettings - def create(settings: AppSettings) -> FastAPI: # factory app = FastAPI( diff --git a/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py b/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py index 380a75733c2..1e2b57aa137 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py +++ b/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py @@ -11,10 +11,8 @@ async def service_info(): "name": __name__.split(".")[0], "version": api_version, # TODO: a way to get first part of the url?? "version_prefix": f"/{api_vtag}", - # TODO: sync this info - "released": { - api_vtag: api_version - } + # TODO: sync this info + "released": {api_vtag: api_version}, } diff --git a/services/api-gateway/src/simcore_service_api_gateway/main.py b/services/api-gateway/src/simcore_service_api_gateway/main.py index 3dbe7bb81eb..9f3b490c46d 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/main.py +++ b/services/api-gateway/src/simcore_service_api_gateway/main.py @@ -33,19 +33,14 @@ def startup_event(): # pylint: disable=unused-variable # ROUTES app.include_router(endpoints_check.router) - app.include_router( - endpoints_auth.router, tags=["auth"], prefix=f"/{api_vtag}" - ) - app.include_router( - endpoints_user.router, tags=["users"], prefix=f"/{api_vtag}" - ) + app.include_router(endpoints_auth.router, tags=["auth"], prefix=f"/{api_vtag}") + app.include_router(endpoints_user.router, tags=["users"], prefix=f"/{api_vtag}") # SUBMODULES setups setup_db(app) # NOTE: add new here! # ... - @app.on_event("shutdown") def shutdown_event(): # pylint: disable=unused-variable log.info("Application shutdown") diff --git a/services/api-gateway/src/simcore_service_api_gateway/schemas.py b/services/api-gateway/src/simcore_service_api_gateway/schemas.py index 04801de87e4..f723f8ec0e0 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/schemas.py +++ b/services/api-gateway/src/simcore_service_api_gateway/schemas.py @@ -18,6 +18,7 @@ class User(BaseModel): email: str = None full_name: str = None + class UserInDB(User): hashed_password: str disabled: bool = None From 85f9985e7b654c723833f488f173363cd547acc1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 22 Apr 2020 21:31:16 +0200 Subject: [PATCH 11/33] redoc: adds vender extensions --- .../application.py | 58 ++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/services/api-gateway/src/simcore_service_api_gateway/application.py b/services/api-gateway/src/simcore_service_api_gateway/application.py index ce6e256eb94..0d345c14b3d 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/application.py +++ b/services/api-gateway/src/simcore_service_api_gateway/application.py @@ -3,27 +3,81 @@ These helpers are typically used with main.the_app singleton instance """ import json +import types from pathlib import Path -from typing import Callable +from typing import Callable, Dict import yaml from fastapi import FastAPI +from fastapi.openapi.docs import get_redoc_html +from fastapi.openapi.utils import get_openapi from .__version__ import api_version, api_vtag from .settings import AppSettings +FAVICON = "https://osparc.io/resource/osparc/favicon.png" +LOGO = "https://raw.githubusercontent.com/ITISFoundation/osparc-manual/b809d93619512eb60c827b7e769c6145758378d0/_media/osparc-logo.svg" + + +def _custom_openapi(zelf: FastAPI) -> Dict: + if not zelf.openapi_schema: + openapi_schema = get_openapi( + title=zelf.title, + version=zelf.version, + openapi_version=zelf.openapi_version, + description=zelf.description, + routes=zelf.routes, + openapi_prefix=zelf.openapi_prefix, + ) + + # ReDoc vendor extensions + # SEE https://github.com/Redocly/redoc/blob/master/docs/redoc-vendor-extensions.md + openapi_schema["info"]["x-logo"] = { + "url": LOGO, + "altText": "osparc-simcore logo", + } + + # + # TODO: load code samples add if function is contained in sample + # TODO: See if openapi-cli does this already + # + openapi_schema["paths"]["/"]["get"]["x-code-samples"] = [ + {"lang": "python", "source": "print('hello world')",}, + ] + + zelf.openapi_schema = openapi_schema + return zelf.openapi_schema + + +def _setup_redoc(app: FastAPI): + from fastapi.applications import Request, HTMLResponse + + async def redoc_html(_req: Request) -> HTMLResponse: + return get_redoc_html( + openapi_url=app.openapi_url, + title=app.title + " - redoc", + redoc_favicon_url=FAVICON, + ) + + app.add_route("/redoc", redoc_html, include_in_schema=False) + def create(settings: AppSettings) -> FastAPI: # factory app = FastAPI( debug=settings.debug, title="Public API Gateway", - description="Platform's API Gateway for external clients", + description="osparc-simcore Public RESTful API Specifications", version=api_version, openapi_url=f"/api/{api_vtag}/openapi.json", + redoc_url=None, ) app.state.settings = settings + app.openapi = types.MethodType(_custom_openapi, app) + + _setup_redoc(app) + return app From 820e59ea61f456de20abe222aee415f06fd35630 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 23 Apr 2020 03:51:51 +0200 Subject: [PATCH 12/33] Minor --- services/api-gateway/README.md | 7 +++++++ services/api-gateway/tests/unit/test_endpoints_check.py | 7 ++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/services/api-gateway/README.md b/services/api-gateway/README.md index 99944b52f28..0f60933bad4 100644 --- a/services/api-gateway/README.md +++ b/services/api-gateway/README.md @@ -14,3 +14,10 @@ Platform's API Gateway for external clients [image-version]https://images.microbadger.com/badges/version/itisfoundation/api-gateway.svg [image-commit]:https://images.microbadger.com/badges/commit/itisfoundation/api-gateway.svg + + + +## References + +- [Design patterns for modern web APIs](https://blog.feathersjs.com/design-patterns-for-modern-web-apis-1f046635215) by D. Luecke +- [API Design Guide](https://cloud.google.com/apis/design/) by Google Cloud diff --git a/services/api-gateway/tests/unit/test_endpoints_check.py b/services/api-gateway/tests/unit/test_endpoints_check.py index fe680f38ef6..d4415479041 100644 --- a/services/api-gateway/tests/unit/test_endpoints_check.py +++ b/services/api-gateway/tests/unit/test_endpoints_check.py @@ -1,6 +1,7 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name +# pylint: disable=unused-variable +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name + import pytest from starlette.testclient import TestClient From 36e14c38f86362d34d463950f40b24a0a05d5fed Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 23 Apr 2020 03:52:54 +0200 Subject: [PATCH 13/33] Drafting client api sdk --- .../api-gateway/tests/unit/test_client_sdk.py | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 services/api-gateway/tests/unit/test_client_sdk.py diff --git a/services/api-gateway/tests/unit/test_client_sdk.py b/services/api-gateway/tests/unit/test_client_sdk.py new file mode 100644 index 00000000000..fbb18ac9481 --- /dev/null +++ b/services/api-gateway/tests/unit/test_client_sdk.py @@ -0,0 +1,218 @@ +# pylint: disable=unused-variable +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name +# pylint: disable=protected-access + + +from pprint import pprint +from typing import Dict, List + +import pytest +from starlette.testclient import TestClient + +from simcore_service_api_gateway import application, endpoints_check +from simcore_service_api_gateway.__version__ import api_vtag +from simcore_service_api_gateway.settings import AppSettings + + +@pytest.fixture +def client(monkeypatch) -> TestClient: + monkeypatch.setenv("POSTGRES_USER", "test") + monkeypatch.setenv("POSTGRES_PASSWORD", "test") + monkeypatch.setenv("POSTGRES_DB", "test") + monkeypatch.setenv("LOGLEVEL", "debug") + monkeypatch.setenv("SC_BOOT_MODE", "production") + + # app + test_settings = AppSettings() + app = application.create(settings=test_settings) + + # routes + app.include_router(endpoints_check.router, tags=["check"]) + + # test client: + # Context manager to trigger events: https://fastapi.tiangolo.com/advanced/testing-events/ + with TestClient(app) as cli: + yield cli + + +# DEV --------------------------------------------------------------------- +import attr +import aiohttp + +from yarl import URL +from typing import Optional, Dict, Any +import json + + +# simcore_api_sdk.abc.py +import abc as _abc + + +@attr.s(auto_attribs=True) +class ApiResponse: + status: int + headers: Dict + body: Dict + + +@attr.s(auto_attribs=True) +class ApiConfig: + session: aiohttp.ClientSession + api_key: str = attr.ib(repr=False) + api_secret: str = attr.ib(repr=False) + base_url: URL = URL(f"https://api.osparc.io/{api_vtag}/") + + # TODO: add validation here + + +class API(_abc.ABC): + def __init__(self, cfg: ApiConfig, *, parent=None): + self._cfg = cfg + + async def _make_request( + self, + method: str, + url: str, + *, + url_params: Optional[Dict] = None, + body_params: Optional[Dict] = None, + headers: Optional[Dict] = None, + body: bytes = b"", + **requester_params: Any, + ) -> ApiResponse: + filled_url = self._cfg.base_url # format_url(url, url_params) + + # TODO: it is always json !! + if body_params is not None: + body = json.dumps(body_params) + + resp: aiohttp.ClientResponse = await self._cfg.session.request( + method, filled_url, body, headers, **requester_params + ) + + response = ApiResponse( + status=resp.status, headers=resp.headers, body=await resp.json() + ) + return response + + +# simcore_api_sdk/v0/__init__.py +# from ._openapi import ApiSession + +# simcore_api_sdk/v0/me_api.py +from attr import NOTHING + + +class MeAPI(API): + async def get(self): + pass + + async def update(self, *, name: str = NOTHING, full_name: str = NOTHING): + """ + Only writable fields can be updated + """ + + +@attr.s(auto_attribs=True) +class StudiesAPI(API): + _next_page_token: int = NOTHING + + async def list( + self, + *, + page_size: int = NOTHING, + keep_page_token: bool = False, + order_by: str = NOTHING, + filter_fields: str = NOTHING, + ): + pass + + async def get(self, uid: str): + pass + + async def create(self): + pass + + async def update(self, uid: str, *, from_other=None, **study_fields): + # TODO: how update fields like a.b.c.?? + pass + + async def remove(self, uid: str) -> None: + # wait ?? + pass + + +# simcore_api_sdk/v0/_openapi.py +class ApiSession: + def __init__( + self, api_key: str, api_secret: str, base_url: URL = NOTHING, + ): + # TODO: setup auth here + self.session = aiohttp.ClientSession(auth=None) + + cfg = ApiConfig(self.session, api_key, api_secret, base_url) + self._cfg = cfg + + # API + self.me = MeAPI(cfg) + self.studies = StudiesAPI(cfg) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.session.close() + + +# ---------------------------------------------------- + +async def test_client_sdk(): + # TODO: design SDK for these calls + # TODO: these examples should run test tests and automaticaly added to redoc + + # from simcore_api_sdk.v0 import ApiSession + + async with ApiSession(api_key="1234", api_secret="secret") as api: + + # GET /me is a special resource that is unique + me: Dict = await api.me.get() + pprint(me) + + await api.me.update(name="pcrespov", full_name="Pedro Crespo") + + # corresponds to the studies I have access ?? + + ## https://cloud.google.com/apis/design/standard_methods + + # GET /studies + studies: List[Dict] = await api.studies.list() + + # Implements Pagination: https://cloud.google.com/apis/design/design_patterns#list_pagination + first_studies = await api.studies.list(page_size=3, keep_page_token=True) + assert api.studies._next_page_token != NOTHING + + next_5_studies = await api.studies.list(page_size=5) + + # Results ordering: https://cloud.google.com/apis/design/design_patterns#sorting_order + sorted_studies: List[Dict] = await api.studies.list(order_by="foo desc,bar") + + # List filter field: https://cloud.google.com/apis/design/naming_convention#list_filter_field + studies: List[Dict] = await api.studies.list(filter_fields="foo.zoo, bar") + assert studies[0] + + # GET /studies/{prj_id} + prj: Dict = await api.studies.get("1234") + + # POST /studies + new_prj: Dict = await api.studies.create() + + # PUT or PATCH /studies/{prj_id} + # this is a patch + await api.studies.update(prj.id, description="Bar") + + # this is a put: using copy_from + await api.studies.update(prj.id, copy_from=new_prj) + + # DELETE /studies/{prj_id} + await api.studies.remove(prj.id) From 606e3b839a7f2b96cb18993077d2fdb0f2e58017 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 27 Apr 2020 11:35:33 +0200 Subject: [PATCH 14/33] Kills processes in 8001 upon make down --- services/api-gateway/Makefile | 6 +++++- services/api-gateway/tests/unit/test_client_sdk.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/services/api-gateway/Makefile b/services/api-gateway/Makefile index b525f54c514..3909b8934fd 100644 --- a/services/api-gateway/Makefile +++ b/services/api-gateway/Makefile @@ -36,12 +36,16 @@ tests-integration: ## runs integration tests against local+production images .PHONY: run-devel down run-devel: .env-devel down ## runs app on host with pg fixture for development + # running current app export $(shell grep -v '^#' $< | xargs -d '\n'); \ docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml up --detach; \ uvicorn simcore_service_api_gateway.main:the_app --reload --port=8001 --host=0.0.0.0 down: ## stops pg fixture - docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml down + # stopping extra services + -@docker-compose -f $(CURDIR)/tests/utils/docker-compose.yml down + # killing any process using port 8001 + -@fuser --kill --verbose --namespace tcp 8001 .PHONY: build diff --git a/services/api-gateway/tests/unit/test_client_sdk.py b/services/api-gateway/tests/unit/test_client_sdk.py index fbb18ac9481..869d1559aa0 100644 --- a/services/api-gateway/tests/unit/test_client_sdk.py +++ b/services/api-gateway/tests/unit/test_client_sdk.py @@ -179,6 +179,7 @@ async def test_client_sdk(): me: Dict = await api.me.get() pprint(me) + # can update SOME entries await api.me.update(name="pcrespov", full_name="Pedro Crespo") # corresponds to the studies I have access ?? From 04ce0fce5f74e0d550c7272c89d9553ac4a306fb Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 27 Apr 2020 19:43:45 +0200 Subject: [PATCH 15/33] Defining API entrypoints --- .../application.py | 2 +- .../endpoints_check.py | 6 +-- .../endpoints_studies.py | 53 +++++++++++++++++++ .../endpoints_user.py | 12 ++--- .../src/simcore_service_api_gateway/main.py | 17 ++++-- .../tests/unit/test_endpoints_check.py | 4 +- 6 files changed, 77 insertions(+), 17 deletions(-) create mode 100644 services/api-gateway/src/simcore_service_api_gateway/endpoints_studies.py diff --git a/services/api-gateway/src/simcore_service_api_gateway/application.py b/services/api-gateway/src/simcore_service_api_gateway/application.py index 0d345c14b3d..604e2346ee6 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/application.py +++ b/services/api-gateway/src/simcore_service_api_gateway/application.py @@ -41,7 +41,7 @@ def _custom_openapi(zelf: FastAPI) -> Dict: # TODO: load code samples add if function is contained in sample # TODO: See if openapi-cli does this already # - openapi_schema["paths"]["/"]["get"]["x-code-samples"] = [ + openapi_schema["paths"]["/meta"]["get"]["x-code-samples"] = [ {"lang": "python", "source": "print('hello world')",}, ] diff --git a/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py b/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py index 1e2b57aa137..406c6c1fa99 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py +++ b/services/api-gateway/src/simcore_service_api_gateway/endpoints_check.py @@ -5,8 +5,8 @@ router = APIRouter() -@router.get("/") -async def service_info(): +@router.get("/meta") +async def get_service_metadata(): return { "name": __name__.split(".")[0], "version": api_version, @@ -17,6 +17,6 @@ async def service_info(): @router.get("/health") -async def health_check(): +async def check_service_health(): # TODO: if not, raise ServiceUnavailable (use diagnostic concept as in webserver) return diff --git a/services/api-gateway/src/simcore_service_api_gateway/endpoints_studies.py b/services/api-gateway/src/simcore_service_api_gateway/endpoints_studies.py new file mode 100644 index 00000000000..33adcd50a7d --- /dev/null +++ b/services/api-gateway/src/simcore_service_api_gateway/endpoints_studies.py @@ -0,0 +1,53 @@ +from fastapi import APIRouter, Security + +from .auth import get_current_active_user +from .schemas import User + +router = APIRouter() + + +@router.get("/studies") +async def list_studies( + current_user: User = Security(get_current_active_user, scopes=["projects"]) +): + return [{"project_id": "Foo", "owner": current_user.username}] + + +@router.get("/studies/{study_id}") +async def get_study( + study_id: str, + current_user: User = Security(get_current_active_user, scopes=["projects"]), +): + return [{"project_id": study_id, "owner": current_user.username}] + + +@router.post("/studies") +async def create_study( + current_user: User = Security(get_current_active_user, scopes=["projects"]) +): + return {"project_id": "Foo", "owner": current_user.username} + + +@router.put("/studies/{study_id}") +async def replace_study( + study_id: str, + current_user: User = Security(get_current_active_user, scopes=["projects"]), +): + return {"project_id": study_id, "owner": current_user.username} + + +@router.patch("/studies/{study_id}") +async def update_study( + study_id: str, + current_user: User = Security(get_current_active_user, scopes=["projects"]), +): + return {"project_id": study_id, "owner": current_user.username} + + +@router.delete("/studies/{study_id}") +async def delete_study( + study_id: str, + current_user: User = Security(get_current_active_user, scopes=["projects"]), +): + _data = {"project_id": study_id, "owner": current_user.username} + return None diff --git a/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py b/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py index 7634bf972d3..489b6b17b9a 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py +++ b/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py @@ -6,13 +6,11 @@ router = APIRouter() -@router.get("/users/me/", response_model=User) -async def read_users_me(current_user: User = Depends(get_current_active_user)): +@router.get("/user", response_model=User) +async def get_my_profile(current_user: User = Depends(get_current_active_user)): return current_user -@router.get("/users/me/projects/") -async def list_own_projects( - current_user: User = Security(get_current_active_user, scopes=["projects"]) -): - return [{"project_id": "Foo", "owner": current_user.username}] +@router.patch("/user", response_model=User) +async def update_my_profile(current_user: User = Depends(get_current_active_user)): + return current_user diff --git a/services/api-gateway/src/simcore_service_api_gateway/main.py b/services/api-gateway/src/simcore_service_api_gateway/main.py index 9f3b490c46d..c2168caa9ad 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/main.py +++ b/services/api-gateway/src/simcore_service_api_gateway/main.py @@ -4,11 +4,17 @@ from fastapi import FastAPI -from . import application, endpoints_auth, endpoints_check, endpoints_user +from . import ( + application, + endpoints_auth, + endpoints_check, + endpoints_studies, + endpoints_user, +) from .__version__ import api_vtag from .db import setup_db -from .utils.remote_debug import setup_remote_debugging from .settings import AppSettings +from .utils.remote_debug import setup_remote_debugging current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -33,8 +39,11 @@ def startup_event(): # pylint: disable=unused-variable # ROUTES app.include_router(endpoints_check.router) - app.include_router(endpoints_auth.router, tags=["auth"], prefix=f"/{api_vtag}") - app.include_router(endpoints_user.router, tags=["users"], prefix=f"/{api_vtag}") + app.include_router(endpoints_auth.router, tags=["Token"], prefix=f"/{api_vtag}") + app.include_router(endpoints_user.router, tags=["User"], prefix=f"/{api_vtag}") + app.include_router( + endpoints_studies.router, tags=["Studies"], prefix=f"/{api_vtag}" + ) # SUBMODULES setups setup_db(app) diff --git a/services/api-gateway/tests/unit/test_endpoints_check.py b/services/api-gateway/tests/unit/test_endpoints_check.py index d4415479041..afff4432ff3 100644 --- a/services/api-gateway/tests/unit/test_endpoints_check.py +++ b/services/api-gateway/tests/unit/test_endpoints_check.py @@ -31,7 +31,7 @@ def client(monkeypatch) -> TestClient: yield cli -def test_read_service_info(client: TestClient): - response = client.get("/") +def test_read_service_meta(client: TestClient): + response = client.get("/meta") assert response.status_code == 200 assert response.json()["version"] == api_version From c8f3b2adb6c36ffbc3b1e45cc986559f72f65ddd Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 00:43:07 +0200 Subject: [PATCH 16/33] minimum tests pass --- .../src/simcore_service_api_gateway/endpoints_user.py | 2 +- services/api-gateway/tests/unit/test_client_sdk.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py b/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py index 489b6b17b9a..f1ecd24af79 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py +++ b/services/api-gateway/src/simcore_service_api_gateway/endpoints_user.py @@ -1,4 +1,4 @@ -from fastapi import APIRouter, Depends, Security +from fastapi import APIRouter, Depends from .auth import get_current_active_user from .schemas import User diff --git a/services/api-gateway/tests/unit/test_client_sdk.py b/services/api-gateway/tests/unit/test_client_sdk.py index 869d1559aa0..d3134443653 100644 --- a/services/api-gateway/tests/unit/test_client_sdk.py +++ b/services/api-gateway/tests/unit/test_client_sdk.py @@ -166,7 +166,7 @@ async def __aexit__(self, exc_type, exc, tb): # ---------------------------------------------------- - +@pytest.mark.skip(reason="Under dev") async def test_client_sdk(): # TODO: design SDK for these calls # TODO: these examples should run test tests and automaticaly added to redoc From 5f10ab5a7f57a4c9fce5e5742b7924d6b056d56f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 00:49:34 +0200 Subject: [PATCH 17/33] fixes frontend linter --- .../desktop/preferences/pages/SecurityPage.js | 137 ++++++++++++------ .../desktop/preferences/window/APIKeyBase.js | 46 ++++++ .../preferences/window/CreateAPIKey.js | 66 +++++++++ .../desktop/preferences/window/ShowAPIKey.js | 75 ++++++++++ 4 files changed, 278 insertions(+), 46 deletions(-) create mode 100644 services/web/client/source/class/osparc/desktop/preferences/window/APIKeyBase.js create mode 100644 services/web/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js create mode 100644 services/web/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js diff --git a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js index 3f8093b2c96..f99e43085ec 100644 --- a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js +++ b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js @@ -24,7 +24,7 @@ */ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { - extend:osparc.desktop.preferences.pages.BasePage, + extend: osparc.desktop.preferences.pages.BasePage, construct: function() { const iconSrc = "@FontAwesome5Solid/shield-alt/24"; @@ -32,15 +32,74 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { this.base(arguments, title, iconSrc); this.add(this.__createPasswordSection()); - this.add(this.__createTokensSection()); + this.add(this.__createInternalTokensSection()); + this.add(this.__createExternalTokensSection()); + + this.__rebuildTokensList(); }, members: { - __tokensList: null, + __internalTokensList: null, + __externalTokensList: null, - __createTokensSection: function() { + __createInternalTokensSection: function() { // layout - const box = this._createSectionBox(this.tr("Access Tokens")); + const box = this._createSectionBox(this.tr("oSPARC API Tokens")); + + const label = this._createHelpLabel(this.tr( + "Tokens to access oSPARC API." + )); + box.add(label); + + const tokensList = this.__internalTokensList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); + box.add(tokensList); + + const requestTokenBtn = this.__requestTokenBtn = new osparc.ui.form.FetchButton(this.tr("Create oSPARC Token")).set({ + allowGrowX: false + }); + requestTokenBtn.addListener("execute", () => { + this.__requestOsparcToken(); + }, this); + box.add(requestTokenBtn); + + return box; + }, + + __requestOsparcToken: function() { + if (!osparc.data.Permissions.getInstance().canDo("preferences.token.create", true)) { + return; + } + + const createAPIKeyWindow = new osparc.desktop.preferences.window.CreateAPIKey("hello", "world"); + createAPIKeyWindow.addListener("finished", keyLabel => { + const params = { + data: { + "service": "osparc", + "keyLabel": keyLabel.getData() + } + }; + createAPIKeyWindow.close(); + this.__requestTokenBtn.setFetching(true); + osparc.data.Resources.fetch("tokens", "post", params) + .then(data => { + this.__rebuildTokensList(); + const showAPIKeyWindow = new osparc.desktop.preferences.window.ShowAPIKey("hello", "world"); + showAPIKeyWindow.center(); + showAPIKeyWindow.open(); + console.log(data); + }) + .catch(err => { + osparc.component.message.FlashMessenger.getInstance().logAs(this.tr("Failed creating oSPARC API token"), "ERROR"); + console.error(err); + }) + .finally(() => this.__requestTokenBtn.setFetching(false)); + }, this); + createAPIKeyWindow.open(); + }, + + __createExternalTokensSection: function() { + // layout + const box = this._createSectionBox(this.tr("External service Tokens")); const label = this._createHelpLabel(this.tr( "List of API tokens to access external services. Currently, \ @@ -48,58 +107,61 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { )); box.add(label); - let linkBtn = new osparc.ui.form.LinkButton(this.tr("To DAT-Core"), "https://app.blackfynn.io"); + const linkBtn = new osparc.ui.form.LinkButton(this.tr("To DAT-Core"), "https://app.blackfynn.io"); box.add(linkBtn); - const tokensList = this.__tokensList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); + const tokensList = this.__externalTokensList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); box.add(tokensList); - this.__rebuildTokensList(); return box; }, __rebuildTokensList: function() { - this.__tokensList.removeAll(); + this.__internalTokensList.removeAll(); + this.__externalTokensList.removeAll(); osparc.data.Resources.get("tokens") .then(tokensList => { if (tokensList.length) { - for (let i=0; i console.error(err)); }, __createEmptyTokenForm: function() { - let form = new qx.ui.form.Form(); + const form = new qx.ui.form.Form(); // FIXME: for the moment this is fixed since it has to be a unique id - let newTokenService = new qx.ui.form.TextField(); + const newTokenService = new qx.ui.form.TextField(); newTokenService.set({ value: "blackfynn-datcore", readOnly: true }); form.add(newTokenService, this.tr("Service")); - // TODO: - let newTokenKey = new qx.ui.form.TextField(); + const newTokenKey = new qx.ui.form.TextField(); newTokenKey.set({ placeholder: this.tr("Introduce token key here") }); form.add(newTokenKey, this.tr("Key")); - let newTokenSecret = new qx.ui.form.TextField(); + const newTokenSecret = new qx.ui.form.TextField(); newTokenSecret.set({ placeholder: this.tr("Introduce token secret here") }); form.add(newTokenSecret, this.tr("Secret")); - let addTokenBtn = new qx.ui.form.Button(this.tr("Add")); + const addTokenBtn = new qx.ui.form.Button(this.tr("Add")); addTokenBtn.setWidth(100); addTokenBtn.addListener("execute", e => { if (!osparc.data.Permissions.getInstance().canDo("preferences.token.create", true)) { @@ -122,10 +184,11 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { }, __createValidTokenForm: function(token) { + const label = token["keyLabel"] || token["service"]; const service = token["service"]; const height = 20; - const iconHeight = height-6; + const iconHeight = height - 6; const gr = new qx.ui.layout.Grid(10, 3); gr.setColumnFlex(1, 1); gr.setRowHeight(0, height); @@ -133,37 +196,19 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { gr.setRowHeight(2, height); const grid = new qx.ui.container.Composite(gr); - const nameLabel = new qx.ui.basic.Label(this.tr("Token name")); + const nameLabel = new qx.ui.basic.Label(service); grid.add(nameLabel, { row: 0, column: 0 }); - const nameVal = new qx.ui.basic.Label(service); + const nameVal = new qx.ui.basic.Label(label); grid.add(nameVal, { row: 0, column: 1 }); - /* - const showTokenIcon = "@FontAwesome5Solid/edit/"+iconHeight; - const showTokenBtn = new qx.ui.form.Button(null, showTokenIcon); - showTokenBtn.addListener("execute", e => { - const treeItemRenamer = new osparc.component.widget.Renamer(nameVal.getValue()); - treeItemRenamer.addListener("labelChanged", ev => { - const newLabel = ev.getData()["newLabel"]; - nameVal.setValue(newLabel); - }, this); - treeItemRenamer.center(); - treeItemRenamer.open(); - }, this); - grid.add(showTokenBtn, { - row: 0, - column: 2 - }); - */ - - const delTokenBtn = new qx.ui.form.Button(null, "@FontAwesome5Solid/trash-alt/"+iconHeight); + const delTokenBtn = new qx.ui.form.Button(null, "@FontAwesome5Solid/trash-alt/" + iconHeight); delTokenBtn.addListener("execute", e => { if (!osparc.data.Permissions.getInstance().canDo("preferences.token.delete", true)) { return; @@ -187,32 +232,32 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { __createPasswordSection: function() { // layout - let box = this._createSectionBox(this.tr("Password")); + const box = this._createSectionBox(this.tr("Password")); - let currentPassword = new qx.ui.form.PasswordField().set({ + const currentPassword = new qx.ui.form.PasswordField().set({ required: true, placeholder: this.tr("Your current password") }); box.add(currentPassword); - let newPassword = new qx.ui.form.PasswordField().set({ + const newPassword = new qx.ui.form.PasswordField().set({ required: true, placeholder: this.tr("Your new password") }); box.add(newPassword); - let confirm = new qx.ui.form.PasswordField().set({ + const confirm = new qx.ui.form.PasswordField().set({ required: true, placeholder: this.tr("Retype your new password") }); box.add(confirm); - let manager = new qx.ui.form.validation.Manager(); + const manager = new qx.ui.form.validation.Manager(); manager.setValidator(function(_itemForms) { return osparc.auth.core.Utils.checkSamePasswords(newPassword, confirm); }); - let resetBtn = new qx.ui.form.Button("Reset Password").set({ + const resetBtn = new qx.ui.form.Button("Reset Password").set({ allowGrowX: false }); box.add(resetBtn); diff --git a/services/web/client/source/class/osparc/desktop/preferences/window/APIKeyBase.js b/services/web/client/source/class/osparc/desktop/preferences/window/APIKeyBase.js new file mode 100644 index 00000000000..134dfa1d64f --- /dev/null +++ b/services/web/client/source/class/osparc/desktop/preferences/window/APIKeyBase.js @@ -0,0 +1,46 @@ +/* ************************************************************************ + osparc - the simcore frontend + https://osparc.io + Copyright: + 2020 IT'IS Foundation, https://itis.swiss + License: + MIT: https://opensource.org/licenses/MIT + Authors: + * Odei Maiz (odeimaiz) +************************************************************************ */ + +/** + * + */ + +qx.Class.define("osparc.desktop.preferences.window.APIKeyBase", { + extend: qx.ui.window.Window, + type: "abstract", + + construct: function(caption, infoText) { + this.base(arguments, caption); + + this.set({ + layout: new qx.ui.layout.VBox(5), + autoDestroy: true, + modal: true, + showMaximize: false, + showMinimize: false, + width: 350 + }); + + this.__addInfoText(infoText); + + this.center(); + }, + + members: { + __addInfoText: function(infoText) { + const introLabel = new qx.ui.basic.Label(infoText).set({ + padding: 5, + rich: true + }); + this._add(introLabel); + } + } +}); diff --git a/services/web/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js b/services/web/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js new file mode 100644 index 00000000000..aa987fc0043 --- /dev/null +++ b/services/web/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js @@ -0,0 +1,66 @@ +/* ************************************************************************ + osparc - the simcore frontend + https://osparc.io + Copyright: + 2020 IT'IS Foundation, https://itis.swiss + License: + MIT: https://opensource.org/licenses/MIT + Authors: + * Odei Maiz (odeimaiz) +************************************************************************ */ + +/** + * + */ + +qx.Class.define("osparc.desktop.preferences.window.CreateAPIKey", { + extend: osparc.desktop.preferences.window.APIKeyBase, + + construct: function() { + const caption = this.tr("Create oSPARC API Key"); + const infoText = this.tr("Key names must be unique."); + this.base(arguments, caption, infoText); + + this.__populateWindow(); + }, + + events: { + "finished": "qx.event.type.Data" + }, + + members: { + __populateWindow: function() { + const hBox1 = new qx.ui.container.Composite(new qx.ui.layout.HBox(10)).set({ + padding: 5 + }); + const sTitle = new qx.ui.basic.Label(this.tr("API Key")).set({ + width: 50, + alignY: "middle" + }); + hBox1.add(sTitle); + const labelEditor = new qx.ui.form.TextField(); + this.add(labelEditor, { + flex: 1 + }); + hBox1.add(labelEditor, { + flex: 1 + }); + this._add(hBox1); + + const hBox2 = new qx.ui.container.Composite(new qx.ui.layout.HBox(10)).set({ + padding: 5 + }); + hBox2.add(new qx.ui.core.Spacer(), { + flex: 1 + }); + const confirmBtn = new qx.ui.form.Button(this.tr("Confirm")); + confirmBtn.addListener("execute", e => { + const keyLabel = labelEditor.getValue(); + this.fireDataEvent("finished", keyLabel); + }, this); + hBox2.add(confirmBtn); + + this._add(hBox2); + } + } +}); diff --git a/services/web/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js b/services/web/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js new file mode 100644 index 00000000000..dbf5ba8dc89 --- /dev/null +++ b/services/web/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js @@ -0,0 +1,75 @@ +/* ************************************************************************ + osparc - the simcore frontend + https://osparc.io + Copyright: + 2020 IT'IS Foundation, https://itis.swiss + License: + MIT: https://opensource.org/licenses/MIT + Authors: + * Odei Maiz (odeimaiz) +************************************************************************ */ + +/** + * + */ + +qx.Class.define("osparc.desktop.preferences.window.ShowAPIKey", { + extend: osparc.desktop.preferences.window.APIKeyBase, + + construct: function(key, secret) { + const caption = this.tr("oSPARC API Key"); + const infoText = this.tr("For your protection, store your access keys securely and do not share them. You will not be able to access the key again once this window is closed."); + this.base(arguments, caption, infoText); + + this.__populateTokens(key, secret); + }, + + members: { + __populateTokens: function(key, secret) { + const hBox1 = this.__createEntry(this.tr("Key:"), key); + this._add(hBox1); + + const hBox2 = this.__createEntry(this.tr("Secret:"), secret); + this._add(hBox2); + + const hBox3 = new qx.ui.container.Composite(new qx.ui.layout.HBox(10)).set({ + padding: 5 + }); + const copyAPIKeyBtn = new qx.ui.form.Button(this.tr("Copy API Key")); + copyAPIKeyBtn.addListener("execute", e => { + if (osparc.utils.Utils.copyTextToClipboard(key)) { + copyAPIKeyBtn.setIcon("@FontAwesome5Solid/check/12"); + } + }); + hBox3.add(copyAPIKeyBtn, { + width: "50%" + }); + const copyAPISecretBtn = new qx.ui.form.Button(this.tr("Copy API Secret")); + copyAPISecretBtn.addListener("execute", e => { + if (osparc.utils.Utils.copyTextToClipboard(secret)) { + copyAPISecretBtn.setIcon("@FontAwesome5Solid/check/12"); + } + }); + hBox3.add(copyAPISecretBtn, { + width: "50%" + }); + this._add(hBox3); + }, + + __createEntry: function(title, label) { + const hBox = new qx.ui.container.Composite(new qx.ui.layout.HBox(10)).set({ + padding: 5 + }); + const sTitle = new qx.ui.basic.Label(title).set({ + rich: true, + width: 40 + }); + hBox.add(sTitle); + const sLabel = new qx.ui.basic.Label(label).set({ + selectable: true + }); + hBox.add(sLabel); + return hBox; + } + } +}); From 940b0fe627074015b9de6a890df5bf5c5b11af65 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 08:26:19 +0200 Subject: [PATCH 18/33] Minor --- services/api-gateway/tests/unit/test_client_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api-gateway/tests/unit/test_client_sdk.py b/services/api-gateway/tests/unit/test_client_sdk.py index d3134443653..d5b713d2a1c 100644 --- a/services/api-gateway/tests/unit/test_client_sdk.py +++ b/services/api-gateway/tests/unit/test_client_sdk.py @@ -41,7 +41,7 @@ def client(monkeypatch) -> TestClient: import aiohttp from yarl import URL -from typing import Optional, Dict, Any +from typing import Optional, Any import json From d06526fa310f5f0014bf17c7d77704dc944f492c Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 08:52:38 +0200 Subject: [PATCH 19/33] Cleanup front-end messages/labels --- .../osparc/desktop/preferences/pages/SecurityPage.js | 10 +++++----- .../osparc/desktop/preferences/window/CreateAPIKey.js | 2 +- .../osparc/desktop/preferences/window/ShowAPIKey.js | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js index f99e43085ec..63e68866a96 100644 --- a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js +++ b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js @@ -44,17 +44,17 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { __createInternalTokensSection: function() { // layout - const box = this._createSectionBox(this.tr("oSPARC API Tokens")); + const box = this._createSectionBox(this.tr("API Keys")); const label = this._createHelpLabel(this.tr( - "Tokens to access oSPARC API." + "List API keys associated to your account." )); box.add(label); const tokensList = this.__internalTokensList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); box.add(tokensList); - const requestTokenBtn = this.__requestTokenBtn = new osparc.ui.form.FetchButton(this.tr("Create oSPARC Token")).set({ + const requestTokenBtn = this.__requestTokenBtn = new osparc.ui.form.FetchButton(this.tr("Create API Key")).set({ allowGrowX: false }); requestTokenBtn.addListener("execute", () => { @@ -89,7 +89,7 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { console.log(data); }) .catch(err => { - osparc.component.message.FlashMessenger.getInstance().logAs(this.tr("Failed creating oSPARC API token"), "ERROR"); + osparc.component.message.FlashMessenger.getInstance().logAs(this.tr("Failed creating API token"), "ERROR"); console.error(err); }) .finally(() => this.__requestTokenBtn.setFetching(false)); @@ -99,7 +99,7 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { __createExternalTokensSection: function() { // layout - const box = this._createSectionBox(this.tr("External service Tokens")); + const box = this._createSectionBox(this.tr("External Service Tokens")); const label = this._createHelpLabel(this.tr( "List of API tokens to access external services. Currently, \ diff --git a/services/web/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js b/services/web/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js index aa987fc0043..e069b528e65 100644 --- a/services/web/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js +++ b/services/web/client/source/class/osparc/desktop/preferences/window/CreateAPIKey.js @@ -17,7 +17,7 @@ qx.Class.define("osparc.desktop.preferences.window.CreateAPIKey", { extend: osparc.desktop.preferences.window.APIKeyBase, construct: function() { - const caption = this.tr("Create oSPARC API Key"); + const caption = this.tr("Create API Key"); const infoText = this.tr("Key names must be unique."); this.base(arguments, caption, infoText); diff --git a/services/web/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js b/services/web/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js index dbf5ba8dc89..95b1856991d 100644 --- a/services/web/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js +++ b/services/web/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js @@ -17,7 +17,7 @@ qx.Class.define("osparc.desktop.preferences.window.ShowAPIKey", { extend: osparc.desktop.preferences.window.APIKeyBase, construct: function(key, secret) { - const caption = this.tr("oSPARC API Key"); + const caption = this.tr("API Key"); const infoText = this.tr("For your protection, store your access keys securely and do not share them. You will not be able to access the key again once this window is closed."); this.base(arguments, caption, infoText); From c15257778feee0bb46170cbdfe0709e92073b997 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 10:13:56 +0200 Subject: [PATCH 20/33] Adds new table for client credentials api_keys --- .../16ee7d73b9cc_adds_api_keys_table.py | 37 +++++++++++++++++++ .../models/api_keys.py | 34 +++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/16ee7d73b9cc_adds_api_keys_table.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/api_keys.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/16ee7d73b9cc_adds_api_keys_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/16ee7d73b9cc_adds_api_keys_table.py new file mode 100644 index 00000000000..b9c7a9d0a9a --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/16ee7d73b9cc_adds_api_keys_table.py @@ -0,0 +1,37 @@ +"""Adds api_keys table + +Revision ID: 16ee7d73b9cc +Revises: f3555bb4bc34 +Create Date: 2020-04-28 08:11:42.785688+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '16ee7d73b9cc' +down_revision = 'f3555bb4bc34' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('api_keys', + sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('display_name', sa.String(), nullable=False), + sa.Column('user_id', sa.BigInteger(), nullable=False), + sa.Column('api_key', sa.String(), nullable=False), + sa.Column('api_secret', sa.String(), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('display_name', 'user_id', name='display_name_userid_uniqueness') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('api_keys') + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/api_keys.py b/packages/postgres-database/src/simcore_postgres_database/models/api_keys.py new file mode 100644 index 00000000000..cd3ee02e421 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/api_keys.py @@ -0,0 +1,34 @@ +""" API keys to access public gateway + + +These keys grant the client authorization to the API resources + + +--------+ +---------------+ + | |--(A)- Authorization Request ->| Resource | + |client | | Owner | Authorization request + | |<-(B)-- Authorization Grant ---| | + +--------+ +---------------+ + +""" +import sqlalchemy as sa + +from .base import metadata +from .users import users + +api_keys = sa.Table( + "api_keys", + metadata, + sa.Column("id", sa.BigInteger, nullable=False, primary_key=True), + sa.Column("display_name", sa.String, nullable=False), + sa.Column( + "user_id", + sa.BigInteger, + sa.ForeignKey(users.c.id, ondelete="CASCADE"), + nullable=False, + ), + sa.Column("api_key", sa.String, nullable=False), + sa.Column("api_secret", sa.String, nullable=False), + sa.UniqueConstraint( + "display_name", "user_id", name="display_name_userid_uniqueness" + ), +) From c5b7c54294599c49c005156b2492690e256b1f8c Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 10:57:38 +0200 Subject: [PATCH 21/33] Added api-keys section on webserver API --- api/specs/webserver/openapi-auth.yaml | 68 +++++++++++++++++++ api/specs/webserver/openapi.yaml | 4 +- .../api/v0/openapi.yaml | 66 ++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/api/specs/webserver/openapi-auth.yaml b/api/specs/webserver/openapi-auth.yaml index a1be7f5105e..60778bcb4a1 100644 --- a/api/specs/webserver/openapi-auth.yaml +++ b/api/specs/webserver/openapi-auth.yaml @@ -210,6 +210,60 @@ paths: "3XX": description: redirection to specific ui application page + /auth/api-keys: + get: + summary: lists display names of API keys by this user + tags: + - authentication + operationId: list_api_keys + responses: + "200": + description: returns the display names of API keys + content: + application/json: + schema: + type: array + items: + type: string + "401": + description: unauthorized to list keys + post: + summary: creates API keys to access public API + tags: + - authentication + operationId: create_api_key + requestBody: + description: user registration + content: + application/json: + schema: + $ref: "#/components/schemas/ApiKeyName" + responses: + "200": + description: Authorization granted returning API key + content: + application/json: + schema: + $ref: "#/components/schemas/ApiKeyGranted" + "400": + description: key name requested is invalid + "401": + description: unauthorized to create a key + + delete: + summary: deletes API key by name + operationId: delete_api_key + requestBody: + description: deletes given api key by name + content: + application/json: + schema: + $ref: "#/components/schemas/ApiKeyName" + responses: + "204": + description: api key successfully deleted + "401": + description: unauthorized to delete a key components: responses: DefaultErrorResponse: @@ -221,3 +275,17 @@ components: client_session_id: type: string example: 5ac57685-c40f-448f-8711-70be1936fd63 + ApiKeyName: + type: object + properties: + display_name: + type: string + ApiKeyGranted: + type: object + properties: + display_name: + type: string + api_key: + type: string + api_secret: + type: string diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 50c90c21a6d..d367d6dc53f 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -49,7 +49,7 @@ paths: # DIAGNOSTICS --------------------------------------------------------- /: $ref: "./openapi-diagnostics.yaml#/paths/~1" - + /health: $ref: "./openapi-diagnostics.yaml#/paths/~1health" @@ -85,6 +85,8 @@ paths: /auth/confirmation/{code}: $ref: "./openapi-auth.yaml#/paths/~1auth~1confirmation~1{code}" + /auth/api-keys: + $ref: "./openapi-auth.yaml#/paths/~1auth~1api-keys" # USER SETTINGS ------------------------------------------------------------------ /me: diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 79dc0d645ae..737dbb030a7 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2006,6 +2006,72 @@ paths: responses: 3XX: description: redirection to specific ui application page + /auth/api-keys: + get: + summary: lists display names of API keys by this user + tags: + - authentication + operationId: list_api_keys + responses: + '200': + description: returns the display names of API keys + content: + application/json: + schema: + type: array + items: + type: string + '401': + description: unauthorized to list keys + post: + summary: creates API keys to access public API + tags: + - authentication + operationId: create_api_key + requestBody: + description: user registration + content: + application/json: + schema: + type: object + properties: + display_name: + type: string + responses: + '200': + description: Authorization granted returning API key + content: + application/json: + schema: + type: object + properties: + display_name: + type: string + api_key: + type: string + api_secret: + type: string + '400': + description: key name requested is invalid + '401': + description: unauthorized to create a key + delete: + summary: deletes API key by name + operationId: delete_api_key + requestBody: + description: deletes given api key by name + content: + application/json: + schema: + type: object + properties: + display_name: + type: string + responses: + '204': + description: api key successfully deleted + '401': + description: unauthorized to delete a key /me: get: operationId: get_my_profile From 2941e6377540476c54473c487f83f21762320f6c Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 13:37:06 +0200 Subject: [PATCH 22/33] Minors --- .../src/simcore_postgres_database/webserver_models.py | 2 ++ services/api-gateway/src/simcore_service_api_gateway/auth.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/packages/postgres-database/src/simcore_postgres_database/webserver_models.py b/packages/postgres-database/src/simcore_postgres_database/webserver_models.py index 0d322010d8d..26c786510da 100644 --- a/packages/postgres-database/src/simcore_postgres_database/webserver_models.py +++ b/packages/postgres-database/src/simcore_postgres_database/webserver_models.py @@ -12,6 +12,7 @@ from .models.user_to_projects import user_to_projects from .models.users import UserRole, UserStatus, users from .models.tags import tags, study_tags +from .models.api_keys import api_keys __all__ = [ "users", @@ -27,4 +28,5 @@ "comp_pipeline", "tags", "study_tags", + "api_keys", ] diff --git a/services/api-gateway/src/simcore_service_api_gateway/auth.py b/services/api-gateway/src/simcore_service_api_gateway/auth.py index 3ed8c670a20..ea8b526c4b2 100644 --- a/services/api-gateway/src/simcore_service_api_gateway/auth.py +++ b/services/api-gateway/src/simcore_service_api_gateway/auth.py @@ -19,6 +19,10 @@ +--------+ +---------------+ Figure 1: Abstract Protocol Flow + +SEE + - https://oauth.net/2/ + - https://tools.ietf.org/html/rfc6749 """ # TODO: this module shall delegate the auth functionality to a separate service From 7385de9baebb344c99ce43314ad5b844233de89d Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 13:38:17 +0200 Subject: [PATCH 23/33] Adding api-keys handlers --- .../login/api_keys_handlers.py | 114 ++++++++++++++++++ .../simcore_service_webserver/login/routes.py | 4 + .../tests/unit/with_dbs/test_api_keys.py | 78 ++++++++++++ 3 files changed, 196 insertions(+) create mode 100644 services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py create mode 100644 services/web/server/tests/unit/with_dbs/test_api_keys.py diff --git a/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py b/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py new file mode 100644 index 00000000000..e0b87253c41 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py @@ -0,0 +1,114 @@ +import logging +import uuid as uuidlib +from typing import Dict + +import sqlalchemy as sa +from aiohttp import web + +import simcore_postgres_database.webserver_models as orm +from servicelib.application_keys import APP_DB_ENGINE_KEY +from servicelib.aiopg_utils import DatabaseError + +from .decorators import RQT_USERID_KEY, login_required +from .utils import get_random_string + +log = logging.getLogger(__name__) + + +def generate_api_credentials() -> Dict[str, str]: + credentials: Dict = dict.fromkeys(("api_key", "api_secret"), "") + for name in credentials: + value = get_random_string(20) + credentials[name] = str(uuidlib.uuid5(uuidlib.NAMESPACE_DNS, value)) + return credentials + + +class CRUD: + # pylint: disable=no-value-for-parameter + + def __init__(self, request: web.Request): + self.engine = request.app.get(APP_DB_ENGINE_KEY) + self.userid: int = request.get(RQT_USERID_KEY, -1) + + async def list_api_key_names(self): + async with self.engine.acquire() as conn: + stmt = orm.api_keys.select([orm.api_keys.c.display_name]).where( + orm.users.c.user_id == self.userid + ) + + res = await conn.execute(stmt) + rows = await res.fetchall() + return list(rows) + + async def create(self, name: str, *, api_key: str, api_secret: str): + async with self.engine.acquire() as conn: + stmt = orm.api_keys.insert().values( + display_name=name, + user_id=self.userid, + api_key=api_key, + api_secret=api_secret, + ) + await conn.execute(stmt) + + async def delete_api_key(self, name: str): + async with self.engine.acquire() as conn: + stmt = orm.api_keys.delete().where( + sa.and_( + orm.users.c.user_id == self.userid, + orm.api_keys.c.display_name == name, + ) + ) + await conn.execute(stmt) + + +@login_required +async def list_api_keys(request: web.Request): + """ + GET /auth/api-keys + """ + crud = CRUD(request) + names = await crud.list_api_key_names() + return names + + +@login_required +async def create_api_key(request: web.Request): + """ + POST /auth/api-keys + """ + body = await request.json() + display_name = body.get("display_name") + + credentials = generate_api_credentials() + try: + crud = CRUD(request) + await crud.create(display_name, **credentials) + except DatabaseError as err: + log.warning("Failed to create API key %d", display_name, exc_info=err) + raise web.HTTPBadRequest(reason="Invalid API key name: already exists") + + return { + "display_name": display_name, + "api_key": credentials["api_key"], + "api_secret": credentials["api_secret"], + } + + +@login_required +async def delete_api_key(request: web.Request): + """ + DELETE /auth/api-keys + """ + + body = await request.json() + display_name = body.get("display_name") + + try: + crud = CRUD(request) + await crud.delete_api_key(display_name) + except DatabaseError as err: + log.warning( + "Failed to delete API key %d. Ignoring error", display_name, exc_info=err + ) + + raise web.HTTPNoContent diff --git a/services/web/server/src/simcore_service_webserver/login/routes.py b/services/web/server/src/simcore_service_webserver/login/routes.py index 61b78959bd7..8a8554217e6 100644 --- a/services/web/server/src/simcore_service_webserver/login/routes.py +++ b/services/web/server/src/simcore_service_webserver/login/routes.py @@ -13,6 +13,7 @@ from servicelib.rest_routing import iter_path_operations, map_handlers_with_operations from . import handlers as login_handlers +from . import api_keys_handlers log = logging.getLogger(__name__) @@ -42,6 +43,9 @@ def include_path(tuple_object): "auth_change_email": login_handlers.change_email, "auth_change_password": login_handlers.change_password, "auth_confirmation": login_handlers.email_confirmation, + "create_api_key": api_keys_handlers.create_api_key, + "delete_api_key": api_keys_handlers.delete_api_key, + "list_api_keys": api_keys_handlers.list_api_keys, } routes = map_handlers_with_operations( diff --git a/services/web/server/tests/unit/with_dbs/test_api_keys.py b/services/web/server/tests/unit/with_dbs/test_api_keys.py new file mode 100644 index 00000000000..3a28b37290e --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/test_api_keys.py @@ -0,0 +1,78 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +from aiohttp import web +import pytest + +from pytest_simcore.helpers.utils_assert import assert_status +from pytest_simcore.helpers.utils_login import LoggedUser +from simcore_service_webserver.db_models import UserRole + + +@pytest.fixture() +async def logged_user(client, user_role: UserRole): + """ adds a user in db and logs in with client + + NOTE: `user_role` fixture is defined as a parametrization below!!! + """ + async with LoggedUser( + client, + {"role": user_role.name}, + check_if_succeeds=user_role != UserRole.ANONYMOUS, + ) as user: + print("-----> logged in user", user_role) + yield user + print("<----- logged out user", user_role) + + +@pytest.mark.parametrize( + "user_role,expected", + [ + (UserRole.ANONYMOUS, web.HTTPUnauthorized), + (UserRole.GUEST, web.HTTPOk), + (UserRole.USER, web.HTTPOk), + (UserRole.TESTER, web.HTTPOk), + ], +) +async def test_create_api_keys(client, logged_user, user_role, expected): + resp = await client.post("/v0/auth/api-keys", json={"display_name": "foo"}) + + data, errors = await assert_status(resp, expected) + + if not errors: + client.app + + +@pytest.mark.parametrize( + "user_role,expected", + [ + (UserRole.ANONYMOUS, web.HTTPUnauthorized), + (UserRole.GUEST, web.HTTPOk), + (UserRole.USER, web.HTTPOk), + (UserRole.TESTER, web.HTTPOk), + ], +) +async def test_list_api_keys(client, logged_user, user_role, expected): + resp = await client.get("/v0/auth/api-keys") + data, errors = await assert_status(resp, expected) + + if not errors: + assert not data + + with UserApiKeys(client.app, logged_user.user.user_id, ['foo', 'bar', 'beta']): + resp = await client.get("/v0/auth/api-keys") + data, _ = await assert_status(resp, expected) + assert data == ['foo', 'bar', 'beta'] + +@pytest.mark.parametrize( + "user_role,expected", + [ + (UserRole.ANONYMOUS, web.HTTPUnauthorized), + (UserRole.GUEST, web.HTTPNoContent), + (UserRole.USER, web.HTTPNoContent), + (UserRole.TESTER, web.HTTPNoContent), + ], +) +async def test_delete_api_keys(client, logged_user, user_role, expected): + resp = await client.delete("/v0/auth/api-keys", json={"display_name": "foo"}) + data, errors = await assert_status(resp, expected) From 1b151e2a95b5a0cd1e2a163005fbb56391af3bac Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 15:08:35 +0200 Subject: [PATCH 24/33] Tests for api-keys pass --- .../simcore_service_webserver/db_models.py | 2 + .../login/api_keys_handlers.py | 24 ++-- .../server/tests/unit/with_dbs/conftest.py | 18 +-- .../tests/unit/with_dbs/test_api_keys.py | 107 +++++++++++------- 4 files changed, 96 insertions(+), 55 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/db_models.py b/services/web/server/src/simcore_service_webserver/db_models.py index 8113263eaf2..952ec926693 100644 --- a/services/web/server/src/simcore_service_webserver/db_models.py +++ b/services/web/server/src/simcore_service_webserver/db_models.py @@ -11,6 +11,7 @@ users, tags, study_tags, + api_keys, ) # TODO: roles table that maps every role with allowed tasks e.g. read/write,...?? @@ -25,4 +26,5 @@ "metadata", "tags", "study_tags", + "api_keys", ) diff --git a/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py b/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py index e0b87253c41..3d78d79ed98 100644 --- a/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py +++ b/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py @@ -1,6 +1,6 @@ import logging import uuid as uuidlib -from typing import Dict +from typing import Dict, List import sqlalchemy as sa from aiohttp import web @@ -9,6 +9,8 @@ from servicelib.application_keys import APP_DB_ENGINE_KEY from servicelib.aiopg_utils import DatabaseError +from aiopg.sa.result import ResultProxy, RowProxy + from .decorators import RQT_USERID_KEY, login_required from .utils import get_random_string @@ -32,15 +34,16 @@ def __init__(self, request: web.Request): async def list_api_key_names(self): async with self.engine.acquire() as conn: - stmt = orm.api_keys.select([orm.api_keys.c.display_name]).where( - orm.users.c.user_id == self.userid + stmt = sa.select([orm.api_keys.c.display_name,]).where( + orm.api_keys.c.user_id == self.userid ) - res = await conn.execute(stmt) - rows = await res.fetchall() - return list(rows) + res: ResultProxy = await conn.execute(stmt) + rows: List[RowProxy] = await res.fetchall() + return [row.get(0) for row in rows] if rows else [] async def create(self, name: str, *, api_key: str, api_secret: str): + async with self.engine.acquire() as conn: stmt = orm.api_keys.insert().values( display_name=name, @@ -48,13 +51,16 @@ async def create(self, name: str, *, api_key: str, api_secret: str): api_key=api_key, api_secret=api_secret, ) - await conn.execute(stmt) + + res: ResultProxy = await conn.execute(stmt) + print(res) + async def delete_api_key(self, name: str): async with self.engine.acquire() as conn: stmt = orm.api_keys.delete().where( sa.and_( - orm.users.c.user_id == self.userid, + orm.api_keys.c.user_id == self.userid, orm.api_keys.c.display_name == name, ) ) @@ -85,7 +91,7 @@ async def create_api_key(request: web.Request): await crud.create(display_name, **credentials) except DatabaseError as err: log.warning("Failed to create API key %d", display_name, exc_info=err) - raise web.HTTPBadRequest(reason="Invalid API key name: already exists") + raise web.HTTPBadRequest(reason="Invalid API key name: already exists") from err return { "display_name": display_name, diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 0521c762477..43374448ef0 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -17,20 +17,20 @@ from typing import Dict from uuid import uuid4 +import aioredis import pytest +import redis +import socketio import sqlalchemy as sa +import trafaret_config from yarl import URL -import aioredis -import redis +import simcore_service_webserver.db_models as orm import simcore_service_webserver.utils -import socketio -import trafaret_config from servicelib.aiopg_utils import DSN from servicelib.rest_responses import unwrap_envelope from simcore_service_webserver.application import create_application from simcore_service_webserver.application_config import app_schema as app_schema -from simcore_service_webserver.db_models import confirmations, metadata, users ## current directory current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -117,11 +117,15 @@ def postgres_db(app_cfg, postgres_service): # Configures db and initializes tables # Uses syncrounous engine for that engine = sa.create_engine(url, isolation_level="AUTOCOMMIT") - metadata.create_all(bind=engine, tables=[users, confirmations], checkfirst=True) + orm.metadata.create_all( + bind=engine, + tables=[orm.users, orm.confirmations, orm.api_keys], + checkfirst=True, + ) yield engine - metadata.drop_all(engine) + orm.metadata.drop_all(engine) engine.dispose() diff --git a/services/web/server/tests/unit/with_dbs/test_api_keys.py b/services/web/server/tests/unit/with_dbs/test_api_keys.py index 3a28b37290e..0e8b743ea2a 100644 --- a/services/web/server/tests/unit/with_dbs/test_api_keys.py +++ b/services/web/server/tests/unit/with_dbs/test_api_keys.py @@ -1,12 +1,15 @@ # pylint:disable=unused-variable # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -from aiohttp import web + +import attr import pytest +from aiohttp import web from pytest_simcore.helpers.utils_assert import assert_status from pytest_simcore.helpers.utils_login import LoggedUser from simcore_service_webserver.db_models import UserRole +from simcore_service_webserver.login.api_keys_handlers import CRUD as ApiKeysCRUD @pytest.fixture() @@ -20,37 +23,46 @@ async def logged_user(client, user_role: UserRole): {"role": user_role.name}, check_if_succeeds=user_role != UserRole.ANONYMOUS, ) as user: - print("-----> logged in user", user_role) + print("-----> logged in user as", user_role) yield user - print("<----- logged out user", user_role) + print("<----- logged out user as", user_role) -@pytest.mark.parametrize( - "user_role,expected", - [ - (UserRole.ANONYMOUS, web.HTTPUnauthorized), - (UserRole.GUEST, web.HTTPOk), - (UserRole.USER, web.HTTPOk), - (UserRole.TESTER, web.HTTPOk), - ], -) -async def test_create_api_keys(client, logged_user, user_role, expected): - resp = await client.post("/v0/auth/api-keys", json={"display_name": "foo"}) +@pytest.fixture() +async def fake_user_api_keys(client, logged_user): + names = ["foo", "bar", "beta", "alpha"] - data, errors = await assert_status(resp, expected) + @attr.s(auto_attribs=True) + class Adapter: + app: web.Application + userid: int + + def get(self, *_args): + return self.userid + + crud = ApiKeysCRUD(Adapter(client.app, logged_user['id'])) + + for name in names: + await crud.create(name, api_key=f"{name}-key", api_secret=f"{name}-secret") + + yield names + + for name in names: + await crud.delete_api_key(name) - if not errors: - client.app + +# TESTS --------- + +USER_ACCESS_PARAMETERS = [ + (UserRole.ANONYMOUS, web.HTTPUnauthorized), + # TODO: (UserRole.GUEST, web.HTTPUnauthorized), + (UserRole.USER, web.HTTPOk), + (UserRole.TESTER, web.HTTPOk), +] @pytest.mark.parametrize( - "user_role,expected", - [ - (UserRole.ANONYMOUS, web.HTTPUnauthorized), - (UserRole.GUEST, web.HTTPOk), - (UserRole.USER, web.HTTPOk), - (UserRole.TESTER, web.HTTPOk), - ], + "user_role,expected", USER_ACCESS_PARAMETERS, ) async def test_list_api_keys(client, logged_user, user_role, expected): resp = await client.get("/v0/auth/api-keys") @@ -59,20 +71,37 @@ async def test_list_api_keys(client, logged_user, user_role, expected): if not errors: assert not data - with UserApiKeys(client.app, logged_user.user.user_id, ['foo', 'bar', 'beta']): - resp = await client.get("/v0/auth/api-keys") - data, _ = await assert_status(resp, expected) - assert data == ['foo', 'bar', 'beta'] -@pytest.mark.parametrize( - "user_role,expected", - [ - (UserRole.ANONYMOUS, web.HTTPUnauthorized), - (UserRole.GUEST, web.HTTPNoContent), - (UserRole.USER, web.HTTPNoContent), - (UserRole.TESTER, web.HTTPNoContent), - ], -) -async def test_delete_api_keys(client, logged_user, user_role, expected): - resp = await client.delete("/v0/auth/api-keys", json={"display_name": "foo"}) +@pytest.mark.parametrize("user_role,expected", USER_ACCESS_PARAMETERS) +async def test_create_api_keys(client, logged_user, user_role, expected): + resp = await client.post("/v0/auth/api-keys", json={"display_name": "foo"}) + data, errors = await assert_status(resp, expected) + + if not errors: + assert data["display_name"] == "foo" + assert "api_key" in data + assert "api_secret" in data + + resp = await client.get("/v0/auth/api-keys") + data, _ = await assert_status(resp, expected) + assert sorted(data) == [ + "foo", + ] + + +@pytest.mark.parametrize("user_role,expected", [ + (UserRole.ANONYMOUS, web.HTTPUnauthorized), + (UserRole.GUEST, web.HTTPNoContent), + (UserRole.USER, web.HTTPNoContent), + (UserRole.TESTER, web.HTTPNoContent), +]) +async def test_delete_api_keys( + client, fake_user_api_keys, logged_user, user_role, expected +): + resp = await client.delete("/v0/auth/api-keys", json={"display_name": "foo"}) + await assert_status(resp, expected) + + for name in fake_user_api_keys: + resp = await client.delete("/v0/auth/api-keys", json={"display_name": name}) + await assert_status(resp, expected) From 57ae255242b410927ffbed2bd947cd846bee49d6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 15:45:20 +0200 Subject: [PATCH 25/33] Minor fixes in API --- api/specs/webserver/openapi-auth.yaml | 2 ++ .../server/src/simcore_service_webserver/api/v0/openapi.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/api/specs/webserver/openapi-auth.yaml b/api/specs/webserver/openapi-auth.yaml index 60778bcb4a1..7213be13cce 100644 --- a/api/specs/webserver/openapi-auth.yaml +++ b/api/specs/webserver/openapi-auth.yaml @@ -252,6 +252,8 @@ paths: delete: summary: deletes API key by name + tags: + - authentication operationId: delete_api_key requestBody: description: deletes given api key by name diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 737dbb030a7..b04df8c2573 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2057,6 +2057,8 @@ paths: description: unauthorized to create a key delete: summary: deletes API key by name + tags: + - authentication operationId: delete_api_key requestBody: description: deletes given api key by name From 3ca2655871e3ad6384598b9869c22391448480ab Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 15:46:05 +0200 Subject: [PATCH 26/33] =?UTF-8?q?webserver=20api=20version:=200.4.0=20?= =?UTF-8?q?=E2=86=92=200.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/specs/webserver/openapi.yaml | 2 +- services/web/server/VERSION | 2 +- services/web/server/setup.cfg | 2 +- services/web/server/setup.py | 2 +- .../server/src/simcore_service_webserver/api/v0/openapi.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index d367d6dc53f..79f2a1c59e4 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.0 info: title: "osparc-simcore RESTful API" - version: 0.4.0 + version: 0.5.0 description: "RESTful API designed for web clients" contact: name: IT'IS Foundation diff --git a/services/web/server/VERSION b/services/web/server/VERSION index 1d0ba9ea182..8f0916f768f 100644 --- a/services/web/server/VERSION +++ b/services/web/server/VERSION @@ -1 +1 @@ -0.4.0 +0.5.0 diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 4d1512adf59..1b748814bff 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.4.0 +current_version = 0.5.0 commit = True message = webserver api version: {current_version} → {new_version} tag = False diff --git a/services/web/server/setup.py b/services/web/server/setup.py index 58aab19e2fc..0edfbabb7da 100644 --- a/services/web/server/setup.py +++ b/services/web/server/setup.py @@ -23,7 +23,7 @@ def read_reqs(reqs_path: Path): setup( name="simcore-service-webserver", - version="0.4.0", + version="0.5.0", packages=find_packages(where="src"), package_dir={"": "src",}, include_package_data=True, diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index b04df8c2573..7c71b30c2ff 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.0 info: title: osparc-simcore RESTful API - version: 0.4.0 + version: 0.5.0 description: RESTful API designed for web clients contact: name: IT'IS Foundation From 83b321eae98c7a649bdb05007514409307d8082b Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 15:47:53 +0200 Subject: [PATCH 27/33] updates codeowners --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 36f37eb20fe..3c11d09f3d7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,7 @@ Makefile @pcrespov, @sanderegg /scripts/demo @odeimaiz, @pcrespov /scripts/json-schema-to-openapi-schema @sanderegg /scripts/template-projects @odeimaiz, @pcrespov +/services/api-gateway @pcrespov /services/catalog @pcrespov /services/sidecar @pcrespov, @mguidon /services/web/client @odeimaiz, @oetiker, @ignapas From 3b8a69431e1fdaeb1f8054d3e4fe68c8187fbbb8 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Tue, 28 Apr 2020 16:14:48 +0200 Subject: [PATCH 28/33] connects front-end by @odeimaiz (#27) * apiKeys resource added * apiKeyName in delete goes in body * working * display error message given by backend --- .../source/class/osparc/data/Permissions.js | 2 + .../source/class/osparc/data/Resources.js | 19 ++ .../desktop/preferences/pages/SecurityPage.js | 282 +++++++++++------- .../client/source/class/osparc/store/Store.js | 4 + .../security_roles.py | 4 + 5 files changed, 197 insertions(+), 114 deletions(-) diff --git a/services/web/client/source/class/osparc/data/Permissions.js b/services/web/client/source/class/osparc/data/Permissions.js index 637e339421e..0131bf33baf 100644 --- a/services/web/client/source/class/osparc/data/Permissions.js +++ b/services/web/client/source/class/osparc/data/Permissions.js @@ -136,6 +136,8 @@ qx.Class.define("osparc.data.Permissions", { "studies.user.create", "storage.datcore.read", "preferences.user.update", + "preferences.apikey.create", + "preferences.apikey.delete", "preferences.token.create", "preferences.token.delete", "preferences.tag", diff --git a/services/web/client/source/class/osparc/data/Resources.js b/services/web/client/source/class/osparc/data/Resources.js index d119025a24a..1ac6f949c74 100644 --- a/services/web/client/source/class/osparc/data/Resources.js +++ b/services/web/client/source/class/osparc/data/Resources.js @@ -213,6 +213,25 @@ qx.Class.define("osparc.data.Resources", { } } }, + /* + * API-KEYS + */ + apiKeys: { + endpoints: { + get: { + method: "GET", + url: statics.API + "/auth/api-keys" + }, + post: { + method: "POST", + url: statics.API + "/auth/api-keys" + }, + delete: { + method: "DELETE", + url: statics.API + "/auth/api-keys" + } + } + }, /* * TOKENS */ diff --git a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js index 63e68866a96..4843cffd3c0 100644 --- a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js +++ b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js @@ -32,17 +32,81 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { this.base(arguments, title, iconSrc); this.add(this.__createPasswordSection()); - this.add(this.__createInternalTokensSection()); - this.add(this.__createExternalTokensSection()); + this.add(this.__createAPIKeysSection()); + this.add(this.__createTokensSection()); + + this.__rebuildAPIKeysList(); this.__rebuildTokensList(); }, members: { - __internalTokensList: null, - __externalTokensList: null, + __apiKeysList: null, + __tokensList: null, + __requestAPIKeyBtn: null, - __createInternalTokensSection: function() { + __createPasswordSection: function() { + // layout + const box = this._createSectionBox(this.tr("Password")); + + const currentPassword = new qx.ui.form.PasswordField().set({ + required: true, + placeholder: this.tr("Your current password") + }); + box.add(currentPassword); + + const newPassword = new qx.ui.form.PasswordField().set({ + required: true, + placeholder: this.tr("Your new password") + }); + box.add(newPassword); + + const confirm = new qx.ui.form.PasswordField().set({ + required: true, + placeholder: this.tr("Retype your new password") + }); + box.add(confirm); + + const manager = new qx.ui.form.validation.Manager(); + manager.setValidator(function(_itemForms) { + return osparc.auth.core.Utils.checkSamePasswords(newPassword, confirm); + }); + + const resetBtn = new qx.ui.form.Button("Reset Password").set({ + allowGrowX: false + }); + box.add(resetBtn); + + resetBtn.addListener("execute", () => { + if (manager.validate()) { + const params = { + data: { + current: currentPassword.getValue(), + new: newPassword.getValue(), + confirm: confirm.getValue() + } + }; + osparc.data.Resources.fetch("password", "post", params) + .then(data => { + osparc.component.message.FlashMessenger.getInstance().log(data); + [currentPassword, newPassword, confirm].forEach(item => { + item.resetValue(); + }); + }) + .catch(err => { + console.error(err); + osparc.component.message.FlashMessenger.getInstance().logAs(this.tr("Failed to reset password"), "ERROR"); + [currentPassword, newPassword, confirm].forEach(item => { + item.resetValue(); + }); + }); + } + }); + + return box; + }, + + __createAPIKeysSection: function() { // layout const box = this._createSectionBox(this.tr("API Keys")); @@ -51,53 +115,102 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { )); box.add(label); - const tokensList = this.__internalTokensList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); - box.add(tokensList); + const apiKeysList = this.__apiKeysList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); + box.add(apiKeysList); - const requestTokenBtn = this.__requestTokenBtn = new osparc.ui.form.FetchButton(this.tr("Create API Key")).set({ + const requestAPIKeyBtn = this.__requestAPIKeyBtn = new osparc.ui.form.FetchButton(this.tr("Create API Key")).set({ allowGrowX: false }); - requestTokenBtn.addListener("execute", () => { - this.__requestOsparcToken(); + requestAPIKeyBtn.addListener("execute", () => { + this.__requestAPIKey(); }, this); - box.add(requestTokenBtn); + box.add(requestAPIKeyBtn); return box; }, - __requestOsparcToken: function() { - if (!osparc.data.Permissions.getInstance().canDo("preferences.token.create", true)) { + __requestAPIKey: function() { + if (!osparc.data.Permissions.getInstance().canDo("preferences.apikey.create", true)) { return; } - const createAPIKeyWindow = new osparc.desktop.preferences.window.CreateAPIKey("hello", "world"); + const createAPIKeyWindow = new osparc.desktop.preferences.window.CreateAPIKey(); createAPIKeyWindow.addListener("finished", keyLabel => { const params = { data: { - "service": "osparc", - "keyLabel": keyLabel.getData() + "display_name": keyLabel.getData() } }; createAPIKeyWindow.close(); - this.__requestTokenBtn.setFetching(true); - osparc.data.Resources.fetch("tokens", "post", params) + this.__requestAPIKeyBtn.setFetching(true); + osparc.data.Resources.fetch("apiKeys", "post", params) .then(data => { - this.__rebuildTokensList(); - const showAPIKeyWindow = new osparc.desktop.preferences.window.ShowAPIKey("hello", "world"); + this.__rebuildAPIKeysList(); + + const key = data["api_key"]; + const secret = data["api_secret"]; + const showAPIKeyWindow = new osparc.desktop.preferences.window.ShowAPIKey(key, secret); showAPIKeyWindow.center(); showAPIKeyWindow.open(); - console.log(data); }) .catch(err => { - osparc.component.message.FlashMessenger.getInstance().logAs(this.tr("Failed creating API token"), "ERROR"); - console.error(err); + osparc.component.message.FlashMessenger.getInstance().logAs(err.message, "ERROR"); }) - .finally(() => this.__requestTokenBtn.setFetching(false)); + .finally(() => this.__requestAPIKeyBtn.setFetching(false)); }, this); createAPIKeyWindow.open(); }, - __createExternalTokensSection: function() { + __rebuildAPIKeysList: function() { + this.__apiKeysList.removeAll(); + osparc.data.Resources.get("apiKeys") + .then(apiKeysList => { + if (apiKeysList.length) { + for (let i = 0; i < apiKeysList.length; i++) { + const apiKeyForm = this.__createValidAPIKeyForm(apiKeysList[i]); + this.__apiKeysList.add(apiKeyForm); + } + } + }) + .catch(err => console.error(err)); + }, + + __createValidAPIKeyForm: function(apiKeyLabel) { + const grid = this.__createValidEntryForm(); + + const nameLabel = new qx.ui.basic.Label(apiKeyLabel); + grid.add(nameLabel, { + row: 0, + column: 0 + }); + + const delAPIKeyBtn = new qx.ui.form.Button(null, "@FontAwesome5Solid/trash-alt/14"); + delAPIKeyBtn.addListener("execute", e => { + this.__deleteAPIKey(apiKeyLabel); + }, this); + grid.add(delAPIKeyBtn, { + row: 0, + column: 2 + }); + + return grid; + }, + + __deleteAPIKey: function(apiKeyLabel) { + if (!osparc.data.Permissions.getInstance().canDo("preferences.apikey.delete", true)) { + return; + } + const params = { + data: { + "display_name": apiKeyLabel + } + }; + osparc.data.Resources.fetch("apiKeys", "delete", params) + .then(() => this.__rebuildAPIKeysList()) + .catch(err => console.error(err)); + }, + + __createTokensSection: function() { // layout const box = this._createSectionBox(this.tr("External Service Tokens")); @@ -110,29 +223,24 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { const linkBtn = new osparc.ui.form.LinkButton(this.tr("To DAT-Core"), "https://app.blackfynn.io"); box.add(linkBtn); - const tokensList = this.__externalTokensList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); + const tokensList = this.__tokensList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); box.add(tokensList); return box; }, __rebuildTokensList: function() { - this.__internalTokensList.removeAll(); - this.__externalTokensList.removeAll(); + this.__tokensList.removeAll(); osparc.data.Resources.get("tokens") .then(tokensList => { if (tokensList.length) { for (let i = 0; i < tokensList.length; i++) { const tokenForm = this.__createValidTokenForm(tokensList[i]); - if (tokensList[i].service === "osparc") { - this.__internalTokensList.add(tokenForm); - } else { - this.__externalTokensList.add(tokenForm); - } + this.__tokensList.add(tokenForm); } } else { const emptyForm = this.__createEmptyTokenForm(); - this.__externalTokensList.add(new qx.ui.form.renderer.Single(emptyForm)); + this.__tokensList.add(new qx.ui.form.renderer.Single(emptyForm)); } }) .catch(err => console.error(err)); @@ -184,43 +292,25 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { }, __createValidTokenForm: function(token) { - const label = token["keyLabel"] || token["service"]; - const service = token["service"]; - - const height = 20; - const iconHeight = height - 6; - const gr = new qx.ui.layout.Grid(10, 3); - gr.setColumnFlex(1, 1); - gr.setRowHeight(0, height); - gr.setRowHeight(1, height); - gr.setRowHeight(2, height); - const grid = new qx.ui.container.Composite(gr); + const grid = this.__createValidEntryForm(); + const service = token["service"]; const nameLabel = new qx.ui.basic.Label(service); grid.add(nameLabel, { row: 0, column: 0 }); + const label = token["keyLabel"] || token["service"]; const nameVal = new qx.ui.basic.Label(label); grid.add(nameVal, { row: 0, column: 1 }); - const delTokenBtn = new qx.ui.form.Button(null, "@FontAwesome5Solid/trash-alt/" + iconHeight); + const delTokenBtn = new qx.ui.form.Button(null, "@FontAwesome5Solid/trash-alt/14"); delTokenBtn.addListener("execute", e => { - if (!osparc.data.Permissions.getInstance().canDo("preferences.token.delete", true)) { - return; - } - const params = { - url: { - service - } - }; - osparc.data.Resources.fetch("tokens", "delete", params, service) - .then(() => this.__rebuildTokensList()) - .catch(err => console.error(err)); + this.__deleteToken(service); }, this); grid.add(delTokenBtn, { row: 0, @@ -230,65 +320,29 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { return grid; }, - __createPasswordSection: function() { - // layout - const box = this._createSectionBox(this.tr("Password")); - - const currentPassword = new qx.ui.form.PasswordField().set({ - required: true, - placeholder: this.tr("Your current password") - }); - box.add(currentPassword); - - const newPassword = new qx.ui.form.PasswordField().set({ - required: true, - placeholder: this.tr("Your new password") - }); - box.add(newPassword); - - const confirm = new qx.ui.form.PasswordField().set({ - required: true, - placeholder: this.tr("Retype your new password") - }); - box.add(confirm); - - const manager = new qx.ui.form.validation.Manager(); - manager.setValidator(function(_itemForms) { - return osparc.auth.core.Utils.checkSamePasswords(newPassword, confirm); - }); - - const resetBtn = new qx.ui.form.Button("Reset Password").set({ - allowGrowX: false - }); - box.add(resetBtn); - - resetBtn.addListener("execute", () => { - if (manager.validate()) { - const params = { - data: { - current: currentPassword.getValue(), - new: newPassword.getValue(), - confirm: confirm.getValue() - } - }; - osparc.data.Resources.fetch("password", "post", params) - .then(data => { - osparc.component.message.FlashMessenger.getInstance().log(data); - [currentPassword, newPassword, confirm].forEach(item => { - item.resetValue(); - }); - }) - .catch(err => { - console.error(err); - osparc.component.message.FlashMessenger.getInstance().logAs(this.tr("Failed to reset password"), "ERROR"); - [currentPassword, newPassword, confirm].forEach(item => { - item.resetValue(); - }); - }); + __deleteToken: function(service) { + if (!osparc.data.Permissions.getInstance().canDo("preferences.token.delete", true)) { + return; + } + const params = { + url: { + service } - }); + }; + osparc.data.Resources.fetch("tokens", "delete", params, service) + .then(() => this.__rebuildTokensList()) + .catch(err => console.error(err)); + }, - return box; + __createValidEntryForm: function() { + const height = 20; + const gr = new qx.ui.layout.Grid(10, 3); + gr.setColumnFlex(1, 1); + gr.setRowHeight(0, height); + gr.setRowHeight(1, height); + gr.setRowHeight(2, height); + const grid = new qx.ui.container.Composite(gr); + return grid; } } }); diff --git a/services/web/client/source/class/osparc/store/Store.js b/services/web/client/source/class/osparc/store/Store.js index 6a6d70b4e4b..d34aa389e4f 100644 --- a/services/web/client/source/class/osparc/store/Store.js +++ b/services/web/client/source/class/osparc/store/Store.js @@ -73,6 +73,10 @@ qx.Class.define("osparc.store.Store", { check: "Object", init: {} }, + apiKeys: { + check: "Array", + init: [] + }, tokens: { check: "Array", init: [] diff --git a/services/web/server/src/simcore_service_webserver/security_roles.py b/services/web/server/src/simcore_service_webserver/security_roles.py index dcd69710af2..9a7267cc189 100644 --- a/services/web/server/src/simcore_service_webserver/security_roles.py +++ b/services/web/server/src/simcore_service_webserver/security_roles.py @@ -56,6 +56,8 @@ "project.tag.*", # "study.tag" "user.profile.update", # "preferences.user.update", # "preferences.role.update" + "user.apikey.*", # "preferences.apikey.create", + # "preferences.apikey.delete" "user.tokens.*", # "preferences.token.create", # "preferences.token.delete" "tag.crud.*" # "preferences.tag" @@ -85,6 +87,8 @@ ### "studies.user.create", ### "storage.datcore.read", ### "preferences.user.update", +### "preferences.apikey.create", +### "preferences.apikey.delete", ### "preferences.token.create", ### "preferences.token.delete", ### "study.node.create", From 656a1fe97b80e352c355e6ba7d11c52a61b1f679 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 16:30:24 +0200 Subject: [PATCH 29/33] Sets role-based access to allow only users to create tokens --- .../login/api_keys_handlers.py | 12 ++++++++---- .../web/server/tests/unit/with_dbs/test_api_keys.py | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py b/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py index 3d78d79ed98..bfb8e796b76 100644 --- a/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py +++ b/services/web/server/src/simcore_service_webserver/login/api_keys_handlers.py @@ -4,13 +4,13 @@ import sqlalchemy as sa from aiohttp import web +from aiopg.sa.result import ResultProxy, RowProxy import simcore_postgres_database.webserver_models as orm -from servicelib.application_keys import APP_DB_ENGINE_KEY from servicelib.aiopg_utils import DatabaseError +from servicelib.application_keys import APP_DB_ENGINE_KEY -from aiopg.sa.result import ResultProxy, RowProxy - +from ..security_api import check_permission from .decorators import RQT_USERID_KEY, login_required from .utils import get_random_string @@ -55,7 +55,6 @@ async def create(self, name: str, *, api_key: str, api_secret: str): res: ResultProxy = await conn.execute(stmt) print(res) - async def delete_api_key(self, name: str): async with self.engine.acquire() as conn: stmt = orm.api_keys.delete().where( @@ -72,6 +71,8 @@ async def list_api_keys(request: web.Request): """ GET /auth/api-keys """ + await check_permission(request, "user.apikey.*") + crud = CRUD(request) names = await crud.list_api_key_names() return names @@ -82,6 +83,8 @@ async def create_api_key(request: web.Request): """ POST /auth/api-keys """ + await check_permission(request, "user.apikey.*") + body = await request.json() display_name = body.get("display_name") @@ -105,6 +108,7 @@ async def delete_api_key(request: web.Request): """ DELETE /auth/api-keys """ + await check_permission(request, "user.apikey.*") body = await request.json() display_name = body.get("display_name") diff --git a/services/web/server/tests/unit/with_dbs/test_api_keys.py b/services/web/server/tests/unit/with_dbs/test_api_keys.py index 0e8b743ea2a..618ba3594c5 100644 --- a/services/web/server/tests/unit/with_dbs/test_api_keys.py +++ b/services/web/server/tests/unit/with_dbs/test_api_keys.py @@ -55,7 +55,7 @@ def get(self, *_args): USER_ACCESS_PARAMETERS = [ (UserRole.ANONYMOUS, web.HTTPUnauthorized), - # TODO: (UserRole.GUEST, web.HTTPUnauthorized), + (UserRole.GUEST, web.HTTPUnauthorized), (UserRole.USER, web.HTTPOk), (UserRole.TESTER, web.HTTPOk), ] @@ -92,7 +92,7 @@ async def test_create_api_keys(client, logged_user, user_role, expected): @pytest.mark.parametrize("user_role,expected", [ (UserRole.ANONYMOUS, web.HTTPUnauthorized), - (UserRole.GUEST, web.HTTPNoContent), + (UserRole.GUEST, web.HTTPUnauthorized), (UserRole.USER, web.HTTPNoContent), (UserRole.TESTER, web.HTTPNoContent), ]) From 32f2c77874afbacce63b795e82cef6d46acd6a1d Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 16:34:44 +0200 Subject: [PATCH 30/33] Tests access to new entrypoints --- services/web/server/tests/unit/with_dbs/test_api_keys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/test_api_keys.py b/services/web/server/tests/unit/with_dbs/test_api_keys.py index 618ba3594c5..21bff5f5dbc 100644 --- a/services/web/server/tests/unit/with_dbs/test_api_keys.py +++ b/services/web/server/tests/unit/with_dbs/test_api_keys.py @@ -55,7 +55,7 @@ def get(self, *_args): USER_ACCESS_PARAMETERS = [ (UserRole.ANONYMOUS, web.HTTPUnauthorized), - (UserRole.GUEST, web.HTTPUnauthorized), + (UserRole.GUEST, web.HTTPForbidden), (UserRole.USER, web.HTTPOk), (UserRole.TESTER, web.HTTPOk), ] @@ -92,7 +92,7 @@ async def test_create_api_keys(client, logged_user, user_role, expected): @pytest.mark.parametrize("user_role,expected", [ (UserRole.ANONYMOUS, web.HTTPUnauthorized), - (UserRole.GUEST, web.HTTPUnauthorized), + (UserRole.GUEST, web.HTTPForbidden), (UserRole.USER, web.HTTPNoContent), (UserRole.TESTER, web.HTTPNoContent), ]) From 1f819ef916e1c75ffed63f8469fd3991bef08efb Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 16:34:53 +0200 Subject: [PATCH 31/33] Update responses --- api/specs/webserver/openapi-auth.yaml | 14 +++++++++++--- .../simcore_service_webserver/api/v0/openapi.yaml | 12 +++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/api/specs/webserver/openapi-auth.yaml b/api/specs/webserver/openapi-auth.yaml index 7213be13cce..a15de1b64ca 100644 --- a/api/specs/webserver/openapi-auth.yaml +++ b/api/specs/webserver/openapi-auth.yaml @@ -226,7 +226,10 @@ paths: items: type: string "401": - description: unauthorized to list keys + description: requires login to list keys + "403": + description: not enough permissions to list keys + post: summary: creates API keys to access public API tags: @@ -248,7 +251,9 @@ paths: "400": description: key name requested is invalid "401": - description: unauthorized to create a key + description: requires login to create a key + "403": + description: not enough permissions to create a key delete: summary: deletes API key by name @@ -265,7 +270,10 @@ paths: "204": description: api key successfully deleted "401": - description: unauthorized to delete a key + description: requires login to delete a key + "403": + description: not enough permissions to delete a key + components: responses: DefaultErrorResponse: diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 7c71b30c2ff..52a7222ca86 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2022,7 +2022,9 @@ paths: items: type: string '401': - description: unauthorized to list keys + description: requires login to list keys + '403': + description: not enough permissions to list keys post: summary: creates API keys to access public API tags: @@ -2054,7 +2056,9 @@ paths: '400': description: key name requested is invalid '401': - description: unauthorized to create a key + description: requires login to create a key + '403': + description: not enough permissions to create a key delete: summary: deletes API key by name tags: @@ -2073,7 +2077,9 @@ paths: '204': description: api key successfully deleted '401': - description: unauthorized to delete a key + description: requires login to delete a key + '403': + description: not enough permissions to delete a key /me: get: operationId: get_my_profile From d84d10ec93b17469a169da13a4876b9b6bd44136 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 16:36:26 +0200 Subject: [PATCH 32/33] =?UTF-8?q?webserver=20api=20version:=200.5.0=20?= =?UTF-8?q?=E2=86=92=200.5.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/specs/webserver/openapi.yaml | 2 +- services/web/server/VERSION | 2 +- services/web/server/setup.cfg | 2 +- services/web/server/setup.py | 2 +- .../server/src/simcore_service_webserver/api/v0/openapi.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 79f2a1c59e4..4b438345b17 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.0 info: title: "osparc-simcore RESTful API" - version: 0.5.0 + version: 0.5.1 description: "RESTful API designed for web clients" contact: name: IT'IS Foundation diff --git a/services/web/server/VERSION b/services/web/server/VERSION index 8f0916f768f..4b9fcbec101 100644 --- a/services/web/server/VERSION +++ b/services/web/server/VERSION @@ -1 +1 @@ -0.5.0 +0.5.1 diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 1b748814bff..49687f0e9f4 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.5.0 +current_version = 0.5.1 commit = True message = webserver api version: {current_version} → {new_version} tag = False diff --git a/services/web/server/setup.py b/services/web/server/setup.py index 0edfbabb7da..22d90f81a1f 100644 --- a/services/web/server/setup.py +++ b/services/web/server/setup.py @@ -23,7 +23,7 @@ def read_reqs(reqs_path: Path): setup( name="simcore-service-webserver", - version="0.5.0", + version="0.5.1", packages=find_packages(where="src"), package_dir={"": "src",}, include_package_data=True, diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 52a7222ca86..151a5a01d4b 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.0 info: title: osparc-simcore RESTful API - version: 0.5.0 + version: 0.5.1 description: RESTful API designed for web clients contact: name: IT'IS Foundation From 00e9bf659e1fd750a683756fea5ca7ea88887b43 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 28 Apr 2020 16:39:56 +0200 Subject: [PATCH 33/33] Minor cleanup --- .../server/tests/unit/with_dbs/test_api_keys.py | 17 ++++++++++------- .../server/tests/unit/with_dbs/test_projects.py | 8 ++------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/test_api_keys.py b/services/web/server/tests/unit/with_dbs/test_api_keys.py index 21bff5f5dbc..5f61ab231a0 100644 --- a/services/web/server/tests/unit/with_dbs/test_api_keys.py +++ b/services/web/server/tests/unit/with_dbs/test_api_keys.py @@ -40,7 +40,7 @@ class Adapter: def get(self, *_args): return self.userid - crud = ApiKeysCRUD(Adapter(client.app, logged_user['id'])) + crud = ApiKeysCRUD(Adapter(client.app, logged_user["id"])) for name in names: await crud.create(name, api_key=f"{name}-key", api_secret=f"{name}-secret") @@ -90,12 +90,15 @@ async def test_create_api_keys(client, logged_user, user_role, expected): ] -@pytest.mark.parametrize("user_role,expected", [ - (UserRole.ANONYMOUS, web.HTTPUnauthorized), - (UserRole.GUEST, web.HTTPForbidden), - (UserRole.USER, web.HTTPNoContent), - (UserRole.TESTER, web.HTTPNoContent), -]) +@pytest.mark.parametrize( + "user_role,expected", + [ + (UserRole.ANONYMOUS, web.HTTPUnauthorized), + (UserRole.GUEST, web.HTTPForbidden), + (UserRole.USER, web.HTTPNoContent), + (UserRole.TESTER, web.HTTPNoContent), + ], +) async def test_delete_api_keys( client, fake_user_api_keys, logged_user, user_role, expected ): diff --git a/services/web/server/tests/unit/with_dbs/test_projects.py b/services/web/server/tests/unit/with_dbs/test_projects.py index 88ea68c5643..8c277ad47d6 100644 --- a/services/web/server/tests/unit/with_dbs/test_projects.py +++ b/services/web/server/tests/unit/with_dbs/test_projects.py @@ -19,8 +19,7 @@ from pytest_simcore.helpers.utils_assert import assert_status from pytest_simcore.helpers.utils_login import LoggedUser -from pytest_simcore.helpers.utils_projects import (NewProject, - delete_all_projects) +from pytest_simcore.helpers.utils_projects import NewProject, delete_all_projects from servicelib.application import create_safe_application from servicelib.application_keys import APP_CONFIG_KEY from servicelib.rest_responses import unwrap_envelope @@ -700,9 +699,7 @@ async def test_close_project( calls = [ call(client.server.app, user_project["uuid"], logged_user["id"]), ] - mocked_director_subsystem[ - "get_running_interactive_services" - ].has_calls(calls) + mocked_director_subsystem["get_running_interactive_services"].has_calls(calls) mocked_director_subsystem["get_running_interactive_services"].reset_mock() # close project @@ -719,7 +716,6 @@ async def test_close_project( mocked_director_subsystem["stop_service"].has_calls(calls) - @pytest.mark.parametrize( "user_role, expected", [