Is310/valid api response (ITISFoundation#311)

Updates servicelib
- validation of api-response against openapi specs
- automatic routing of handles against operation-id in specs
- middleware factories to envelop responses and errors. Configurable per api version. 
- package testing
- BUG: response validator only works with single-file oas (currently disabled in webserver and storage!)

Author: pcrespov

Makefile

@@ -100,7 +100,7 @@ run_test:
 	pytest -v services/apihub/tests
 	pytest --cov=pytest_docker -v packages/pytest_docker/tests
 	pytest --cov=s3wrapper -v packages/s3wrapper/tests
-	pytest --cov=simcore_sdk -v packages/simcore-sdk/tests
+	pytest --cov=servicelib -v packages/service-library/tests
 	pytest --cov=simcore_service_webserver -v services/web/server/tests/unit
 	pytest --cov=simcore_service_webserver -v services/web/server/tests/login
 	pytest --cov=simcore_service_director -v services/director/tests

api/specs/webserver/v0/openapi.yaml

@@ -11,17 +11,25 @@ info:
     url: https://github.com/ITISFoundation/osparc-simcore/blob/master/LICENSE
 servers:
   - description: Development server
-    url: http://{host}:{port}/v0
+    url: http://{host}:{port}/{basePath}
     variables:
       host:
         default: 'localhost'
       port:
         default: '8001'
+      basePath:
+        enum:
+          - v0
+        default: v0
   - description: Production server
-    url: https://webserver:{port}/v0
+    url: '{publicUrl}/{basePath}'
     variables:
-      port:
-        default: '9081'
+      publicUrl:
+        default: 'https://osparc.io'
+      basePath:
+        enum:
+          - v0
+        default: v0
 tags:
 - name: admins
   description: Secured Admin-only calls
@@ -98,6 +106,8 @@ paths:
             application/json:
               schema:
                 $ref: './components/schemas/log_message.yaml#/LogMessageEnveloped'
+        '400':
+            $ref: '#/components/responses/DataError_BadRequest_400'
         '409':
             $ref: '#/components/responses/DataError_Conflict_409'
         '422':
@@ -127,6 +137,7 @@ paths:
           $ref: '#/components/responses/DefaultErrorResponse'
   /auth/logout:
     get:
+      operationId: auth_logout
       responses:
         '200':
           description: Succesfully logged out
@@ -137,7 +148,28 @@ paths:
         default:
           $ref: '#/components/responses/DefaultErrorResponse'
   #/auth/reset-password:
-  #/auth/change-email:
+  /auth/change-email:
+    post:
+      operationId: auth_change_email
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                new_email:
+                  type: string
+                  #format: email
+      responses:
+        '200':
+          description: User has been succesfully registered.
+          content:
+            application/json:
+              schema:
+                $ref: './components/schemas/log_message.yaml#/LogMessageEnveloped'
+        default:
+          $ref: '#/components/responses/DefaultErrorResponse'
+
   #/auth/change-password:
   /auth/confirmation/{code}:
     get:
@@ -148,7 +180,7 @@ paths:
           required: true
           schema:
             type: string
-            format: uuid
+            #format: uuid
       responses:
         default:
           $ref: '#/components/responses/OK_NoContent_204'

packages/service-library/Makefile

@@ -76,13 +76,14 @@ docs: ## generate Sphinx HTML documentation, including API docs
 servedocs: docs ## compile the docs watching for changes
 	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
 
-release: dist ## package and upload a release
-	twine upload dist/*
-
 dist: clean ## builds source and wheel package
 	python setup.py sdist
 	python setup.py bdist_wheel
 	ls -l dist
 
 install: clean ## install the package to the active Python's site-packages
 	python setup.py install
+
+dev: clean
+	pip install -r requirements/dev.txt
+	pip list

packages/service-library/requirements/dev.txt

@@ -3,5 +3,4 @@
 bumpversion
 coverage
 
-pytest==3.4.2
-pytest-runner==2.11.1
+-r ../tests/requirements.txt

packages/service-library/setup.py

@@ -42,7 +42,6 @@ def list_packages(*parts):
         'License :: OSI Approved :: MIT License',
         'Natural Language :: English',
         'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
     ],
     long_description=readme + '\n\n' + history,
     license="MIT license",

packages/service-library/src/servicelib/aiopg_utils.py

@@ -12,9 +12,9 @@
 import logging
 import warnings
 
-log = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
-warnings.warn("DO NOT USER, STILL UNDER DEVELOPMENT")
+warnings.warn("DO NOT USE IN PRODUCTION, STILL UNDER DEVELOPMENT")
 
 @attr.s(auto_attribs=True)
 class AiopgExecutor:
@@ -42,9 +42,7 @@ def _compile(self, sql, *multiparams, **params):
 
     async def execute(self):
         async with self.engine.acquire() as conn:
-            log.debug(self.statement)
-            import pdb; pdb.set_trace()
-
+            logger.debug(self.statement)
             resp = await conn.execute(self.statement)
             return resp
 

packages/service-library/src/servicelib/openapi.py

@@ -12,9 +12,14 @@
 OAI_VERSION = '3.0.1'
 OAI_VERSION_URL = 'https://github.com/OAI/OpenAPI-Specification/blob/master/versions/%s.md'%OAI_VERSION
 
+# alias
+OpenApiSpec = Spec
+
+
 # TODO: ensure openapi_core.__version__ is up-to-date with OAI_VERSION
 
-def create_specs(openapi_path: Path) -> Spec:
+def create_specs(openapi_path: Path) -> OpenApiSpec:
+    # TODO: spec_from_file and spec_from_url
     with openapi_path.open() as f:
         spec_dict = yaml.safe_load(f)
 

packages/service-library/src/servicelib/openapi_validation.py

@@ -9,11 +9,12 @@
 from openapi_core import shortcuts
 from openapi_core.schema.specs.models import Spec as OpenApiSpec
 from openapi_core.validation.request.validators import RequestValidator
+from openapi_core.validation.response.validators import ResponseValidator
 
 from .openapi_wrappers import (PARAMETERS_KEYS, AiohttpOpenAPIRequest,
                                AiohttpOpenAPIResponse)
 
-log = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 #from openapi_core.wrappers.mock import MockRequest
 
@@ -32,7 +33,6 @@ async def validate_request(request: web.Request, spec: OpenApiSpec):
 
     return result.parameters, result.body, result.errors
 
-
 async def validate_parameters(spec: OpenApiSpec, request: web.Request):
     req = await AiohttpOpenAPIRequest.create(request)
     return shortcuts.validate_parameters(spec, req)
@@ -41,6 +41,9 @@ async def validate_body(spec: OpenApiSpec, request: web.Request):
     req = await AiohttpOpenAPIRequest.create(request)
     return shortcuts.validate_body(spec, req)
 
+
+
+
 async def validate_data(spec: OpenApiSpec, request, response: web.Response):
 
     if isinstance(request, web.Request):
@@ -58,7 +61,26 @@ async def validate_data(spec: OpenApiSpec, request, response: web.Response):
         req = request
 
     res = await AiohttpOpenAPIResponse.create(response)
-    return shortcuts.validate_data(spec, req, res)
+
+    validator = ResponseValidator(spec)
+    result = validator.validate(req, res)
+
+    result.raise_for_errors()
+
+    return result.data
+
+async def validate_response(spec: OpenApiSpec, request: web.Request, response: web.Response):
+    """
+      Validates server response against openapi specs
+
+      Raises exceptions OpenAPIError, OpenAPIMappingError
+    """
+    validator = ResponseValidator(spec)
+
+    req = await AiohttpOpenAPIRequest.create(request)
+    res = AiohttpOpenAPIResponse(response, response.text) # FIXME:ONLY IN SERVER side. Async in client!
+    result = validator.validate(req, res)
+    result.raise_for_errors()
 
 
 __all__ = (

packages/service-library/src/servicelib/openapi_wrappers.py

@@ -12,6 +12,7 @@
 
 CAPTURES = re.compile(r'\(\?P<([_a-zA-Z][_a-zA-Z0-9]+)>(.[^)]+)\)')
 PARAMETERS_KEYS = ('path', 'query', 'header', 'cookie')
+PATH_KEY, QUERY_KEY, HEADER_KEY, COOKIE_KEY = PARAMETERS_KEYS
 
 class AiohttpOpenAPIRequest(BaseOpenAPIRequest):
     wrappedcls = web.Request
@@ -109,16 +110,19 @@ class AiohttpOpenAPIResponse(BaseOpenAPIResponse):
 
     def __init__(self, response: web.Response, data: str):
         self._response = response
-        self._data = data
+        self._text = data
 
     @staticmethod
     async def create(response: web.Response):
-        data = await response.text()
-        return AiohttpOpenAPIResponse(response, data)
+        text = await response.text()
+        return AiohttpOpenAPIResponse(response, text)
 
     @property
-    def data(self) -> str:
-        return self._data
+    def body(self) -> str:
+        return self._text
+
+    # BUG: not part of BaseOpenAPIResponse but used in openapi-core
+    data = body
 
     @property
     def status_code(self) -> int:

packages/service-library/src/servicelib/resources.py

@@ -5,7 +5,7 @@
 import pathlib
 import pkg_resources
 from pathlib import Path
-#import typing
+from typing import TextIO
 import attr
 
 
@@ -21,16 +21,17 @@ class ResourcesFacade:
     distribution_name: str
     config_folder: str
 
-    def exists(self, resource_name: str):
+    def exists(self, resource_name: str) -> bool:
         return pkg_resources.resource_exists(self.package_name, resource_name)
 
-    def stream(self, resource_name: str):
+    def stream(self, resource_name: str) -> TextIO:
+        # TODO: check if read-only and if so, rename
         return pkg_resources.resource_stream(self.package_name, resource_name)
 
-    def listdir(self, resource_name: str):
+    def listdir(self, resource_name: str) -> str:
         return pkg_resources.resource_listdir(self.package_name, resource_name)
 
-    def isdir(self, resource_name: str):
+    def isdir(self, resource_name: str) -> str:
         return pkg_resources.resource_isdir(self.package_name, resource_name)
 
     def get_path(self, resource_name: str) -> Path:

packages/service-library/src/servicelib/response_utils.py

@@ -0,0 +1,24 @@
+""" rest - json data encoders/decodes
+
+"""
+import attr
+import json
+
+
+class DataEncoder(json.JSONEncoder):
+    """
+        Customized json encoder for rest data models
+
+        Extra encoding of:
+            - attr.s-like classes
+
+    TODO: extend to more types like apiset
+    """
+    def default(self, o): #pylint: disable=E0202
+        if attr.has(o.__class__):
+            return attr.asdict(o)
+        return json.JSONEncoder.default(self, o)
+
+
+def jsonify(payload):
+    return json.dumps(payload, cls=DataEncoder)