♻️ Maintenance/refactors servicelib (#2516)

- adds extra aiohttp to separate dependencies exclusive to aiohttp-services
- can install and test using minimal installation (e.g. for fastapi-based services) or w/ aiohttp extra (for aiohttp-based services)

Author: pcrespov

.github/workflows/ci-testing-deploy.yml

@@ -804,6 +804,63 @@ jobs:
           name: codeclimate-${{ github.job }}-coverage
           path: codeclimate.${{ github.job }}_coverage.json
 
+  unit-test-service-library-aiohttp:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
+    name: "[unit] service-library[aiohttp]"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: [3.8]
+        os: [ubuntu-20.04]
+        docker_buildx: [v0.5.1]
+        docker_compose: [1.29.1]
+        include:
+          - docker_compose: 1.29.1
+            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: ${{ matrix.docker_buildx }}
+          driver: docker
+      - name: setup docker-compose
+        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
+      - name: setup python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - uses: actions/cache@v2
+        name: getting cached data
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-service-library-${{ hashFiles('packages/service-library/requirements/ci[aiohttp].txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-service-library-
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: install
+        run: ./ci/github/unit-testing/service-library.bash install_aiohttp
+      - name: test
+        run: ./ci/github/unit-testing/service-library.bash test_aiohttp
+      - uses: codecov/[email protected]
+        with:
+          flags: unittests #optional
+      - name: prepare codeclimate coverage file
+        run: |
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
+          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
+          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
+      - name: upload codeclimate coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: codeclimate-${{ github.job }}-coverage
+          path: codeclimate.${{ github.job }}_coverage.json
+
   unit-test-settings-library:
     timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] settings-library"

Makefile

@@ -382,6 +382,7 @@ pylint: ## Runs python linter framework's wide
 											-not -path "*sandbox*" \
 											-not -path "*-sdk/python*" \
 											-not -path "*generated_code*" \
+											-not -path "*build*" \
 											-not -path "*datcore.py" \
 											-not -path "*web/server*"))"
 	# See exit codes and command line https://pylint.readthedocs.io/en/latest/user_guide/run.html#exit-codes

ci/github/unit-testing/service-library.bash

@@ -11,7 +11,20 @@ install() {
     pip list -v
 }
 
+install_aiohttp() {
+    bash ci/helpers/ensure_python_pip.bash
+    pushd packages/service-library; pip3 install -r requirements/ci[aiohttp].txt; popd;
+    pip list -v
+}
+
 test() {
+    pytest --cov=servicelib --durations=10 --cov-append \
+          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+          -v -m "not travis" --ignore=packages/service-library/tests/aiohttp \
+          packages/service-library/tests
+}
+
+test_aiohttp() {
     pytest --cov=servicelib --durations=10 --cov-append \
           --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
           -v -m "not travis" packages/service-library/tests

ci/helpers/requirements.in

@@ -4,3 +4,4 @@
 celery==4.4.7
 docker==4.3.1
 pyjwt==1.7.1
+aiohttp==3.7.4.post0

ci/helpers/requirements.txt

@@ -1,27 +1,39 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
-#    pip-compile
+#    pip-compile requirements.in
 #
+aiohttp==3.7.4.post0
+    # via -r requirements.in
 amqp==2.6.1
     # via kombu
+async-timeout==3.0.1
+    # via aiohttp
+attrs==21.2.0
+    # via aiohttp
 billiard==3.6.3.0
     # via celery
 celery==4.4.7
     # via -r requirements.in
 certifi==2020.6.20
     # via requests
 chardet==3.0.4
-    # via requests
+    # via
+    #   aiohttp
+    #   requests
 docker==4.3.1
     # via -r requirements.in
 idna==2.10
-    # via requests
-importlib-metadata==2.0.0
-    # via kombu
+    # via
+    #   requests
+    #   yarl
 kombu==4.6.11
     # via celery
+multidict==5.1.0
+    # via
+    #   aiohttp
+    #   yarl
 pyjwt==1.7.1
     # via -r requirements.in
 pytz==2020.1
@@ -32,6 +44,8 @@ six==1.15.0
     # via
     #   docker
     #   websocket-client
+typing-extensions==3.10.0.2
+    # via aiohttp
 urllib3==1.25.10
     # via requests
 vine==1.3.0
@@ -40,5 +54,5 @@ vine==1.3.0
     #   celery
 websocket-client==0.57.0
     # via docker
-zipp==3.2.0
-    # via importlib-metadata
+yarl==1.6.3
+    # via aiohttp

docs/no-upstream-push.md

@@ -19,7 +19,7 @@ your local repository.
     $ git remote add upstream [email protected]:GITUSER/osparc-simcore.git
 
     \path-to\osparc-simcore (master -> origin)
-    $ git remote set-url upstream --push "You shall not push but use PR instead"
+    $ git remote set-url upstream --push "You_shall_not_push_but_use_PR_instead"
 
     \path-to\osparc-simcore (master -> origin)
     $ git remote -vv

packages/pytest-simcore/src/pytest_simcore/helpers/utils_assert.py

@@ -6,7 +6,7 @@
 
 from aiohttp import ClientResponse
 from aiohttp.web import HTTPError, HTTPException, HTTPInternalServerError, HTTPNoContent
-from servicelib.rest_responses import unwrap_envelope
+from servicelib.aiohttp.rest_responses import unwrap_envelope
 
 
 async def assert_status(

packages/pytest-simcore/src/pytest_simcore/helpers/utils_tokens.py

@@ -8,7 +8,7 @@
 from sqlalchemy import JSON, String, cast
 from sqlalchemy.sql import and_  # , or_, not_
 
-from servicelib.aiopg_utils import DSN
+from servicelib.common_aiopg_utils import DSN
 from simcore_service_webserver.db_models import metadata, tokens, users
 from simcore_service_webserver.login.utils import get_random_string
 

packages/service-library/Makefile

@@ -3,14 +3,42 @@
 #
 include ../../scripts/common.Makefile
 
-
-.PHONY: install-dev install-prod install-ci
-install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode
+help:	## overwrites and calls common help function
+	$(MAKE) -f ../../scripts/common.Makefile $@
+	@echo '🚨 BEWARE there are issues with brakets in names `make test[aiohttp]` will raise errors:'
+	@echo '✅` `make "test[aiohttp]"` works as intended'
+	@echo '👓 Please note: when INSTALLING, DEVELOPING and TESTING '
+	@echo 'servicelib support for aiohttp extras is not included:'
+	@echo '- `make install-dev`: without aiohttp extras'
+	@echo '- `make "install-dev[aiohttp]"`: with aiohttp extras'
+    
+.PHONY: install-dev install-dev[aiohttp] install-ci install-ci[aiohttp]
+install-dev install-dev[aiohttp] install-ci install-ci[aiohttp]: _check_venv_active ## install app in development/production or CI mode
 	# installing in $(subst install-,,$@) mode
 	pip-sync requirements/$(subst install-,,$@).txt
 
 
-.PHONY: tests
-tests: ## runs unit tests
-	# running unit tests
+.PHONY: test-dev
+test-dev: ## runs unit tests in without aiohtto extras
+	# running test-dev
+	@pytest -vv --exitfirst --failed-first --durations=10 --pdb --ignore=tests/aiohttp $(CURDIR)/tests
+
+.PHONY: test-dev[aiohttp]
+test-dev[aiohttp]: ## runs unit tests and aiohtto extras
+	# running test-dev[aiohttp]
 	@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests
+
+.PHONY: tests
+tests: ## runs unit tests in without aiohtto extras
+	# running tests
+	@pytest -vv --color=yes --exitfirst --failed-first --durations=10 \
+	--cov-config=../../.coveragerc --cov=servicelib --cov-report=term-missing \
+	--pdb --ignore=tests/aiohttp $(CURDIR)/tests
+
+.PHONY: tests[aiohttp]
+tests[aiohttp]: ## runs unit tests and aiohtto extras
+	# running tests[aiohttp]
+	@pytest -vv --color=yes --exitfirst --failed-first --durations=10 \
+	--cov-config=../../.coveragerc --cov=servicelib --cov-report=term-missing \
+	--pdb $(CURDIR)/tests
+

packages/service-library/requirements/Makefile

@@ -4,3 +4,4 @@
 include ../../../requirements/base.Makefile
 
 # Add here any extra explicit dependency: e.g. _migration.txt: _base.txt
+_aiohttp.in: _base.txt

packages/service-library/requirements/_aiohttp.in

@@ -0,0 +1,20 @@
+#
+# Dependencies for exclusive to aiohttp-based services
+#
+#
+
+--constraint ../../../requirements/constraints.txt
+--constraint ./_base.in
+
+openapi-core==0.12.0     # frozen until https://github.com/ITISFoundation/osparc-simcore/pull/1396 is CLOSED
+lazy-object-proxy~=1.4.3 # cannot upgrade due to contraints in openapi-core
+
+aiohttp
+aiozipkin
+aiopg[sa]
+jsonschema
+prometheus_client
+attrs
+trafaret
+
+werkzeug

packages/service-library/requirements/_aiohttp.txt

@@ -0,0 +1,100 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    pip-compile --output-file=requirements/_aiohttp.txt --strip-extras requirements/_aiohttp.in
+#
+aiohttp==3.7.4.post0
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/./../../../requirements/constraints.txt
+    #   -r requirements/_aiohttp.in
+    #   aiozipkin
+aiopg==1.3.1
+    # via -r requirements/_aiohttp.in
+aiozipkin==0.7.1
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/./../../../requirements/constraints.txt
+    #   -r requirements/_aiohttp.in
+async-timeout==3.0.1
+    # via
+    #   aiohttp
+    #   aiopg
+attrs==20.3.0
+    # via
+    #   -c requirements/././constraints.txt
+    #   -r requirements/_aiohttp.in
+    #   aiohttp
+    #   jsonschema
+    #   openapi-core
+chardet==4.0.0
+    # via aiohttp
+greenlet==1.1.1
+    # via sqlalchemy
+idna==2.10
+    # via
+    #   -c requirements/././constraints.txt
+    #   yarl
+isodate==0.6.0
+    # via
+    #   openapi-core
+    #   openapi-schema-validator
+jsonschema==3.2.0
+    # via
+    #   -r requirements/_aiohttp.in
+    #   openapi-schema-validator
+    #   openapi-spec-validator
+lazy-object-proxy==1.4.3
+    # via
+    #   -r requirements/_aiohttp.in
+    #   openapi-core
+multidict==5.1.0
+    # via
+    #   aiohttp
+    #   yarl
+openapi-core==0.12.0
+    # via -r requirements/_aiohttp.in
+openapi-schema-validator==0.1.5
+    # via openapi-spec-validator
+openapi-spec-validator==0.3.1
+    # via openapi-core
+prometheus-client==0.11.0
+    # via -r requirements/_aiohttp.in
+psycopg2-binary==2.9.1
+    # via
+    #   aiopg
+    #   sqlalchemy
+pyrsistent==0.18.0
+    # via jsonschema
+pyyaml==5.4.1
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/./../../../requirements/constraints.txt
+    #   -c requirements/./_base.in
+    #   openapi-spec-validator
+six==1.16.0
+    # via
+    #   isodate
+    #   jsonschema
+    #   openapi-core
+    #   openapi-schema-validator
+    #   openapi-spec-validator
+sqlalchemy==1.4.23
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/./../../../requirements/constraints.txt
+    #   aiopg
+strict-rfc3339==0.7
+    # via openapi-core
+trafaret==2.1.0
+    # via -r requirements/_aiohttp.in
+typing-extensions==3.10.0.2
+    # via aiohttp
+werkzeug==2.0.1
+    # via -r requirements/_aiohttp.in
+yarl==1.6.3
+    # via aiohttp
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools