FREEZE_XieXie (#1669)

- UI/UX improvements (#1657)
- Bump yarl from 1.4.2 to 1.5.1 in /packages/postgres-database (#1665)
- Bump ujson from 3.0.0 to 3.1.0 in /packages/service-library (#1664)
- Bump pytest-docker from 0.7.2 to 0.8.0 in /packages/service-library (#1647)
- Improving storage performance (#1659)
- Bump aiozipkin from 0.6.0 to 0.7.0 in /packages/service-library (#1642)
- Theming (#1656)
- Platform stability:  (#1645)
- is1594 fix and re-activate e2e testing (#1620)
- 2 bugs fixed + Some improvements (#1634)
- Fixes default (#1640)
- Bump lodash from 4.17.15 to 4.17.19 (#1639)
- Is1585/cleanup storage (#1586)
- Fixes on publish studies handling (#1632)
- Some enhancements and bug fixes (#1608)
- Improve e2e  (#1631)
- filter studies by name before deleting them (#1629)
- Maintenance/upgrades test tools (#1628)
- Bugfix/concurent opening projects (#1598)
- Bugfix/allow reading groups anonymous user (#1615)
- Bump docker from 4.2.1 to 4.2.2 in /packages/postgres-database (#1605)
- fix testing if node has gpu support (#1604)
- [bugfix] Invalidate cache before starting a study (#1602)
- Feature/fix e2e 2 (#1600)
- fix deploy not needing e2e testing since it is disabled
- reduce cardinality of metrics (#1593)
- Excudes e2e stage from include until fixed (#1595)
- Shared project concurrency (frontend) (#1591)
- Homogenize studies and services (#1569)
- [feature] UI Fine grained access - project locking and notification
- Bugfix/apiserver does not need sslheaders (#1564)
- Cleanup catalog service (#1582)
- Maintenance/cleanup api server (#1578)
- Adds support for GPU scheduling of computational services (#1553)
- Maintenance/upgrades and tooling (#1546)
- Is1570/study fails 500 (#1572)
- Bump faker from 4.1.0 to 4.1.1 in /packages/postgres-database (#1573)
- maintenance fix codecov reports (#1568)
- Manage groups, Share studies (#1512)
- Is/add notebook migration script (#1565)
- Is1269/api-server upgrade (#1475)
- added simcore_webserver_service in pytest simcore package (#1563)
- add traefik endpoint to api-gateway (#1555)

Author: odeimaiz

.codeclimate.yml

@@ -78,3 +78,4 @@ exclude_patterns:
   - "**/migrations/"
   - "**/*.js"
   - "**/pytest-simcore/"
+  - "**/sandbox/"

.codecov.yml

@@ -0,0 +1,56 @@
+codecov:
+    require_ci_to_pass: yes
+    branch: master
+
+coverage:
+  precision: 1
+  round: down
+  range: "70...100"
+
+  status:
+    project:
+      default:
+        informational: true
+        threshold: 1%
+        paths:
+          - api
+          - packages
+          - services
+      api:
+        informational: true
+        threshold: 1%
+        paths:
+          - api
+      packages:
+        informational: true
+        threshold: 1%
+        paths:
+          - packages
+      services:
+        informational: true
+        threshold: 1%
+        paths:
+          - services
+
+
+    patch:
+      default:
+        informational: true
+        threshold: 1%
+        paths:
+          - api
+          - packages
+          - services
+
+parsers:
+  gcov:
+    branch_detection:
+      conditional: yes
+      loop: yes
+      method: no
+      macro: no
+
+comment:
+  layout: "reach,diff,flags,tree"
+  behavior: default
+  require_changes: no

.env-devel

@@ -47,6 +47,7 @@ TRACING_ZIPKIN_ENDPOINT=http://jaeger:9411
 
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
 
+WEBSERVER_HOST=webserver
 WEBSERVER_LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0
 WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED=0
 # python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key())"

.github/CODEOWNERS

@@ -13,7 +13,7 @@ Makefile                                  @pcrespov, @sanderegg
 /scripts/demo                             @odeimaiz, @pcrespov
 /scripts/json-schema-to-openapi-schema    @sanderegg
 /scripts/template-projects                @odeimaiz, @pcrespov
-/services/api-gateway                     @pcrespov
+/services/api-server                      @pcrespov
 /services/catalog                         @pcrespov
 /services/sidecar                         @pcrespov, @mguidon
 /services/web/client                      @odeimaiz, @oetiker, @ignapas

.github/workflows/ci-testing-deploy.yml

@@ -56,8 +56,8 @@ jobs:
       - name: test
         run: ./ci/github/unit-testing/api.bash test
 
-  unit-test-api-gateway:
-    name: Unit-testing api-gateway
+  unit-test-api-server:
+    name: Unit-testing api-server
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -86,22 +86,22 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: install
-        run: ./ci/github/unit-testing/api-gateway.bash install
+        run: ./ci/github/unit-testing/api-server.bash install
       - name: test
-        run: ./ci/github/unit-testing/api-gateway.bash test
+        run: ./ci/github/unit-testing/api-server.bash test
       - uses: codecov/codecov-action@v1
         with:
           flags: unittests #optional
       - name: prepare codeclimate coverage file
         run: |
           curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
           chmod +x ./cc-test-reporter
-          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.unit_api_gateway_coverage.json coverage.xml
+          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.unit_api_server_coverage.json coverage.xml
       - name: upload codeclimate coverage
         uses: actions/upload-artifact@v1
         with:
-          name: unit_api_gateway_coverage
-          path: codeclimate.unit_api_gateway_coverage.json
+          name: unit_api_server_coverage
+          path: codeclimate.unit_api_server_coverage.json
 
   unit-test-catalog:
     name: Unit-testing catalog
@@ -748,6 +748,8 @@ jobs:
         run: ./ci/github/system-testing/swarm-deploy.bash clean_up
 
   system-test-e2e:
+    # FIXME: skip the job until make it faster and more reliable
+    # https://github.com/ITISFoundation/osparc-simcore/issues/1594
     name: System-testing e2e
     needs: [build-test-images]
     runs-on: ${{ matrix.os }}
@@ -794,8 +796,15 @@ jobs:
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-node-
-      - name: install
-        run: ./ci/github/system-testing/e2e.bash install
+      - name: setup images
+        run: ./ci/github/system-testing/e2e.bash setup_images
+      - name: setup environment
+        run: ./ci/github/system-testing/e2e.bash setup_environment
+      - name: setup registry
+        run: ./ci/github/system-testing/e2e.bash setup_registry
+      - name: setup database
+        timeout-minutes: 2
+        run: ./ci/github/system-testing/e2e.bash setup_database
       - name: test
         run: ./ci/github/system-testing/e2e.bash test
       - name: recover docker logs
@@ -838,7 +847,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/download-artifact@v1
         with:
-          name: unit_api_gateway_coverage
+          name: unit_api_server_coverage
       - uses: actions/download-artifact@v1
         with:
           name: unit_catalog_coverage
@@ -873,7 +882,7 @@ jobs:
         run: |
           mkdir all_coverages
           cp \
-            unit_api_gateway_coverage/*.json \
+            unit_api_server_coverage/*.json \
             unit_catalog_coverage/*.json \
             unit_director_coverage/*.json \
             unit_sidecar_coverage/*.json \

.gitignore

@@ -110,7 +110,8 @@ ENV/
 node_modules/
 
 # IDEs config
-.vscode/
+.vscode/launch.json
+.vscode/settings.json
 
 # Manual overrides
 services/docker-compose.override.yml
@@ -150,3 +151,7 @@ prof/
 
 # outputs from make
 .stack-*.yml
+
+
+# Copies
+services/**/.codeclimate.yml

.pylintrc

@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-whitelist=
+extension-pkg-whitelist=pydantic
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.

.travis.yml

@@ -107,9 +107,9 @@ jobs:
       after_failure:
         - unbuffer bash ci/travis/unit-testing/webserver.bash after_failure
 
-    # test python, api-gateway ----------------------------------------------------------------------
+    # test python, api-server ----------------------------------------------------------------------
     - stage: build / unit-testing
-      name: api-gateway
+      name: api-server
       language: python
       python:
         - "3.6"
@@ -118,17 +118,17 @@ jobs:
         - docker
       cache: pip
       before_install:
-        - sudo bash ci/travis/unit-testing/api-gateway.bash before_install
+        - sudo bash ci/travis/unit-testing/api-server.bash before_install
       install:
-        - unbuffer bash ci/travis/unit-testing/api-gateway.bash install
+        - unbuffer bash ci/travis/unit-testing/api-server.bash install
       before_script:
-        - unbuffer bash ci/travis/unit-testing/api-gateway.bash before_script
+        - unbuffer bash ci/travis/unit-testing/api-server.bash before_script
       script:
-        - unbuffer bash ci/travis/unit-testing/api-gateway.bash script
+        - unbuffer bash ci/travis/unit-testing/api-server.bash script
       after_success:
-        - unbuffer bash ci/travis/unit-testing/api-gateway.bash after_success
+        - unbuffer bash ci/travis/unit-testing/api-server.bash after_success
       after_failure:
-        - unbuffer bash ci/travis/unit-testing/api-gateway.bash after_failure
+        - unbuffer bash ci/travis/unit-testing/api-server.bash after_failure
 
     # test python, catalog ----------------------------------------------------------------------
     - stage: build / unit-testing
@@ -385,41 +385,7 @@ jobs:
       after_failure:
         - unbuffer bash ci/travis/system-testing/swarm-deploy.bash after_failure
 
-    # system testing e2e testing -----------------------------------------------------------------
-    # https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-on-travis-ci
-    - stage: integration-testing / system-testing
-      name: e2e testing
-      language: node_js
-      node_js:
-        - "10"
-      sudo: required
-      services:
-        - xvfb
-      addons:
-        apt:
-          packages:
-            - libnss3 # This is required to run new chrome on old trusty
-            - expect-dev # for unbuffer: brings color back into travis logs
-            - docker-ce
-            - python3-venv
-      notifications:
-        email: false
-      cache:
-        directories:
-          - node_modules
-      before_install:
-        - sudo bash ci/travis/system-testing/e2e.bash before_install
-      install:
-        - unbuffer bash ci/travis/system-testing/e2e.bash install
-      before_script:
-        - unbuffer bash ci/travis/system-testing/e2e.bash before_script
-      script:
-        - unbuffer bash ci/travis/system-testing/e2e.bash script
-      after_success:
-        - unbuffer bash ci/travis/system-testing/e2e.bash after_success
-      after_failure:
-        - unbuffer bash ci/travis/system-testing/e2e.bash after_failure
-
+    # deployment in master environ ----------------------------------------------------------------
     - stage: deployment
       name: master
       if: branch = master
@@ -434,6 +400,8 @@ jobs:
           script: unbuffer bash ci/deploy/dockerhub-deploy.bash
           on:
             branch: master
+
+    # deployment to staging environ ----------------------------------------------------------------
     - stage: deployment
       name: staging
       if: branch = staging
@@ -448,6 +416,8 @@ jobs:
           script: unbuffer bash ci/deploy/dockerhub-deploy.bash
           on:
             branch: staging
+
+    # deployment to production environ ----------------------------------------------------------------
     - stage: deployment
       name: production/release
       env:
@@ -464,6 +434,7 @@ jobs:
             all_branches: true
             tags: true
             condition: $TRAVIS_TAG =~ ^v[0-9]+.[0-9]+.[0-9]+$
+
 notifications:
   email:
     on_success: never

.vscode-template/README.md

@@ -1,3 +1,4 @@
+// This is a template. Clone and replace extension ".template.json" by ".json"
 {
   // Use IntelliSense to learn about possible attributes.
   // Hover to view descriptions of existing attributes.
@@ -43,6 +44,19 @@
         }
       ]
     },
+    {
+      "name": "Python: Remote Attach sidecar_gpu",
+      "type": "python",
+      "request": "attach",
+      "port": 3007,
+      "host": "127.0.0.1",
+      "pathMappings": [
+        {
+          "localRoot": "${workspaceFolder}",
+          "remoteRoot": "/devel"
+        }
+      ]
+    },
     {
       "name": "Python: Remote Attach storage",
       "type": "python",
@@ -86,4 +100,4 @@
       "port": 9229
     }
   ]
-}
+}

.vscode-template/launch.json renamed to .vscode/launch.template.json

@@ -1,3 +1,4 @@
+// This is a template. Clone and replace extension ".template.json" by ".json"
 {
   "editor.tabSize": 2,
   "editor.insertSpaces": true,
@@ -34,4 +35,4 @@
   "shellcheck.run": "onSave",
   "shellcheck.enableQuickFix": true,
   "python.formatting.provider": "black"
-}
+}