[maintenance] Various improvements on logs to assist debugging (#1763)

* e2e
   - timeout logs to avoid e2e hanging while creating docker logs
   - logs print time-stamps
   - fine-grained steps in e2e to reduce logs
* webserver
   - Adds app settings [add pydantic (will replace traferet in a new PR) ]
   - More metadata: passes build and git commit versions
   - Public metadata published via static.json to the front-end and in v0/config entrypoint
   - Settings logged in console for reference purposes
   - cleanup
* sidecar
   - cleanup
*requirements
   - weekly update of package requirements
   - upgrades to fix vulnerability GHSA-p9pc-299p-vxgp

Author: pcrespov

.github/workflows/ci-testing-deploy.yml

@@ -1058,6 +1058,8 @@ jobs:
             ${{ runner.os }}-node-
       - name: setup images
         run: ./ci/github/system-testing/e2e.bash setup_images
+      - name: setup and start swarm
+        run: ./ci/github/system-testing/e2e.bash setup_and_run_stack
       - name: setup environment
         run: ./ci/github/system-testing/e2e.bash setup_environment
       - name: setup registry
@@ -1070,7 +1072,7 @@ jobs:
       - name: recover docker logs
         id: logs_recovery
         if: always()
-        run: ./ci/github/system-testing/e2e.bash recover_artifacts
+        run: ./ci/github/system-testing/e2e.bash dump_docker_logs
       - name: upload docker logs
         if: always()
         uses: actions/upload-artifact@v2

Makefile

@@ -318,7 +318,7 @@ push-version: tag-version
 
 ## ENVIRONMENT -------------------------------
 
-.PHONY: devenv devenv-all
+.PHONY: devenv devenv-all node-env
 
 .venv:
 	python3 -m venv $@
@@ -338,6 +338,15 @@ devenv-all: devenv ## sets up extra development tools (everything else besides p
 	@$(MAKE_C) scripts/json-schema-to-openapi-schema
 
 
+node_modules: package.json
+	# checking npm installed
+	@npm --version
+	# installing package.json
+	npm install --package-lock
+
+nodenv: node_modules ## builds node_modules local environ (TODO)
+
+
 .env: .env-devel ## creates .env file from defaults in .env-devel
 	$(if $(wildcard $@), \
 	@echo "WARNING #####  $< is newer than $@ ####"; diff -uN $@ $<; false;,\

api/tests/requirements.txt

@@ -6,7 +6,7 @@
 #
 aiohttp==3.6.2            # via -r requirements.in, pytest-aiohttp
 async-timeout==3.0.1      # via aiohttp
-attrs==20.1.0             # via aiohttp, jsonschema, openapi-core, pytest
+attrs==20.2.0             # via aiohttp, jsonschema, openapi-core, pytest
 chardet==3.0.4            # via aiohttp
 coverage==5.2.1           # via -r requirements.in, pytest-cov
 idna-ssl==1.1.0           # via aiohttp
@@ -16,13 +16,13 @@ iniconfig==1.0.1          # via pytest
 isodate==0.6.0            # via openapi-core, openapi-schema-validator
 jsonschema==3.2.0         # via openapi-schema-validator, openapi-spec-validator
 lazy-object-proxy==1.5.1  # via openapi-core
-more-itertools==8.4.0     # via openapi-core, pytest
+more-itertools==8.5.0     # via openapi-core, pytest
 multidict==4.7.6          # via aiohttp, yarl
 openapi-core==0.13.4      # via -r requirements.in
 openapi-schema-validator==0.1.1  # via openapi-core
 openapi-spec-validator==0.2.9  # via openapi-core
 packaging==20.4           # via pytest, pytest-sugar
-parse==1.16.0             # via openapi-core
+parse==1.17.0             # via openapi-core
 pluggy==0.13.1            # via pytest
 py==1.9.0                 # via pytest
 pyparsing==2.4.7          # via packaging

ci/github/system-testing/e2e.bash

@@ -74,13 +74,17 @@ uninstall_insecure_registry() {
 }
 
 setup_images() {
-  echo "--------------- getting simcore docker images..."
+  echo "--------------- preparing docker images..."
   make pull-version || ( (make pull-cache || true) && make build-x tag-version)
   make info-images
 
+}
+
+setup_and_run_stack() {
   # configure simcore for testing with a private registry
   install_insecure_registry
 
+  echo "--------------- starting swarm ..."
   # start simcore and set log-level
   export LOG_LEVEL=WARNING
   make up-version
@@ -135,29 +139,32 @@ setup_database() {
 install() {
   ## shortcut
   setup_images
+  setup_and_run_stack
   setup_environment
   setup_registry
   setup_database
 }
 
 test() {
+  sleep 5
   pushd tests/e2e
   make test
   popd
-
 }
 
-recover_artifacts() {
+dump_docker_logs() {
   # all screenshots are in tests/e2e/screenshots if any
 
   # get docker logs.
-  # WARNING: dumping long logs might take hours!!
-  mkdir simcore_logs
-  (docker service logs --timestamps --tail=300 --details ${SWARM_STACK_NAME}_webserver >simcore_logs/webserver.log 2>&1) || true
-  (docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_director >simcore_logs/director.log 2>&1) || true
-  (docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_storage >simcore_logs/storage.log 2>&1) || true
-  (docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_sidecar >simcore_logs/sidecar.log 2>&1) || true
-  (docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_catalog >simcore_logs/catalog.log 2>&1) || true
+  # NOTE: dumping logs sometimes hangs. Introducing a timeout
+  mkdir --parents simcore_logs
+  (timeout 30 docker service logs --timestamps --tail=300 --details ${SWARM_STACK_NAME}_webserver >simcore_logs/webserver.log 2>&1) || true
+  (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_director  >simcore_logs/director.log 2>&1) || true
+  (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_storage   >simcore_logs/storage.log 2>&1) || true
+  (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_sidecar   >simcore_logs/sidecar.log 2>&1) || true
+  (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_catalog   >simcore_logs/catalog.log 2>&1) || true
+  (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_migration >simcore_logs/migration.log 2>&1) || true
+  (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_postgres >simcore_logs/postgres.log 2>&1) || true
 }
 
 clean_up() {

docs/img/.stack-simcore-version.yml.png

@@ -9,6 +9,7 @@
     "eslint-plugin-qx-rules": "^0.1.0",
     "puppeteer": "^1.19.0",
     "puppeteer-to-istanbul": "^1.2.2",
-    "yargs": "^13.3.0"
+    "yargs": "^13.3.0",
+    "yargs-parser": ">=13.1.2"
   }
 }

package-lock.json

@@ -89,11 +89,19 @@ migrate: $(DOT_ENV_FILE) ## basic migration update (use ONLY for development pur
 
 
 
-
 .PHONY: up-pg down-pg up-prod down-prod
 docker-compose-configs = $(wildcard tests/docker-compose*.yml)
 up-pg up-prod: $(docker-compose-configs) ## starts pg server
 	docker-compose -f tests/docker-compose.yml $(if $(findstring -prod,$@),-f tests/docker-compose.prod.yml,) up -d
 
 down-pg down-prod: $(docker-compose-configs) ## stops pg server
 	docker-compose -f tests/docker-compose.yml  $(if $(findstring -prod,$@),-f tests/docker-compose.prod.yml,) down
+
+
+
+.PHONY: auto-doc
+auto-doc: install-dev ## Creates entity relationship diagram (ERD) defined under ``simcore_postgres_database.models``
+	# installing doc dependencies (install-doc)
+	pip install eralchemy
+	# running script
+	python scripts/create_erd.py