diff --git a/.env-devel b/.env-devel index 11764f52218..4dfc245812e 100644 --- a/.env-devel +++ b/.env-devel @@ -1,4 +1,3 @@ -DOCKER_IMAGE_PREFIX=services_ DOCKER_IMAGE_TAG=latest PUBLISHED_HOST_NAME=localhost diff --git a/.gitignore b/.gitignore index 80a41d61939..04eb5092f01 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,4 @@ itisfoundation.github.io/ # pylint-profile output prof/ +!ops/**/build \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 2f014f68457..66e6c78dcfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,27 +8,85 @@ addons: apt: packages: - docker-ce + - expect-dev # for unbuffer: brings color back into travis logs + +stages: + - name: build / unit-testing + if: tag IS blank + - name: integration-testing + if: tag IS blank + - name: system-testing + if: tag IS blank + - deployment jobs: include: - # build cache images ---------------------------------------------------------------------- - - if: branch = master - stage: preparing docker cache images - name: build cache + # build images ---------------------------------------------------------------------- + - stage: build / unit-testing + # in pull request we do not have credentials to push, so this is useless + if: NOT type = pull_request AND tag IS blank + name: build simcore images sudo: required before_install: - - sudo bash ops/travis/build-cache/scripts before_install + - sudo bash ops/travis/build/test-images before_install + install: + - unbuffer bash ops/travis/build/test-images install + before_script: + - unbuffer bash ops/travis/build/test-images before_script + script: + - unbuffer bash ops/travis/build/test-images script + after_success: + - unbuffer bash ops/travis/build/test-images after_success + after_failure: + - unbuffer bash ops/travis/build/test-images after_failure + deploy: + - provider: script + skip_cleanup: true + script: unbuffer bash ops/travis/deploy/test-images + on: + all_branches: true + + # frontend testing ---------------------------------------------------------------------- + - stage: build / unit-testing + name: front end testing + language: node_js + node_js: + - "10" + addons: + apt: + packages: + - libgconf-2-4 + + before_install: + - bash ops/travis/unit-testing/frontend before_install install: - - bash ops/travis/build-cache/scripts install + - bash ops/travis/unit-testing/frontend install before_script: - - bash ops/travis/build-cache/scripts before_script + - bash ops/travis/unit-testing/frontend before_script script: - - bash ops/travis/build-cache/scripts script + - bash ops/travis/unit-testing/frontend script after_success: - - bash ops/travis/build-cache/scripts after_success + - bash ops/travis/unit-testing/frontend after_success + after_failure: + - bash ops/travis/unit-testing/frontend after_failure + + deploy: + # https://docs.travis-ci.com/user/deployment/pages/ + provider: pages + target-branch: master + # Set in travis-ci.org dashboard, marked secure + github-token: $GH_PAGES_TOKEN + skip-cleanup: true + keep-history: false + local-dir: itisfoundation.github.io + repo: ITISFoundation/itisfoundation.github.io + verbose: true + on: + repo: ITISFoundation/osparc-simcore + branch: master # test python, webserver ---------------------------------------------------------------------- - - stage: unit testing + - stage: build / unit-testing name: webserver language: python python: @@ -38,16 +96,18 @@ jobs: before_install: - sudo bash ops/travis/unit-testing/webserver before_install install: - - bash ops/travis/unit-testing/webserver install + - unbuffer bash ops/travis/unit-testing/webserver install before_script: - - bash ops/travis/unit-testing/webserver before_script + - unbuffer bash ops/travis/unit-testing/webserver before_script script: - - bash ops/travis/unit-testing/webserver script + - unbuffer bash ops/travis/unit-testing/webserver script after_success: - - bash ops/travis/unit-testing/webserver after_success + - unbuffer bash ops/travis/unit-testing/webserver after_success + after_failure: + - unbuffer bash ops/travis/unit-testing/webserver after_failure # test python, director ---------------------------------------------------------------------- - - stage: unit testing + - stage: build / unit-testing name: director language: python python: @@ -57,17 +117,19 @@ jobs: before_install: - sudo bash ops/travis/unit-testing/director before_install install: - - bash ops/travis/unit-testing/director install + - unbuffer bash ops/travis/unit-testing/director install before_script: - - bash ops/travis/unit-testing/director before_script + - unbuffer bash ops/travis/unit-testing/director before_script script: - - bash ops/travis/unit-testing/director script + - unbuffer bash ops/travis/unit-testing/director script after_success: - - bash ops/travis/unit-testing/director after_success + - unbuffer bash ops/travis/unit-testing/director after_success + after_failure: + - unbuffer bash ops/travis/unit-testing/director after_failure # test python, sidecar ---------------------------------------------------------------------- # TODO: activate when sidecar tests are READY! - # - stage: unit testing + # - stage: build / unit-testing # name: sidecar # language: python # python: @@ -84,28 +146,11 @@ jobs: # - bash ops/travis/unit-testing/sidecar script # after_success: # - bash ops/travis/unit-testing/sidecar after_success - - # test python, simcore-sdk ---------------------------------------------------------------------- - - stage: unit testing - name: simcore-sdk - language: python - python: - - "3.6" - sudo: required - cache: pip - before_install: - - sudo bash ops/travis/unit-testing/simcore-sdk before_install - install: - - bash ops/travis/unit-testing/simcore-sdk install - before_script: - - bash ops/travis/unit-testing/simcore-sdk before_script - script: - - bash ops/travis/unit-testing/simcore-sdk script - after_success: - - bash ops/travis/unit-testing/simcore-sdk after_success + # after_failure: + # - bash ops/travis/unit-testing/sidecar after_failure # test python, service-library ---------------------------------------------------------------------- - - stage: unit testing + - stage: build / unit-testing name: service-library language: python python: @@ -122,9 +167,11 @@ jobs: - bash ops/travis/unit-testing/service-library script after_success: - bash ops/travis/unit-testing/service-library after_success + after_failure: + - bash ops/travis/unit-testing/service-library after_failure # test python, storage ---------------------------------------------------------------------- - - stage: unit testing + - stage: build / unit-testing name: storage language: python python: @@ -136,17 +183,18 @@ jobs: before_install: - sudo bash ops/travis/unit-testing/storage before_install install: - - bash ops/travis/unit-testing/storage install + - unbuffer bash ops/travis/unit-testing/storage install before_script: - - bash ops/travis/unit-testing/storage before_script + - unbuffer bash ops/travis/unit-testing/storage before_script script: - - bash ops/travis/unit-testing/storage script + - unbuffer bash ops/travis/unit-testing/storage script after_success: - - bash ops/travis/unit-testing/storage after_success - + - unbuffer bash ops/travis/unit-testing/storage after_success + after_failure: + - unbuffer bash ops/travis/unit-testing/storage after_failure # test python, apihub ---------------------------------------------------------------------- - - stage: unit testing + - stage: build / unit-testing name: apihub language: python python: @@ -156,16 +204,18 @@ jobs: before_install: - sudo bash ops/travis/unit-testing/apihub before_install install: - - bash ops/travis/unit-testing/apihub install + - unbuffer bash ops/travis/unit-testing/apihub install before_script: - - bash ops/travis/unit-testing/apihub before_script + - unbuffer bash ops/travis/unit-testing/apihub before_script script: - - bash ops/travis/unit-testing/apihub script + - unbuffer bash ops/travis/unit-testing/apihub script after_success: - - bash ops/travis/unit-testing/apihub after_success + - unbuffer bash ops/travis/unit-testing/apihub after_success + after_failure: + - unbuffer bash ops/travis/unit-testing/apihub after_failure # test python, linting ---------------------------------------------------------------------- - - stage: unit testing + - stage: build / unit-testing name: python linting language: python python: @@ -174,74 +224,59 @@ jobs: before_install: - sudo bash ops/travis/unit-testing/python-linting before_install install: - - bash ops/travis/unit-testing/python-linting install + - unbuffer bash ops/travis/unit-testing/python-linting install before_script: - - bash ops/travis/unit-testing/python-linting before_script + - unbuffer bash ops/travis/unit-testing/python-linting before_script script: - - bash ops/travis/unit-testing/python-linting script + - unbuffer bash ops/travis/unit-testing/python-linting script after_success: - - bash ops/travis/unit-testing/python-linting after_success + - unbuffer bash ops/travis/unit-testing/python-linting after_success + after_failure: + - unbuffer bash ops/travis/unit-testing/python-linting after_failure - # frontend testing ---------------------------------------------------------------------- - - stage: unit testing - name: front end testing - language: node_js - node_js: - - "10" - addons: - apt: - packages: - - libgconf-2-4 -# cache: -# directories: -# - .npm -# - services/web/client/node_modules + # integrate webserver in swarm ------------------------------------------------------------- + - stage: integration-testing + name: webserver in swarm + language: python + python: + - "3.6" + sudo: required + cache: pip before_install: - - bash ops/travis/unit-testing/frontend before_install + - sudo bash ops/travis/integration-testing/webserver before_install install: - - bash ops/travis/unit-testing/frontend install + - unbuffer bash ops/travis/integration-testing/webserver install before_script: - - bash ops/travis/unit-testing/frontend before_script + - unbuffer bash ops/travis/integration-testing/webserver before_script script: - - bash ops/travis/unit-testing/frontend script + - unbuffer bash ops/travis/integration-testing/webserver script after_success: - - bash ops/travis/unit-testing/frontend after_success - - deploy: - # https://docs.travis-ci.com/user/deployment/pages/ - provider: pages - target-branch: master - # Set in travis-ci.org dashboard, marked secure - github-token: $GH_PAGES_TOKEN - skip-cleanup: true - keep-history: false - local-dir: itisfoundation.github.io - repo: ITISFoundation/itisfoundation.github.io - verbose: true - on: - branch: master - + - unbuffer bash ops/travis/integration-testing/webserver after_success + after_failure: + - unbuffer bash ops/travis/integration-testing/webserver after_failure - # integrate webserver in swarm ------------------------------------------------------------- - - stage: integration testing - name: webserver in swarm + # integrate simcore-sdk in swarm ------------------------------------------------------------- + - stage: integration-testing + name: simcore-sdk in swarm language: python python: - "3.6" sudo: required cache: pip before_install: - - sudo bash ops/travis/integration-testing/webserver before_install + - sudo bash ops/travis/integration-testing/simcore-sdk before_install install: - - bash ops/travis/integration-testing/webserver install + - unbuffer bash ops/travis/integration-testing/simcore-sdk install before_script: - - bash ops/travis/integration-testing/webserver before_script + - unbuffer bash ops/travis/integration-testing/simcore-sdk before_script script: - - bash ops/travis/integration-testing/webserver script + - unbuffer bash ops/travis/integration-testing/simcore-sdk script after_success: - - bash ops/travis/integration-testing/webserver after_success + - unbuffer bash ops/travis/integration-testing/simcore-sdk after_success + after_failure: + - unbuffer bash ops/travis/integration-testing/simcore-sdk after_failure - - stage: system testing, staging + - stage: system-testing sudo: required language: python python: @@ -250,28 +285,50 @@ jobs: before_install: - sudo bash ops/travis/system-testing/build_and_run before_install install: - - bash ops/travis/system-testing/build_and_run install + - unbuffer bash ops/travis/system-testing/build_and_run install before_script: - - bash ops/travis/system-testing/build_and_run before_script + - unbuffer bash ops/travis/system-testing/build_and_run before_script script: - - bash ops/travis/system-testing/build_and_run script + - unbuffer bash ops/travis/system-testing/build_and_run script after_success: - - bash ops/travis/system-testing/build_and_run after_success + - unbuffer bash ops/travis/system-testing/build_and_run after_success after_failure: - - bash ops/travis/system-testing/build_and_run after_failure + - unbuffer bash ops/travis/system-testing/build_and_run after_failure + + - stage: deployment + name: master + if: branch = master + git: + depth: false + script: echo "Deploy master" deploy: - provider: script - skip_cleanup: true - script: bash ops/travis/staging/deploy + script: unbuffer bash ops/travis/deploy/master on: branch: master + - stage: deployment + name: staging + if: branch = staging + git: + depth: false + script: echo "Deploy staging" + deploy: - provider: script - skip_cleanup: true - script: ops/travis/production/deploy + script: unbuffer bash ops/travis/deploy/staging on: - branch: master + branch: staging + - stage: deployment + name: production/release + git: + depth: false + script: echo "Deploy production version $TRAVIS_TAG" + deploy: + - provider: script + script: unbuffer bash ops/travis/deploy/production + on: + all_branches: true tags: true - condition: "$TRAVIS_TAG =~ -release$" + condition: $TRAVIS_TAG =~ ^v[0-9]+.[0-9]+.[0-9]+$ notifications: email: on_success: never diff --git a/Makefile b/Makefile index d8140afd19f..54e4a841e61 100644 --- a/Makefile +++ b/Makefile @@ -27,8 +27,6 @@ export DOCKER_COMPOSE=docker-compose export DOCKER=docker endif -export DOCKER_REGISTRY=masu.speag.com -export SERVICES_VERSION=2.8.0 PY_FILES := $(strip $(shell find services packages -iname '*.py' \ -not -path "*egg*" \ @@ -44,20 +42,19 @@ CACHED_SERVICES_LIST := ${SERVICES_LIST} webclient DYNAMIC_SERVICE_FOLDERS_LIST := services/dy-jupyter services/dy-2Dgraph/use-cases services/dy-3dvis services/dy-modeling CLIENT_WEB_OUTPUT:=$(CURDIR)/services/web/client/source-output +export VCS_URL:=$(shell git config --get remote.origin.url) +export VCS_REF:=$(shell git rev-parse --short HEAD) +export VCS_REF_CLIENT:=$(shell git log --pretty=tformat:"%h" -n1 services/web/client) +export VCS_STATUS_CLIENT:=$(if $(shell git status -s),'modified/untracked','clean') +export BUILD_DATE:=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") -VCS_URL:=$(shell git config --get remote.origin.url) -VCS_REF:=$(shell git rev-parse --short HEAD) -VCS_REF_CLIENT:=$(shell git log --pretty=tformat:"%h" -n1 services/web/client) -VCS_STATUS_CLIENT:=$(if $(shell git status -s),'modified/untracked','clean') -export VCS_URL -export VCS_REF -export VCS_REF_CLIENT -export VCS_STATUS_CLIENT - -BUILD_DATE:=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") -export BUILD_DATE - +# using ?= will only set if absent +export DOCKER_IMAGE_TAG ?= latest +$(info DOCKER_IMAGE_TAG set to ${DOCKER_IMAGE_TAG}) +# default to local (no registry) +export DOCKER_REGISTRY ?= itisfoundation +$(info DOCKER_REGISTRY set to ${DOCKER_REGISTRY}) ## Tools ------------------------------------------------------------------------------------------------------ # tools = @@ -83,24 +80,16 @@ endif ## ------------------------------- # Docker build and composition - -.PHONY: build rebuild -# target: build, rebuild: – Builds all core service images. Use `rebuild` to build w/o cache. -build: .env pull-cache - ${DOCKER_COMPOSE} -f services/docker-compose.yml build - -rebuild: - ${DOCKER_COMPOSE} -f services/docker-compose.yml build --no-cache - - -.PHONY: build-devel rebuild-devel .tmp-webclient-build -# target: build-devel, rebuild-devel: – Builds images of core services for development. Use `rebuild` to build w/o cache. -build-devel: .env pull-cache .tmp-webclient-build +.PHONY: build +# target: build: – Builds all core service images. +build: .env .tmp-webclient-build + ${DOCKER_COMPOSE} -f services/docker-compose.yml build --parallel ${SERVICES_LIST}; + +.PHONY: build-devel .tmp-webclient-build +# target: build-devel, rebuild-devel: – Builds images of core services for development. +build-devel: .env .tmp-webclient-build ${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.devel.yml build --parallel -rebuild-devel: .env .tmp-webclient-build - ${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.devel.yml build --no-cache --parallel - # TODO: fixes having services_webclient:build present for services_webserver:production when # targeting services_webserver:development and .tmp-webclient-build: $(CLIENT_WEB_OUTPUT) @@ -117,7 +106,7 @@ endif .PHONY: build-client rebuild-client # target: build-client, rebuild-client: – Builds only webclient and webserver images. Use `rebuild` to build w/o cache -build-client: .env pull-cache +build-client: .env ${DOCKER_COMPOSE} -f services/docker-compose.yml build webclient ${DOCKER_COMPOSE} -f services/docker-compose.yml build webserver @@ -202,13 +191,13 @@ endif .PHONY: pull-cache pull-cache: - ${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.cache.yml pull --ignore-pull-failures + ${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.cache.yml pull .PHONY: build-cache # target: build-cache – Builds service images and tags them as 'cache' -build-cache: pull-cache +build-cache: ${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.cache.yml build --parallel apihub director sidecar storage webclient - ${DOCKER} tag itisfoundation/webclient:cache services_webclient:build + ${DOCKER} tag ${DOCKER_REGISTRY}/webclient:cache services_webclient:build ${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.cache.yml build webserver @@ -220,33 +209,37 @@ push-cache: ## ------------------------------- -# Staging -# TODO: PC->SAN: see ops/travis/system-testing/build_and_run. Could move images FREFIX and TAG there +# registry operations +ifdef DOCKER_REGISTRY_NEW +$(info DOCKER_REGISTRY_NEW set to ${DOCKER_REGISTRY_NEW}) +endif # DOCKER_REGISTRY_NEW -.PHONY: tag push pull create-staging-stack-file +.PHONY: tag push pull create-stack-file #target: tag – Tags service images tag: - for i in $(SERVICES_LIST); do \ - ${DOCKER} tag services_$$i:latest ${DOCKER_IMAGE_PREFIX}$$i:${DOCKER_IMAGE_TAG}; \ +ifndef DOCKER_REGISTRY_NEW + $(error DOCKER_REGISTRY_NEW variable is undefined) +endif +ifndef DOCKER_IMAGE_TAG_NEW + $(error DOCKER_IMAGE_TAG_NEW variable is undefined) +endif + @echo "Tagging from ${DOCKER_REGISTRY}, ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY_NEW}, ${DOCKER_IMAGE_TAG_NEW}" + @for i in $(SERVICES_LIST); do \ + ${DOCKER} tag ${DOCKER_REGISTRY}/$$i:${DOCKER_IMAGE_TAG} ${DOCKER_REGISTRY_NEW}/$$i:${DOCKER_IMAGE_TAG_NEW}; \ done # target: push – Pushes images into a registry push: - for i in $(SERVICES_LIST); do \ - ${DOCKER} push ${DOCKER_IMAGE_PREFIX}$$i:${DOCKER_IMAGE_TAG}; \ - done + ${DOCKER_COMPOSE} -f services/docker-compose.yml push ${SERVICES_LIST} # target: pull – Pulls images from a registry pull: - ${DOCKER_COMPOSE} -f services/docker-compose.yml pull + ${DOCKER_COMPOSE} -f services/docker-compose.yml pull ${SERVICES_LIST} -# target: create-staging-stack-file – use as 'make creat-staging-stack-file output_file=stack.yaml' -create-staging-stack-file: - export DOCKER_IMAGE_PREFIX=itisfoundation/; \ - export DOCKER_IMAGE_TAG=staging-latest; \ +# target: create-stack-file – use as 'make create-stack-file output_file=stack.yaml' +create-stack-file: ${DOCKER_COMPOSE} -f services/docker-compose.yml config > $(output_file) - ## ------------------------------- # Tools @@ -261,6 +254,7 @@ info: @echo '+ VERSION : ${VERSION}' @echo '+ WINDOWS_MODE : ${WINDOWS_MODE}' @echo '+ DOCKER_REGISTRY : ${DOCKER_REGISTRY}' + @echo '+ DOCKER_IMAGE_TAG : ${DOCKER_IMAGE_TAG}' @echo '+ SERVICES_VERSION : ${SERVICES_VERSION}' @echo '+ PY_FILES : $(shell echo $(PY_FILES) | wc -w) files' diff --git a/README.md b/README.md index c3c67ad00ce..2c8be896d9c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # osparc-simcore platform +**WARNING** This application is **still under development**. + [`master`](https://github.com/itisfoundation/osparc-simcore/tree/master) [![Requires.io]](https://requires.io/github/ITISFoundation/osparc-simcore/requirements/?branch=master "State of third party python dependencies") @@ -51,4 +53,6 @@ make down ``` -**WARNING** This application is **still under development** and still not suitable for production purposes. +## Release workflow + +[Git release workflow](ops/README.md) diff --git a/docs/img/git-release-workflow.md b/docs/img/git-release-workflow.md new file mode 100644 index 00000000000..ca8ca2999f8 --- /dev/null +++ b/docs/img/git-release-workflow.md @@ -0,0 +1,69 @@ +# create SVG out of this schema + +[mermaid live editor](https://mermaidjs.github.io/mermaid-live-editor) + +```mermaid +sequenceDiagram +participant Feature +participant Master +participant Hotfix +participant Staging + +Master->>Feature: create feature1 branch +Note over Feature: develop feature1... +Feature-->>Master: Pull Request feature1 +Master->Master: CI: build docker images - tests -> Dockerhub +Master->Master: CD -> master.dev + +Master->>Feature: create feature2 branch +Note over Feature: develop feature2... +Feature-->>Master: Pull Request feature2 +Master->Master: CI: build docker images - tests -> Dockerhub +Master->Master: CD -> master.dev + +Master-->>Staging: Pull Request staging1 +Staging->Staging: CI: build docker images - tests -> Dockerhub +Staging->Staging: CD -> staging.io +Note over Staging: ready for release? + +Master->>Feature: create feature3 branch +Note over Feature: develop feature3... +Feature-->>Master: Pull Request feature3 +Master->Master: CI: build docker images - tests -> Dockerhub +Master->Master: CD -> master.dev + +Master->>Feature: create feature4 branch +Note over Feature: develop feature4... +Feature-->>Master: Pull Request feature4 +Master->Master: CI: build docker images - tests -> Dockerhub +Master->Master: CD -> master.dev + +Master-->>Staging: Pull Request staging2 +Staging->Staging: CI: build docker images - tests -> Dockerhub +Staging->Staging: CD -> staging.io +Note over Staging: ready for release? +Staging->Staging: RELEASE: Tag v1.0.0 - CD -> osparc.io + +Staging->>Hotfix: create hotfix1 branch +Note over Hotfix: fix issue... +Hotfix-->>Staging: Pull request hotfix1 +Staging->Staging: CI: build docker images - tests -> Dockerhub +Staging->Staging: CD -> staging.io +Note over Staging: ready for release? +Staging->Staging: RELEASE: Tag v1.0.1 - CD -> osparc.io +Hotfix-->>Master: Pull request hotfix1 +Master->Master: CI: build docker images - tests -> Dockerhub +Master->Master: CD -> master.dev + +Master->>Feature: create feature10 branch +Note over Feature: develop feature10... +Feature-->>Master: Pull Request feature10 +Master->Master: CI: build docker images - tests -> Dockerhub +Master->Master: CD -> master.dev + +Master->>Feature: create featureN branch +Note over Feature: develop featureN... +Feature-->>Master: Pull Request featureN +Master->Master: CI: build docker images - tests -> Dockerhub +Master->Master: CD -> master.dev +``` diff --git a/docs/img/git-release-workflow.svg b/docs/img/git-release-workflow.svg new file mode 100644 index 00000000000..8958e1e6f4d --- /dev/null +++ b/docs/img/git-release-workflow.svg @@ -0,0 +1,360 @@ +FeatureMasterHotfixStagingcreate feature1 branchdevelop feature1...Pull Request feature1CI: build docker images - tests -> DockerhubCD -> master.devcreate feature2 branchdevelop feature2...Pull Request feature2CI: build docker images - tests -> DockerhubCD -> master.devPull Request staging1CI: build docker images - tests -> DockerhubCD -> staging.ioready for release?create feature3 branchdevelop feature3...Pull Request feature3CI: build docker images - tests -> DockerhubCD -> master.devcreate feature4 branchdevelop feature4...Pull Request feature4CI: build docker images - tests -> DockerhubCD -> master.devPull Request staging2CI: build docker images - tests -> DockerhubCD -> staging.ioready for release?RELEASE: Tag v1.0.0 - CD -> osparc.iocreate hotfix1 branchfix issue...Pull request hotfix1CI: build docker images - tests -> DockerhubCD -> staging.ioready for release?RELEASE: Tag v1.0.1 - CD -> osparc.ioPull request hotfix1CI: build docker images - tests -> DockerhubCD -> master.devcreate feature10 branchdevelop feature10...Pull Request feature10CI: build docker images - tests -> DockerhubCD -> master.devcreate featureN branchdevelop featureN...Pull Request featureNCI: build docker images - tests -> DockerhubCD -> master.devFeatureMasterHotfixStaging \ No newline at end of file diff --git a/ops/README.md b/ops/README.md index 30b43523be9..3bb56a55a1f 100644 --- a/ops/README.md +++ b/ops/README.md @@ -1,3 +1,23 @@ # osparc-simcore -> ops -Operations / Deployment \ No newline at end of file +## Travis / Dockerhub setup + +The osparc-simcore repository provides a Travis-CI [recipe](.travis.yml) that fullfills the [Release Workflow](docs/img/git-release-workflow.svg). + +To this end Travis should be enabled for the repository in each fork and a [Dockerhub](https://hub.docker.com/) account is recommended to push the docker images generated by Travis. + +Configuring your travis settings will speed up the travis CI process by making use of Dockerhub (or another docker registry) to move docker images between stages. + +### Travis configuration + +Define the following secure environment variables in your fork: + +```bash +DOCKER_REGISTRY # this shall be set to your own dockerhub repository account for example: itisfoundation +DOCKER_USERNAME # the docker username (!beware this should be a [secure env variable](https://docs.travis-ci.com/user/environment-variables/#defining-encrypted-variables-in-travisyml)) +DOCKER_PASSWORD # the docker password (!beware this should be a [secure env variable](https://docs.travis-ci.com/user/environment-variables/#defining-encrypted-variables-in-travisyml)) +``` + +### Release workflow + +![Git release workflow](docs/img/git-release-workflow.svg) \ No newline at end of file diff --git a/ops/travis/build-cache/scripts b/ops/travis/build-cache/scripts deleted file mode 100644 index cbc82eb9a54..00000000000 --- a/ops/travis/build-cache/scripts +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail -IFS=$'\n\t' - -before_install() { - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then - bash ops/travis/helpers/install_docker_compose; - bash ops/travis/helpers/show_system_versions; - fi -} - -install() { - echo "nothing to install..." -} - -before_script() { - echo "nothing to do..." -} - -script() { - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then - make build-cache - fi -} - -after_success() { - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - make push-cache - fi -} - -# Check if the function exists (bash specific) -if declare -f "$1" > /dev/null -then - # call arguments verbatim - "$@" -else - # Show a helpful error - echo "'$1' is not a known function name" >&2 - exit 1 -fi diff --git a/ops/travis/build/test-images b/ops/travis/build/test-images new file mode 100755 index 00000000000..d6b76da56a5 --- /dev/null +++ b/ops/travis/build/test-images @@ -0,0 +1,47 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +export DOCKER_IMAGE_TAG=$(exec ops/travis/helpers/build_docker_image_tag) + +before_install() { + bash ops/travis/helpers/install_docker_compose; + bash ops/travis/helpers/show_system_versions; + env +} + +install() { + echo "nothing to install..." +} + +before_script() { + make pull-cache || true + make pull || true +} + +script() { + make build-cache + make build +} + +after_success() { + echo "build succeeded" +} + +after_failure() { + echo "build failed" + env + docker images +} + +# Check if the function exists (bash specific) +if declare -f "$1" > /dev/null +then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi diff --git a/ops/travis/deploy/master b/ops/travis/deploy/master new file mode 100755 index 00000000000..e266498d5a4 --- /dev/null +++ b/ops/travis/deploy/master @@ -0,0 +1,28 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# pull the current tested build +export DOCKER_IMAGE_TAG=$(exec ops/travis/helpers/build_docker_image_tag) +make pull + +# show current images on system +docker images + +# these variable must be available securely from travis +echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + +# re-tag build to master-latest +export DOCKER_REGISTRY_NEW=${DOCKER_REGISTRY} +export DOCKER_IMAGE_TAG_NEW=master-latest +make tag +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG_NEW} +make push + +# re-tag build to master-DATE.BUILD_NUMBER.GIT_SHA +TRAVIS_PLATFORM_VERSION=master-$(date +"%Y-%m-%d").${TRAVIS_BUILD_NUMBER}.$(git rev-parse HEAD) +export DOCKER_IMAGE_TAG_NEW=$TRAVIS_PLATFORM_VERSION +make tag +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG_NEW} +make push diff --git a/ops/travis/deploy/production b/ops/travis/deploy/production new file mode 100755 index 00000000000..e05851e24e4 --- /dev/null +++ b/ops/travis/deploy/production @@ -0,0 +1,35 @@ +#!/bin/bash +# as of http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# pull the tagged staging build +# find the docker image tag +export TAG="${TRAVIS_TAG}" +export ORG=${DOCKER_REGISTRY} +export REPO="webserver" +DOCKER_IMAGE_TAG=$(./ops/travis/helpers/find_staging_version | awk 'END{print}') || exit $? +export DOCKER_IMAGE_TAG +make pull + +# show current images on system +docker images + +# these variable must be available securely from travis +echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + +# re-tag master build to staging-latest +export DOCKER_REGISTRY_NEW=${DOCKER_REGISTRY} +export DOCKER_IMAGE_TAG_NEW=latest +make tag +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG_NEW} +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG_NEW} +make push + +# re-tag master to staging-DATE.BUILD_NUMBER.GIT_SHA +TRAVIS_PLATFORM_PROD_VERSION=${TRAVIS_TAG}-$(date +"%Y-%m-%d").${TRAVIS_BUILD_NUMBER}.$(git rev-parse HEAD) +export DOCKER_IMAGE_TAG_NEW=$TRAVIS_PLATFORM_PROD_VERSION +make tag +export DOCKER_REGISTRY=${DOCKER_REGISTRY_NEW} +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG_NEW} +make push diff --git a/ops/travis/deploy/staging b/ops/travis/deploy/staging new file mode 100755 index 00000000000..7529c9b284c --- /dev/null +++ b/ops/travis/deploy/staging @@ -0,0 +1,30 @@ +#!/bin/bash +# as of http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# pull the tagged master build +export DOCKER_IMAGE_TAG=$(exec ops/travis/helpers/build_docker_image_tag) +make pull + +# show current images on system +docker images + +# these variable must be available securely from travis +echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + +# re-tag master build to staging-latest +export DOCKER_REGISTRY_NEW=${DOCKER_REGISTRY} +export DOCKER_IMAGE_TAG_NEW=staging-latest +make tag +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG_NEW} +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG_NEW} +make push + +# re-tag master to staging-DATE.BUILD_NUMBER.GIT_SHA +TRAVIS_PLATFORM_VERSION=staging-$(date +"%Y-%m-%d").${TRAVIS_BUILD_NUMBER}.$(git rev-parse HEAD) +export DOCKER_IMAGE_TAG_NEW=$TRAVIS_PLATFORM_VERSION +make tag +export DOCKER_REGISTRY=${DOCKER_REGISTRY_NEW} +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG_NEW} +make push diff --git a/ops/travis/staging/deploy b/ops/travis/deploy/test-images similarity index 52% rename from ops/travis/staging/deploy rename to ops/travis/deploy/test-images index 02d9d696048..4b796f2b87d 100755 --- a/ops/travis/staging/deploy +++ b/ops/travis/deploy/test-images @@ -3,18 +3,15 @@ set -euo pipefail IFS=$'\n\t' +export DOCKER_IMAGE_TAG=$(exec ops/travis/helpers/build_docker_image_tag) + # show current images on system docker images # these variable must be available securely from travis echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin -TRAVIS_PLATFORM_STAGE_VERSION=staging-$(date +"%Y-%m-%d").${TRAVIS_BUILD_NUMBER}.$(git rev-parse HEAD) -export DOCKER_IMAGE_PREFIX=itisfoundation/ -export DOCKER_IMAGE_TAG=staging-latest -make tag -make push - -export DOCKER_IMAGE_TAG=$TRAVIS_PLATFORM_STAGE_VERSION -make tag +# push the local cache +make push-cache +# push the local images make push diff --git a/ops/travis/helpers/build_docker_image_tag b/ops/travis/helpers/build_docker_image_tag new file mode 100755 index 00000000000..108f8d7878d --- /dev/null +++ b/ops/travis/helpers/build_docker_image_tag @@ -0,0 +1,38 @@ +#!/bin/bash +# Usage: build_docker_image_tag +# returns the slugified name of the image tag that shall be used +# e.g.: current git branch name +# if on travis, +# if on a branch: returns the name of the travis branch +# if on a pull request: returns the name of the originating branch +# if on master: returns "build" + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +if [ ! -v TRAVIS ] || [ $TRAVIS = "false" ]; then + # no travis here so let's use the git name directly + image_tag=$(git rev-parse --abbrev-ref HEAD) +else + if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then + if [ "${TRAVIS_BRANCH}" = "master" ]; then + # this is master branch and here we use a specific name to prevent naming colision + # with the master-latest build + image_tag="${TRAVIS_BRANCH}-testbuild" + elif [ "${TRAVIS_BRANCH}" = "staging" ]; then + # this is staging branch and here we use a specific name to prevent naming colision + # with the staging-latest build + image_tag="${TRAVIS_BRANCH}-testbuild" + else + image_tag=${TRAVIS_BRANCH} + fi + else + # this is a pull request, let's use the name of the originating branch instead of a boring master + image_tag=${TRAVIS_PULL_REQUEST_BRANCH} + fi +fi + +slugified_name=$(exec ops/travis/helpers/slugify_name $image_tag) + +echo "$slugified_name-latest" diff --git a/ops/travis/helpers/ensure_python_pip b/ops/travis/helpers/ensure_python_pip old mode 100644 new mode 100755 diff --git a/ops/travis/helpers/find_staging_version b/ops/travis/helpers/find_staging_version new file mode 100755 index 00000000000..936ef6ec0b8 --- /dev/null +++ b/ops/travis/helpers/find_staging_version @@ -0,0 +1,31 @@ +#!/bin/bash +# Usage: find_staging_version +# +# returns the full image tag corresponding to the git tag name that shall be used + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +echo "Retrieving SHA for tag ${TAG}" +GIT_COMMIT_SHA=$(git show-ref -s ${TAG}) +echo "Found SHA for tag ${GIT_COMMIT_SHA}" + +# get token +echo "Retrieving token ..." +TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'${DOCKER_USERNAME}'", "password": "'${DOCKER_PASSWORD}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token) + +# output images & tags +echo +echo "Images and tags for organization: ${ORG}" +echo "in repo ${REPO}" +IMAGE_TAGS=$(curl -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/${ORG}/${REPO}/tags/?page_size=100 | jq -r '.results|.[]|.name') +for j in ${IMAGE_TAGS} +do +if [[ ${j} =~ ${GIT_COMMIT_SHA} ]]; then +echo "${j}" +exit 0 +fi +done +# not found +exit 1 diff --git a/ops/travis/helpers/install_docker_compose b/ops/travis/helpers/install_docker_compose old mode 100644 new mode 100755 diff --git a/ops/travis/helpers/show_system_versions b/ops/travis/helpers/show_system_versions old mode 100644 new mode 100755 diff --git a/ops/travis/helpers/slugify_name b/ops/travis/helpers/slugify_name new file mode 100755 index 00000000000..b0b6c454e2b --- /dev/null +++ b/ops/travis/helpers/slugify_name @@ -0,0 +1,10 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +slugify () { + echo "$1" | iconv -t ascii//TRANSLIT | sed -r s/[~\^]+//g | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr A-Z a-z +} + +echo $(slugify $1) diff --git a/ops/travis/helpers/stop_postgres_service b/ops/travis/helpers/stop_postgres_service deleted file mode 100644 index bc0d2169a7a..00000000000 --- a/ops/travis/helpers/stop_postgres_service +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail -IFS=$'\n\t' - -# shutdown postgres because sometimes is is already up ??? -sudo service postgresql stop - -# wait for postgresql to shutdown -while nc -z localhost 5432; do - sleep 1 -done diff --git a/ops/travis/helpers/test_for_changes b/ops/travis/helpers/test_for_changes old mode 100644 new mode 100755 index a2d53420cb6..e8d2a4d9986 --- a/ops/travis/helpers/test_for_changes +++ b/ops/travis/helpers/test_for_changes @@ -9,11 +9,12 @@ IFS=$'\n\t' # or on the push commit range if running on travis # detect travis -if [ ! -v TRAVIS ] || [ $TRAVIS = "false" ]; then +if [ ! -v TRAVIS ] || [ $TRAVIS = "false" ]; then CHANGES=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD master)) else - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ]; then - echo "master branch detected, it's the same as if changes were done" + if [ "$TRAVIS_PULL_REQUEST" = "false" ] && \ + ([ "$TRAVIS_BRANCH" = "master" ] || [ "$TRAVIS_BRANCH" = "staging" ]); then + echo "$TRAVIS_BRANCH branch detected, it's the same as if changes were done" exit 0 fi CHANGES=$(git --no-pager diff --name-only $TRAVIS_COMMIT_RANGE) diff --git a/ops/travis/unit-testing/simcore-sdk b/ops/travis/integration-testing/simcore-sdk similarity index 59% rename from ops/travis/unit-testing/simcore-sdk rename to ops/travis/integration-testing/simcore-sdk index d5670a07a08..152862da391 100755 --- a/ops/travis/unit-testing/simcore-sdk +++ b/ops/travis/integration-testing/simcore-sdk @@ -1,9 +1,11 @@ #!/bin/bash -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ set -euo pipefail IFS=$'\n\t' -FOLDER_CHECKS=(packages/ simcore-sdk) +# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching +export DOCKER_IMAGE_TAG=$(exec ops/travis/helpers/build_docker_image_tag) + +FOLDER_CHECKS=(packages/ simcore-sdk storage/ simcore-sdk .travis.yml) before_install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; @@ -17,17 +19,18 @@ install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; then bash ops/travis/helpers/ensure_python_pip - pushd packages/simcore-sdk; pip3 install -r requirements/ci.txt; popd + pushd packages/simcore-sdk; pip3 install -r requirements/ci.txt; popd; + pip3 install services/storage/client-sdk/python fi } before_script() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; then - pip list -v - # TODO: create integration tests for simcore-sdk - # rename packages/simcore-sdk/tests/node_ports -> packages/simcore-sdk/tests/integration/node_ports - make build + pip freeze + # pull the test images if registry is set up, else build the images + make pull-cache || true + make pull || make build docker images fi } @@ -35,11 +38,9 @@ before_script() { script() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; then - pytest --cov=s3wrapper --cov-append -v packages/s3wrapper/tests - # TODO: split unit/integration tests in simcore-sdk!!! - pytest --cov=simcore_sdk --cov-append -v packages/simcore-sdk/tests + pytest --color=yes --cov=simcore_sdk --cov-append -v packages/simcore-sdk/tests else - echo "No changes detected. Skipping unit-testing of simcore-sdk." + echo "No changes detected. Skipping integration-testing of simcore-sdk." fi } @@ -51,6 +52,11 @@ after_success() { fi } +after_failure() { + docker images + make down +} + # Check if the function exists (bash specific) if declare -f "$1" > /dev/null then diff --git a/ops/travis/integration-testing/webserver b/ops/travis/integration-testing/webserver old mode 100644 new mode 100755 index 203669140ab..edb7afb2137 --- a/ops/travis/integration-testing/webserver +++ b/ops/travis/integration-testing/webserver @@ -3,6 +3,9 @@ set -euo pipefail IFS=$'\n\t' +# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching +export DOCKER_IMAGE_TAG=$(exec ops/travis/helpers/build_docker_image_tag) + before_install() { bash ops/travis/helpers/install_docker_compose bash ops/travis/helpers/show_system_versions @@ -15,18 +18,25 @@ install() { before_script() { pip list -v - make build + make pull-cache || true + make pull || make build docker images } script() { - pytest --cov=simcore_service_webserver -v services/web/server/tests/integration + pytest --cov=simcore_service_webserver --cov-append -v services/web/server/tests/integration # TODO: https://github.com/ITISFoundation/osparc-simcore/issues/560 #pytest --cov=simcore_service_webserver -v services/web/server/tests/integration-proxy } after_success() { coveralls + codecov +} + +after_failure() { + docker images + make down } # Check if the function exists (bash specific) diff --git a/ops/travis/production/deploy b/ops/travis/production/deploy deleted file mode 100644 index 0190646ad14..00000000000 --- a/ops/travis/production/deploy +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail -IFS=$'\n\t' - -echo "currently nothing goes on here...." diff --git a/ops/travis/system-testing/build_and_run b/ops/travis/system-testing/build_and_run index 8fe8345421f..e9ef5a9977b 100755 --- a/ops/travis/system-testing/build_and_run +++ b/ops/travis/system-testing/build_and_run @@ -3,6 +3,9 @@ set -euo pipefail IFS=$'\n\t' +# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching +export DOCKER_IMAGE_TAG=$(exec ops/travis/helpers/build_docker_image_tag) + before_install() { bash ops/travis/helpers/install_docker_compose bash ops/travis/helpers/show_system_versions @@ -11,8 +14,8 @@ before_install() { install() { bash ops/travis/helpers/ensure_python_pip pip3 install -r ops/travis/system-testing/requirements.txt - cp .env-devel .env - make build + make pull-cache || true + make pull || make build } before_script() { @@ -31,6 +34,7 @@ after_success() { } after_failure() { + docker images make down } diff --git a/ops/travis/system-testing/requirements.txt b/ops/travis/system-testing/requirements.txt old mode 100644 new mode 100755 diff --git a/ops/travis/unit-testing/README.md b/ops/travis/unit-testing/README.md index ae716e940a6..741e3406abc 100644 --- a/ops/travis/unit-testing/README.md +++ b/ops/travis/unit-testing/README.md @@ -22,13 +22,10 @@ pytest -v test/unit ![tests table](https://cdn-images-1.medium.com/max/800/1*XPhOBPDzVgSn-n8DhW7D8A.png) - ## Limitations [by May 2019] - - not time-bounded (e.g. can use time limits in the table above) - - poor coverage [![Coverage Status](https://coveralls.io/repos/github/ITISFoundation/osparc-simcore/badge.svg?branch=master)](https://coveralls.io/github/ITISFoundation/osparc-simcore?branch=master) - - some unittest (e.g. ``simcore-sdk``) build all services (contains a test that uses storage!) - +- not time-bounded (e.g. can use time limits in the table above) +- poor coverage [![Coverage Status](https://coveralls.io/repos/github/ITISFoundation/osparc-simcore/badge.svg?branch=master)](https://coveralls.io/github/ITISFoundation/osparc-simcore?branch=master) ## References @@ -37,7 +34,6 @@ pytest -v test/unit - [Python unit testing with Pytest and Mock](https://medium.com/@bfortuner/python-unit-testing-with-pytest-and-mock-197499c4623c) by B. Fortuner - [Test Sizes] by S.Stewards - [unittest.mock]:https://docs.python.org/3/library/unittest.mock.html#module-unittest.mock [pytest-mock]:https://github.com/pytest-dev/pytest-mock [pytest-docker]:https://github.com/AndreLouisCaron/pytest-docker diff --git a/ops/travis/unit-testing/apihub b/ops/travis/unit-testing/apihub index 65af11867ca..c23612ecf07 100755 --- a/ops/travis/unit-testing/apihub +++ b/ops/travis/unit-testing/apihub @@ -3,7 +3,7 @@ set -euo pipefail IFS=$'\n\t' -FOLDER_CHECKS=(api/ apihub) +FOLDER_CHECKS=(api/ apihub .travis.yml) before_install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; @@ -47,6 +47,11 @@ after_success() { fi } +after_failure() { + echo "failure... you can always write something more interesting here..." +} + + # Check if the function exists (bash specific) if declare -f "$1" > /dev/null then diff --git a/ops/travis/unit-testing/director b/ops/travis/unit-testing/director index 875f229ee2e..50aa6929c94 100755 --- a/ops/travis/unit-testing/director +++ b/ops/travis/unit-testing/director @@ -3,7 +3,7 @@ set -euo pipefail IFS=$'\n\t' -FOLDER_CHECKS=(api/ director packages/) +FOLDER_CHECKS=(api/ director packages/ .travis.yml) before_install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; @@ -45,6 +45,10 @@ after_success() { fi } +after_failure() { + echo "failure... you can always write something more interesting here..." +} + # Check if the function exists (bash specific) if declare -f "$1" > /dev/null then diff --git a/ops/travis/unit-testing/frontend b/ops/travis/unit-testing/frontend index 16e7280e468..704772c93b7 100755 --- a/ops/travis/unit-testing/frontend +++ b/ops/travis/unit-testing/frontend @@ -3,7 +3,7 @@ set -euo pipefail IFS=$'\n\t' -FOLDER_CHECKS=(js eslintrc json) +FOLDER_CHECKS=(js eslintrc json .travis.yml) before_install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; @@ -73,6 +73,10 @@ after_success() { fi } +after_failure() { + echo "failure... you can always write something more interesting here..." +} + # Check if the function exists (bash specific) if declare -f "$1" > /dev/null then diff --git a/ops/travis/unit-testing/python-linting b/ops/travis/unit-testing/python-linting index 6a4c1109aa8..5635634c285 100755 --- a/ops/travis/unit-testing/python-linting +++ b/ops/travis/unit-testing/python-linting @@ -3,7 +3,7 @@ set -euo pipefail IFS=$'\n\t' -FOLDER_CHECKS=(.py .pylintrc) +FOLDER_CHECKS=(.py .pylintrc .travis.yml) before_install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; @@ -46,6 +46,10 @@ after_success() { fi } +after_failure() { + echo "failure... you can always write something more interesting here..." +} + # Check if the function exists (bash specific) if declare -f "$1" > /dev/null then diff --git a/ops/travis/unit-testing/service-library b/ops/travis/unit-testing/service-library index a81dfbf3806..4ad9376ccff 100755 --- a/ops/travis/unit-testing/service-library +++ b/ops/travis/unit-testing/service-library @@ -1,10 +1,9 @@ #!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' - -# die on non 0 exit code -set -e - -FOLDER_CHECKS=(packages/ service-library) +FOLDER_CHECKS=(packages/ service-library .travis.yml) before_install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; @@ -46,6 +45,10 @@ after_success() { fi } +after_failure() { + echo "failure..." +} + # Check if the function exists (bash specific) if declare -f "$1" > /dev/null then diff --git a/ops/travis/unit-testing/sidecar b/ops/travis/unit-testing/sidecar index 3f9ee8aa3b4..31dc5f712a5 100755 --- a/ops/travis/unit-testing/sidecar +++ b/ops/travis/unit-testing/sidecar @@ -3,7 +3,7 @@ set -euo pipefail IFS=$'\n\t' -FOLDER_CHECKS=(api/ sidecar packages/) +FOLDER_CHECKS=(api/ sidecar packages/ .travis.yml) before_install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; diff --git a/ops/travis/unit-testing/storage b/ops/travis/unit-testing/storage index ba75146ff24..d897a5639d1 100755 --- a/ops/travis/unit-testing/storage +++ b/ops/travis/unit-testing/storage @@ -3,7 +3,7 @@ set -euo pipefail IFS=$'\n\t' -FOLDER_CHECKS=(api/ storage packages/) +FOLDER_CHECKS=(api/ storage packages/ .travis.yml) before_install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; @@ -45,6 +45,10 @@ after_success() { fi } +after_failure() { + echo "failure... you can always write something more interesting here..." +} + # Check if the function exists (bash specific) if declare -f "$1" > /dev/null then diff --git a/ops/travis/unit-testing/webserver b/ops/travis/unit-testing/webserver index cca023ffa68..aab8cfa91c7 100755 --- a/ops/travis/unit-testing/webserver +++ b/ops/travis/unit-testing/webserver @@ -3,7 +3,7 @@ set -euo pipefail IFS=$'\n\t' -FOLDER_CHECKS=(api/ webserver packages/ services/web) +FOLDER_CHECKS=(api/ webserver packages/ services/web .travis.yml) before_install() { if bash ops/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}"; @@ -45,6 +45,10 @@ after_success() { fi } +after_failure() { + echo "failure... you can always write something more interesting here..." +} + # Check if the function exists (bash specific) if declare -f "$1" > /dev/null then diff --git a/packages/simcore-sdk/tests/node_ports/conftest.py b/packages/simcore-sdk/tests/node_ports/conftest.py index d678c03023b..e4adb3c8416 100644 --- a/packages/simcore-sdk/tests/node_ports/conftest.py +++ b/packages/simcore-sdk/tests/node_ports/conftest.py @@ -6,13 +6,13 @@ import pytest import yarl + from helpers import helpers from simcore_sdk.models.pipeline_models import (Base, ComputationalPipeline, ComputationalTask) from simcore_sdk.node_ports import node_config - @pytest.fixture def user_id()->int: # see fixtures/postgres.py diff --git a/packages/simcore-sdk/tests/node_ports/docker-compose.yml b/packages/simcore-sdk/tests/node_ports/docker-compose.yml index 787bb2d2d98..e9cf70e25f6 100644 --- a/packages/simcore-sdk/tests/node_ports/docker-compose.yml +++ b/packages/simcore-sdk/tests/node_ports/docker-compose.yml @@ -1,7 +1,12 @@ version: '3.6' services: + #TODO: use same integration test framework as in web server instead of fixed docker-compose file + apihub: + image: ${DOCKER_REGISTRY:-itisfoundation}/apihub:${DOCKER_IMAGE_TAG:-latest} + ports: + - '8043:8043' storage: - image: services_storage:latest + image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} ports: - 11111:8080 environment: @@ -34,10 +39,6 @@ services: - POSTGRES_DB=test ports: - "5432:5432" - apihub: - image: services_apihub:latest - ports: - - '8043:8043' # TOOLS ----------------------------------------------------------- adminer: image: adminer diff --git a/packages/simcore-sdk/tests/node_ports/test_data_item.py b/packages/simcore-sdk/tests/node_ports/test_data_item.py index 5a75add99f5..67ef4d29c0a 100644 --- a/packages/simcore-sdk/tests/node_ports/test_data_item.py +++ b/packages/simcore-sdk/tests/node_ports/test_data_item.py @@ -15,13 +15,13 @@ ({"store":"z43", "path":"/simcore/asdlkjf/dsfskr.tmt"}), (None) ]) -def test_default_item(item_value): - with pytest.raises(exceptions.InvalidProtocolError, message="Expecting InvalidProtocolError"): +def test_default_item(item_value): + with pytest.raises(exceptions.InvalidProtocolError): DataItem() - with pytest.raises(exceptions.InvalidProtocolError, message="Expecting InvalidProtocolError"): - DataItem(key="a key") - with pytest.raises(exceptions.InvalidProtocolError, message="Expecting InvalidProtocolError"): - DataItem(value="a value") + with pytest.raises(exceptions.InvalidProtocolError): + DataItem(key="a key") + with pytest.raises(exceptions.InvalidProtocolError): + DataItem(value="a value") item = DataItem(key="one key", value=item_value) assert item.key == "one key" assert item.value == item_value diff --git a/packages/simcore-sdk/tests/node_ports/test_item.py b/packages/simcore-sdk/tests/node_ports/test_item.py index 9fed02319b7..408a9f2ec89 100644 --- a/packages/simcore-sdk/tests/node_ports/test_item.py +++ b/packages/simcore-sdk/tests/node_ports/test_item.py @@ -20,7 +20,7 @@ def create_item(item_type, item_value): value=item_value)) def test_default_item(): - with pytest.raises(exceptions.InvalidProtocolError, message="Expecting InvalidProtocolError"): + with pytest.raises(exceptions.InvalidProtocolError): Item(None, None) @@ -54,14 +54,14 @@ async def test_valid_type(): async def test_invalid_type(): item = create_item("some wrong type", None) - with pytest.raises(exceptions.InvalidProtocolError, message="Expecting InvalidProtocolError") as excinfo: + with pytest.raises(exceptions.InvalidProtocolError) as excinfo: await item.get() assert "Invalid protocol used" in str(excinfo.value) async def test_invalid_value_type(): #pylint: disable=W0612 - with pytest.raises(exceptions.InvalidItemTypeError, message="Expecting InvalidItemTypeError") as excinfo: + with pytest.raises(exceptions.InvalidItemTypeError) as excinfo: create_item("integer", "not an integer") @pytest.mark.parametrize("item_type, item_value_to_set, expected_value", [ @@ -91,6 +91,6 @@ async def test_set_new_value(bucket, item_type, item_value_to_set, expected_valu async def test_set_new_invalid_value(bucket, item_type, item_value_to_set): # pylint: disable=W0613 item = create_item(item_type, None) assert await item.get() is None - with pytest.raises(exceptions.InvalidItemTypeError, message="Expecting InvalidItemTypeError") as excinfo: + with pytest.raises(exceptions.InvalidItemTypeError) as excinfo: await item.set(item_value_to_set) assert "Invalid item type" in str(excinfo.value) diff --git a/packages/simcore-sdk/tests/node_ports/test_itemstlist.py b/packages/simcore-sdk/tests/node_ports/test_itemstlist.py index 80e0af63022..3de8fcd90b3 100644 --- a/packages/simcore-sdk/tests/node_ports/test_itemstlist.py +++ b/packages/simcore-sdk/tests/node_ports/test_itemstlist.py @@ -48,7 +48,7 @@ def test_accessing_by_key(): def test_access_by_wrong_key(): from simcore_sdk.node_ports import exceptions itemslist = create_items_list([("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)]) - with pytest.raises(exceptions.UnboundPortError, message="Expecting UnboundPortError"): + with pytest.raises(exceptions.UnboundPortError): print(itemslist["fdoiht"]) diff --git a/packages/simcore-sdk/tests/node_ports/test_nodeports.py b/packages/simcore-sdk/tests/node_ports/test_nodeports.py index eae23e0230a..4c9c5d64857 100644 --- a/packages/simcore-sdk/tests/node_ports/test_nodeports.py +++ b/packages/simcore-sdk/tests/node_ports/test_nodeports.py @@ -72,10 +72,10 @@ def test_invalid_ports(special_configuration): assert not PORTS.inputs assert not PORTS.outputs - with pytest.raises(exceptions.UnboundPortError, message="Expecting UnboundPortError"): + with pytest.raises(exceptions.UnboundPortError): PORTS.inputs[0] - with pytest.raises(exceptions.UnboundPortError, message="Expecting UnboundPortError"): + with pytest.raises(exceptions.UnboundPortError): PORTS.outputs[0] @@ -120,7 +120,7 @@ async def test_port_file_accessors(special_configuration, storage, filemanager_c PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) assert await PORTS.outputs["out_34"].get() is None # check emptyness - # with pytest.raises(exceptions.S3InvalidPathError, message="Expecting S3InvalidPathError"): + # with pytest.raises(exceptions.S3InvalidPathError): # await PORTS.inputs["in_1"].get() # this triggers an upload to S3 + configuration change @@ -249,7 +249,7 @@ async def test_file_mapping(special_configuration, project_id, node_uuid, filema assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_1", item_alias) invalid_alias = Path("invalid_alias.fjfj") - with pytest.raises(exceptions.PortNotFound, message="Expecting PortNotFound"): + with pytest.raises(exceptions.PortNotFound): await PORTS.set_file_by_keymap(invalid_alias) await PORTS.set_file_by_keymap(file_path) diff --git a/packages/simcore-sdk/tests/node_ports/test_schema_item.py b/packages/simcore-sdk/tests/node_ports/test_schema_item.py index b5b4ffd5778..3c0f95b8dcf 100644 --- a/packages/simcore-sdk/tests/node_ports/test_schema_item.py +++ b/packages/simcore-sdk/tests/node_ports/test_schema_item.py @@ -4,7 +4,7 @@ from simcore_sdk.node_ports._schema_item import SchemaItem def test_default_item(): - with pytest.raises(exceptions.InvalidProtocolError, message="Expecting InvalidProtocolError"): + with pytest.raises(exceptions.InvalidProtocolError): item = SchemaItem() #pylint: disable=W0612 def test_check_item_required_fields(): #pylint: disable=W0612 @@ -15,7 +15,7 @@ def test_check_item_required_fields(): #pylint: disable=W0612 for key in required_parameters: parameters = deepcopy(required_parameters) parameters.pop(key) - with pytest.raises(exceptions.InvalidProtocolError, message="Expecting InvalidProtocolError"): + with pytest.raises(exceptions.InvalidProtocolError): SchemaItem(**parameters) def test_item_construction_default(): @@ -38,4 +38,4 @@ def test_item_construction_with_optional_params(): assert item.displayOrder == 2 assert item.fileToKeyMap == {"file1.txt":"a key"} assert item.defaultValue == "some value" - assert item.widget == {} \ No newline at end of file + assert item.widget == {} diff --git a/services/director/tests/test_dummy_services.py b/services/director/tests/test_dummy_services.py index c09edcd27dd..1e574d8afad 100644 --- a/services/director/tests/test_dummy_services.py +++ b/services/director/tests/test_dummy_services.py @@ -19,7 +19,7 @@ def test_v0_services_nonconformity(configure_schemas_location, push_v0_schema_se for service in services: # validate service - with pytest.raises(Exception, message="expecting json schema validation error"): + with pytest.raises(Exception): json_schema_validator.validate_instance_object(service["service_description"], service_schema) def test_v1_services_conformity(configure_schemas_location, push_services): diff --git a/services/director/tests/test_handlers.py b/services/director/tests/test_handlers.py index d6e1bacecf9..4ff0a96bcc7 100644 --- a/services/director/tests/test_handlers.py +++ b/services/director/tests/test_handlers.py @@ -52,17 +52,17 @@ def _check_services(created_services, services, schema_version="v1"): async def test_services_get(docker_registry, configure_schemas_location, push_services): fake_request = "fake request" # no registry defined - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting HTTP Internal Error as no registry URL is defined"): + with pytest.raises(web_exceptions.HTTPInternalServerError): services_enveloped = await rest.handlers.services_get(fake_request) # wrong registry defined config.REGISTRY_URL = "blahblah" - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting HTTP Internal Error as SSL is enabled by default"): + with pytest.raises(web_exceptions.HTTPInternalServerError): services_enveloped = await rest.handlers.services_get(fake_request) # right registry defined config.REGISTRY_URL = docker_registry - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting HTTP Internal Error as SSL is enabled by default"): + with pytest.raises(web_exceptions.HTTPInternalServerError): services_enveloped = await rest.handlers.services_get(fake_request) # no SSL @@ -128,13 +128,13 @@ async def test_v0_services_conversion_to_new(configure_registry_access, configur async def test_services_by_key_version_get(configure_registry_access, configure_schemas_location, push_services): #pylint: disable=W0613, W0621 fake_request = "fake request" - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting internal server error"): + with pytest.raises(web_exceptions.HTTPInternalServerError): web_response = await rest.handlers.services_by_key_version_get(fake_request, None, None) - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting internal server error"): + with pytest.raises(web_exceptions.HTTPInternalServerError): web_response = await rest.handlers.services_by_key_version_get(fake_request, "whatever", None) - with pytest.raises(web_exceptions.HTTPNotFound, message="Expecting not found error"): + with pytest.raises(web_exceptions.HTTPNotFound): web_response = await rest.handlers.services_by_key_version_get(fake_request, "whatever", "ofwhateverversion") created_services = push_services(3,2) @@ -156,37 +156,37 @@ async def test_services_by_key_version_get(configure_registry_access, configure_ async def _start_get_stop_services(push_services, user_id, project_id): fake_request = "fake request" - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting internal server error"): + with pytest.raises(web_exceptions.HTTPInternalServerError): web_response = await rest.handlers.running_interactive_services_post(fake_request, None, None, None, None, None, None) - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting internal server error"): + with pytest.raises(web_exceptions.HTTPInternalServerError): web_response = await rest.handlers.running_interactive_services_post(fake_request, "None", None, None, None, None, None) - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting internal server error"): + with pytest.raises(web_exceptions.HTTPInternalServerError): web_response = await rest.handlers.running_interactive_services_post(fake_request, "None", "None", None, None, None, None) - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting internal server error"): + with pytest.raises(web_exceptions.HTTPInternalServerError): web_response = await rest.handlers.running_interactive_services_post(fake_request, "None", "None", "None", None, None, None) - with pytest.raises(web_exceptions.HTTPNotFound, message="Expecting not found error"): + with pytest.raises(web_exceptions.HTTPNotFound): web_response = await rest.handlers.running_interactive_services_post(fake_request, "None", "None", "None", "ablah", None, None) - with pytest.raises(web_exceptions.HTTPNotFound, message="Expecting not found error"): + with pytest.raises(web_exceptions.HTTPNotFound): web_response = await rest.handlers.running_interactive_services_post(fake_request, "None", "None", "None", "ablah", "None", None) - with pytest.raises(web_exceptions.HTTPNotFound, message="Expecting not found error"): + with pytest.raises(web_exceptions.HTTPNotFound): web_response = await rest.handlers.running_interactive_services_post(fake_request, "None", "None", "None", "ablah", "None", "None") - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting internal server error"): + with pytest.raises(web_exceptions.HTTPInternalServerError): web_response = await rest.handlers.running_interactive_services_get(fake_request, None) - with pytest.raises(web_exceptions.HTTPNotFound, message="Expecting not found error"): + with pytest.raises(web_exceptions.HTTPNotFound): web_response = await rest.handlers.running_interactive_services_get(fake_request, "service_uuid") - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting internal server error"): + with pytest.raises(web_exceptions.HTTPInternalServerError): web_response = await rest.handlers.running_interactive_services_delete(fake_request, None) - with pytest.raises(web_exceptions.HTTPNotFound, message="Expecting not found error"): + with pytest.raises(web_exceptions.HTTPNotFound): web_response = await rest.handlers.running_interactive_services_delete(fake_request, "service_uuid") created_services = push_services(0,2) assert len(created_services) == 2 @@ -240,7 +240,7 @@ async def _start_get_stop_services(push_services, user_id, project_id): async def test_running_services_post_and_delete_no_swarm(configure_registry_access, configure_schemas_location, push_services, user_id, project_id): #pylint: disable=W0613, W0621 - with pytest.raises(web_exceptions.HTTPInternalServerError, message="Expecting internal error as there is no docker swarm"): + with pytest.raises(web_exceptions.HTTPInternalServerError): await _start_get_stop_services(push_services, user_id, project_id) diff --git a/services/director/tests/test_json_schemas.py b/services/director/tests/test_json_schemas.py index 1d3403d1477..962f2c28268 100644 --- a/services/director/tests/test_json_schemas.py +++ b/services/director/tests/test_json_schemas.py @@ -3,8 +3,8 @@ import pytest from jsonschema import ( - SchemaError, - ValidationError, + SchemaError, + ValidationError, validate) from simcore_service_director import resources @@ -18,7 +18,7 @@ def validate_individual_schemas(list_of_paths): schema_specs = json.load(file_ptr) try: dummy_instance = {} - with pytest.raises(ValidationError, message="Expected a Json schema validation error"): + with pytest.raises(ValidationError): validate(dummy_instance, schema_specs) except SchemaError as err: pytest.fail(err.message) diff --git a/services/director/tests/test_producer.py b/services/director/tests/test_producer.py index 5427d7329ce..78b05e15754 100644 --- a/services/director/tests/test_producer.py +++ b/services/director/tests/test_producer.py @@ -26,7 +26,7 @@ async def push_start_services(number_comp, number_dyn): service_port = pushed_service["internal_port"] service_uuid = str(uuid.uuid1()) service_basepath = "/my/base/path" - with pytest.raises(exceptions.ServiceUUIDNotFoundError, message="expecting service uuid not found error"): + with pytest.raises(exceptions.ServiceUUIDNotFoundError): await producer.get_service_details(service_uuid) # start the service started_service = await producer.start_service(user_id, project_id, service_key, service_version, service_uuid, service_basepath) @@ -56,7 +56,7 @@ async def push_start_services(number_comp, number_dyn): for service in started_services: service_uuid = service["service_uuid"] await producer.stop_service(service_uuid) - with pytest.raises(exceptions.ServiceUUIDNotFoundError, message="expecting service uuid not found error"): + with pytest.raises(exceptions.ServiceUUIDNotFoundError): await producer.get_service_details(service_uuid) diff --git a/services/docker-compose.cache.yml b/services/docker-compose.cache.yml index eed6a4acd77..ccf268dc9e1 100644 --- a/services/docker-compose.cache.yml +++ b/services/docker-compose.cache.yml @@ -1,26 +1,26 @@ version: '3.4' services: apihub: - image: itisfoundation/apihub:cache + image: ${DOCKER_REGISTRY:-itisfoundation}/apihub:cache build: target: production director: - image: itisfoundation/director:cache + image: ${DOCKER_REGISTRY:-itisfoundation}/director:cache build: target: cache sidecar: - image: itisfoundation/sidecar:cache + image: ${DOCKER_REGISTRY:-itisfoundation}/sidecar:cache build: target: cache storage: - image: itisfoundation/storage:cache + image: ${DOCKER_REGISTRY:-itisfoundation}/storage:cache build: target: cache webclient: - image: itisfoundation/webclient:cache + image: ${DOCKER_REGISTRY:-itisfoundation}/webclient:cache build: target: build webserver: - image: itisfoundation/webserver:cache + image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:cache build: target: cache diff --git a/services/docker-compose.yml b/services/docker-compose.yml index ceed1a0d8d4..ebbf2413dd8 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1,14 +1,13 @@ version: '3.4' services: apihub: - image: ${DOCKER_IMAGE_PREFIX}apihub:${DOCKER_IMAGE_TAG} + image: ${DOCKER_REGISTRY:-itisfoundation}/apihub:${DOCKER_IMAGE_TAG:-latest} build: context: ../ dockerfile: services/apihub/Dockerfile cache_from: - - itisfoundation/apihub:cache - - itisfoundation/apihub:staging-latest - - services_apihub:latest + - ${DOCKER_REGISTRY:-itisfoundation}/apihub:cache + - ${DOCKER_REGISTRY:-itisfoundation}/apihub:${DOCKER_IMAGE_TAG:-latest} target: production labels: org.label-schema.schema-version: "1.0" @@ -23,14 +22,13 @@ services: - node.platform.os == linux #-------------------------------------------------------------------- director: - image: ${DOCKER_IMAGE_PREFIX}director:${DOCKER_IMAGE_TAG} + image: ${DOCKER_REGISTRY:-itisfoundation}/director:${DOCKER_IMAGE_TAG:-latest} build: context: ../ dockerfile: services/director/Dockerfile cache_from: - - itisfoundation/director:cache - - itisfoundation/director:staging-latest - - services_director:latest + - ${DOCKER_REGISTRY:-itisfoundation}/director:cache + - ${DOCKER_REGISTRY:-itisfoundation}/director:${DOCKER_IMAGE_TAG:-latest} target: production labels: org.label-schema.schema-version: "1.0" @@ -76,7 +74,7 @@ services: context: ./web/client/ dockerfile: Dockerfile cache_from: - - itisfoundation/webclient:cache + - ${DOCKER_REGISTRY:-itisfoundation}/webclient:cache - services_webclient:latest target: build labels: @@ -93,14 +91,13 @@ services: /bin/sh -c "ls -l build-output;cat build-output/version.txt" webserver: - image: ${DOCKER_IMAGE_PREFIX}webserver:${DOCKER_IMAGE_TAG} + image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} build: context: ../ dockerfile: services/web/Dockerfile cache_from: - - itisfoundation/webserver:cache - - itisfoundation/webserver:staging-latest - - services_webserver:latest + - ${DOCKER_REGISTRY:-itisfoundation}/webserver:cache + - ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} target: production labels: org.label-schema.schema-version: "1.0" @@ -147,16 +144,15 @@ services: - node.role == manager #-------------------------------------------------------------------- sidecar: - image: ${DOCKER_IMAGE_PREFIX}sidecar:${DOCKER_IMAGE_TAG} + image: ${DOCKER_REGISTRY:-itisfoundation}/sidecar:${DOCKER_IMAGE_TAG:-latest} build: # the context for the build is the git repo root directory, this allows to copy # the packages directory into any docker image context: ../ dockerfile: services/sidecar/Dockerfile cache_from: - - itisfoundation/sidecar:cache - - itisfoundation/sidecar:staging-latest - - services_sidecar:latest + - ${DOCKER_REGISTRY:-itisfoundation}/sidecar:cache + - ${DOCKER_REGISTRY:-itisfoundation}/sidecar:${DOCKER_IMAGE_TAG:-latest} target: production labels: org.label-schema.schema-version: "1.0" @@ -195,16 +191,15 @@ services: - minio #-------------------------------------------------------------------- storage: - image: ${DOCKER_IMAGE_PREFIX}storage:${DOCKER_IMAGE_TAG} + image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} build: # the context for the build is the git repo root directory, this allows to copy # the packages directory into any docker image context: ../ dockerfile: services/storage/Dockerfile cache_from: - - itisfoundation/storage:cache - - itisfoundation/storage:staging-latest - - services_storage:latest + - ${DOCKER_REGISTRY:-itisfoundation}/storage:cache + - ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} target: production labels: org.label-schema.schema-version: "1.0" diff --git a/services/web/Dockerfile b/services/web/Dockerfile index a06789ecb8c..882df5f53f3 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -77,18 +77,15 @@ ENV SC_BUILD_TARGET cache COPY --chown=scu:scu packages /build/packages COPY --chown=scu:scu services/web/server /build/services/web/server -# front-end client -COPY --from=services_webclient:build --chown=scu:scu \ - /home/scu/client/build-output \ - /build/services/web/client - - WORKDIR /build/services/web/server RUN $SC_PIP install -r requirements/prod.txt &&\ $SC_PIP list -v - +# front-end client last +COPY --from=services_webclient:build --chown=scu:scu \ + /home/scu/client/build-output \ + /build/services/web/client # --------------------------Production stage ------------------- # Final cleanup up to reduce image size and startup setup # Runs as scu (non-root user) diff --git a/services/web/client/Dockerfile b/services/web/client/Dockerfile index fef53a7e4e3..081f3475905 100644 --- a/services/web/client/Dockerfile +++ b/services/web/client/Dockerfile @@ -71,11 +71,6 @@ CMD ["serve", "--set", "qx.allowUrlSettings=true"] # ------------------------------------------------------------------------------------------ FROM base-stage as build -ARG VCS_URL -ARG VCS_REF -ARG VCS_REF_CLIENT -ARG VCS_STATUS_CLIENT - # 3. prepares docker COPY --chown=scu:scu docker docker RUN chmod +x docker/*.sh @@ -89,6 +84,11 @@ RUN ./docker/install-contrib.sh COPY --chown=scu:scu source source COPY --chown=scu:scu Manifest.json Manifest.json +ARG VCS_URL +ARG VCS_REF +ARG VCS_REF_CLIENT +ARG VCS_STATUS_CLIENT + RUN qx compile --target=build \ --set osparc.vcsOriginUrl=\"${VCS_URL}\" \ --set osparc.vcsRef=\"${VCS_REF}\" \ diff --git a/services/web/server/tests/integration/fixtures/docker_compose.py b/services/web/server/tests/integration/fixtures/docker_compose.py index 82cc4c9cd9e..30f06a8a028 100644 --- a/services/web/server/tests/integration/fixtures/docker_compose.py +++ b/services/web/server/tests/integration/fixtures/docker_compose.py @@ -4,6 +4,7 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import os import socket from copy import deepcopy from pathlib import Path @@ -70,7 +71,6 @@ def docker_compose_file(request, temp_folder, services_docker_compose, devel_env docker_compose_path = temp_folder / 'docker-compose.yml' _recreate_compose_file(core_services, services_docker_compose, docker_compose_path, devel_environ) - yield Path(docker_compose_path) # cleanup @@ -118,7 +118,6 @@ def _recreate_compose_file(keep, services_compose, docker_compose_path, devel_en # remove builds if "build" in service: service.pop("build", None) - service['image'] = "services_{}:latest".format(name) # replaces environs if "environment" in service: _environs = {}