API for solvers with fake backend (#1866)

New extension of the API to run solvers. 
Adds three new resources: jobs, solves and files. 
Current implementation of the backend uses a faker data and services.

Author: pcrespov

packages/models-library/src/models_library/services.py

@@ -4,7 +4,7 @@
     python -c "from models_library.services import ServiceDockerData as cls; print(cls.schema_json(indent=2))" > services-schema.json
 """
 from enum import Enum
-from typing import Dict, List, Optional, Union, Any
+from typing import Any, Dict, List, Optional, Union
 
 from pydantic import (
     BaseModel,
@@ -22,7 +22,9 @@
 
 from .basic_regex import VERSION_RE
 
+# NOTE: needs to end with / !!
 SERVICE_KEY_RE = r"^(simcore)/(services)/(comp|dynamic|frontend)(/[\w/-]+)+$"
+COMPUTATIONAL_SERVICE_KEY_RE = r"^(simcore)/(services)/comp(/[\w/-]+)+$"
 KEY_RE = SERVICE_KEY_RE  # TODO: deprecate this global constant by SERVICE_KEY_RE
 
 PROPERTY_TYPE_RE = r"^(number|integer|boolean|string|data:([^/\s,]+/[^/\s,]+|\[[^/\s,]+/[^/\s,]+(,[^/\s]+/[^/,\s]+)*\]))$"

packages/pytest-simcore/src/pytest_simcore/repository_paths.py

@@ -1,5 +1,4 @@
 # pylint: disable=redefined-outer-name
-
 import sys
 from pathlib import Path
 
@@ -88,3 +87,26 @@ def pylintrc(osparc_simcore_root_dir: Path) -> Path:
     pylintrc = osparc_simcore_root_dir / ".pylintrc"
     assert pylintrc.exists()
     return pylintrc
+
+
+@pytest.fixture(scope="session")
+def tests_dir() -> Path:
+    tdir = (current_dir / "..").resolve()
+    assert tdir.exists()
+    assert tdir.name == "tests"
+    return tdir
+
+
+## PACKAGE and SERVICE DIRECTORY STRUCTURE
+
+
+@pytest.fixture(scope="session")
+def project_slug_dir() -> Path:
+    raise NotImplementedError("Override fixture in project's tests/conftest.py")
+
+
+@pytest.fixture(scope="session")
+def project_tests_dir(project_slug_dir: Path) -> Path:
+    test_dir = project_slug_dir / "tests"
+    assert test_dir.exists()
+    return test_dir

scripts/common-service.Makefile

@@ -70,7 +70,7 @@ shell: ## runs shell in production container
 
 .PHONY: tail
 tail: ## tails log of $(APP_NAME) container
-	docker logs --follow $(shell docker ps -f "name=$(APP_NAME)*" --format {{.ID}}) > $(APP_NAME).log 2>&1
+	docker logs --follow $(shell docker ps -f "name=$(APP_NAME)*" --format {{.ID}}) 2>&1
 
 
 .PHONY: info

scripts/common.Makefile

@@ -35,6 +35,9 @@ VCS_REF       := $(shell git rev-parse --short HEAD)
 NOW_TIMESTAMP := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 
+# relevant repo folders
+SCRIPTS_DIR := $(abspath $(REPO_BASE_DIR)/scripts)
+
 # virtual env
 VENV_DIR      := $(abspath $(REPO_BASE_DIR)/.venv)
 

services/api-server/.env-devel

@@ -1,6 +1,8 @@
 #
 # Environment variables used to configure this service
 #
+FAKE_API_SERVER_ENABLED=0
+DEBUG=0
 
 # SEE services/api-server/src/simcore_service_api_server/auth_security.py
 SECRET_KEY=d0d0397de2c85ad26ffd4a0f9643dfe3a0ca3937f99cf3c2e174e11b5ef79880
@@ -22,3 +24,14 @@ WEBSERVER_ENABLED=1
 WEBSERVER_HOST=webserver
 # Take from general .env-devel
 WEBSERVER_SESSION_SECRET_KEY=REPLACE ME with a key of at least length 32.
+
+
+# catalog
+CATALOG_ENABLED=1
+CATALOG_HOST=catalog
+
+# storage
+STORAGE_HOST=storage
+
+# director
+DIRECTO2_HOST=director-v2

services/api-server/.env-fake-standalone

@@ -0,0 +1,39 @@
+#
+# Environment variables used to configure this service stand-alone in FAKE mode
+#
+FAKE_API_SERVER_ENABLED=1
+
+# SEE services/api-server/src/simcore_service_api_server/auth_security.py
+SECRET_KEY=d0d0397de2c85ad26ffd4a0f9643dfe3a0ca3937f99cf3c2e174e11b5ef79880
+
+# SEE services/api-server/src/simcore_service_api_server/settings.py
+LOG_LEVEL=DEBUG
+
+POSTGRES_ENABLED=0
+POSTGRES_USER=test
+POSTGRES_PASSWORD=test
+POSTGRES_DB=test
+POSTGRES_HOST=localhost
+
+# Enables debug
+SC_BOOT_MODE=production
+
+
+# webserver
+WEBSERVER_ENABLED=0
+WEBSERVER_HOST=webserver
+# Take from general .env-devel
+WEBSERVER_SESSION_SECRET_KEY=REPLACE ME with a key of at least length 32.
+
+
+# catalog
+CATALOG_ENABLED=0
+CATALOG_HOST=catalog
+
+# storage
+STORAGE_ENABLED=0
+STORAGE_HOST=storage
+
+# director
+DIRECTOR2_ENABLED=0
+DIRECTO2_HOST=director-v2

services/api-server/.gitignore

@@ -1,4 +1,2 @@
 # outputs from makefile
 client
-docker-compose.yml
-.env

services/api-server/Makefile

@@ -4,63 +4,98 @@
 include ../../scripts/common.Makefile
 include ../../scripts/common-service.Makefile
 
-
 .PHONY: reqs
 reqs: ## compiles pip requirements (.in -> .txt)
 	@$(MAKE_C) requirements reqs
 
 
-
-
-# DEVELOPMENT TOOLS ########
+# DEVELOPMENT TOOLS ###########################################################################
 
 .env:
 	cp .env-devel $@
 
-docker-compose.yml:
+DOCKER_COMPOSE_EXTRA_FILE:=.docker-compose-extra-ignore.yml
+$(DOCKER_COMPOSE_EXTRA_FILE):
 	cp $(CURDIR)/tests/utils/docker-compose.yml $@
 
-.PHONY: run-devel down
-run-devel: .env docker-compose.yml down ## runs app on host with pg fixture for development [for development]
-	# Starting db (under $<)
-	docker-compose up --detach
-	# Creating db-tables: user=key, password=secret
-	@$(MAKE) db-tables
-	# start app (under $<)
-	uvicorn simcore_service_api_server.__main__:the_app \
-		--reload --reload-dir $(SRC_DIR) \
-		--port=8000 --host=0.0.0.0
 
 .PHONY: db-tables
 db-tables: .env ## upgrades and create tables [for development]
 	# Upgrading and creating tables
 	export $(shell grep -v '^#' $< | xargs  -d '\n'); \
 	python3 tests/utils/init-pg.py
 
+
+
 .PHONY: db-migration
 db-migration: .env ## runs discover and upgrade on running pg-db [for development]
 	# Creating tables
 	export $(shell grep -v '^#' $< | xargs  -d '\n'); \
 	sc-pg discover && sc-pg upgrade
 
-down: docker-compose.yml ## stops pg fixture
+
+.PHONY: down
+down: $(DOCKER_COMPOSE_EXTRA_FILE)## stops pg fixture
 	# stopping extra services
 	-@docker-compose -f $< down
-	# killing any process using port 8000
+	# killing any process using port 8000 (app) and 3000 (debug)
 	-@fuser --kill --verbose --namespace tcp 8000
+	-@fuser --kill --verbose --namespace tcp 3000
 
 
+# TODO: These are all different ways to run the server for dev-purposes .
+
+.PHONY: run-devel
+run-devel: .env $(DOCKER_COMPOSE_EXTRA_FILE) down
+	# Starting db (under $<)
+	docker-compose --file $(DOCKER_COMPOSE_EXTRA_FILE)  up --detach
+	sleep 2
+	# Creating db-tables: user=key, password=secret
+	@$(MAKE) db-tables
+	# start app (under $<)
+	uvicorn simcore_service_api_server.__main__:the_app \
+		--reload --reload-dir $(SRC_DIR) \
+		--port=8000 --host=0.0.0.0 \
+		--log-level debug
+
+
+.PHONY: run-fake run-fake-devel
+run-fake: # starts a fake server in a container
+	docker run -it \
+		--env-file .env-fake-standalone \
+		--publish 8000:8000 \
+		local/${APP_NAME}:production
+	# Open http://172.0.0.1:8000/dev/doc
 
-# BUILD ########
+
+run-fake-devel: # starts a fake server in a dev-container
+	docker run -it \
+		--env-file .env-fake-standalone \
+	  --env SC_BOOT_MODE=debug-ptvsd \
+    --env LOG_LEVEL=debug \
+    --env DEBUG=true \
+		--publish 8000:8000 \
+		--publish 3006:3000 \
+		--volume $(REPO_BASE_DIR)/services/api-server:/devel/services/api-server \
+		--volume $(REPO_BASE_DIR)/packages:/devel/packages \
+		local/${APP_NAME}:development
+	# Open http://172.0.0.1:8000/dev/doc
+
+
+# BUILD ###########################################################################
 
 .PHONY: openapi-specs openapi.json
 openapi-specs: openapi.json
 openapi.json: .env
 	# generating openapi specs file
 	python3 -c "import json; from $(APP_PACKAGE_NAME).__main__ import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@
+	# validates OAS file: $@
+	@cd $(CURDIR); \
+	$(SCRIPTS_DIR)/openapi-generator-cli.bash validate --input-spec /local/$@
 
 
-# GENERATION python client ########
+
+# GENERATION python client ###########################################################################
 .PHONY: python-client generator-help
 
 # SEE https://openapi-generator.tech/docs/usage#generate
@@ -69,12 +104,10 @@ openapi.json: .env
 # TODO: put instead to additional-props.yaml and --config=openapi-generator/python-config.yaml
 # TODO: copy this code to https://github.com/ITISFoundation/osparc-simcore-python-client/blob/master/Makefile
 #
-
 # NOTE: assumes this repo exists
 GIT_USER_ID := ITISFoundation
 GIT_REPO_ID := osparc-simcore-python-client
 
-SCRIPTS_DIR := $(abspath $(CURDIR)/../../scripts)
 GENERATOR_NAME := python
 
 ADDITIONAL_PROPS := \
@@ -94,23 +127,28 @@ comma := ,
 client:
 	# cloning $(GIT_USER_ID)/$(GIT_REPO_ID) -> $@
 	git clone [email protected]:$(GIT_USER_ID)/$(GIT_REPO_ID).git $@
-	cd client; git checkout -b "upgrade-${APP_VERSION}"
+	# TODO: if fails, add -b
+	cd client; git checkout "upgrade-${APP_VERSION}"
 
 
 python-client: client openapi.json ## runs python client generator
+	# copies latest version of the openapi
+	@cp openapi.json client/api/
+	# generates
 	cd $(CURDIR); \
 	$(SCRIPTS_DIR)/openapi-generator-cli.bash generate \
 		--generator-name=$(GENERATOR_NAME) \
 		--git-user-id=$(GIT_USER_ID)\
 		--git-repo-id=$(GIT_REPO_ID)\
-		--http-user-agent="osparc-api/{packageVersion}/{language}"\
+		--http-user-agent="osparc-api/$(APP_VERSION)/python"\
 		--input-spec=/local/openapi.json \
 		--output=/local/client \
 		--additional-properties=$(subst $(space),$(comma),$(strip $(ADDITIONAL_PROPS)))\
 		--package-name=osparc\
 		--release-note="Updated to $(APP_VERSION)"
 
 
+
 generator-help: ## help on client-api generator
 	# generate help
 	@$(SCRIPTS_DIR)/openapi-generator-cli.bash help generate

services/api-server/README.md

@@ -21,7 +21,7 @@ Setup environment
 ```cmd
 make devenv
 source .venv/bin/activate
-cd services/api-service
+cd services/api-server
 make install-dev
 ```
 Then

services/api-server/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

services/api-server/docker/boot.sh

@@ -30,7 +30,7 @@ if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ]
 then
   # NOTE: ptvsd is programmatically enabled inside of the service
   # this way we can have reload in place as well
-  exec uvicorn simcore_service_api_server.__main__:the_app --reload --host 0.0.0.0
+  exec uvicorn simcore_service_api_server.__main__:the_app --reload --host 0.0.0.0 --reload-dir services/api-server/src/simcore_service_api_server
 else
   exec simcore-service-api-server
 fi