diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 57474212ec6..8705bf17e2e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,7 +29,8 @@ Makefile @pcrespov @sanderegg /services/clusters-keeper/ @sanderegg /services/datcore-adapter/ @sanderegg /services/director*/ @sanderegg @pcrespov @GitHK -/services/docker-compose*.yml @sanderegg @mrnicegyu11 @YuryHrytsuk +/services/docker-compose.yml @sanderegg @mrnicegyu11 @YuryHrytsuk +/services/docker-compose.*.yml @sanderegg /services/dynamic-sidecar/ @GitHK /services/efs-guardian/ @matusdrobuliak66 /services/invitations/ @pcrespov diff --git a/Makefile b/Makefile index af831365f8e..e1e51612e5d 100644 --- a/Makefile +++ b/Makefile @@ -357,6 +357,8 @@ endef show-endpoints: @$(_show_endpoints) +export HOST_UV_CACHE_DIR := $(shell uv cache dir) + up-devel: .stack-simcore-development.yml .init-swarm $(CLIENT_WEB_OUTPUT) ## Deploys local development stack, qx-compile+watch and ops stack (pass 'make ops_disabled=1 up-...' to disable) # Start compile+watch front-end container [front-end] @$(MAKE_C) services/static-webserver/client down compile-dev flags=--watch diff --git a/packages/service-library/src/servicelib/fastapi/cancellation_middleware.py b/packages/service-library/src/servicelib/fastapi/cancellation_middleware.py index a10b8788042..8116869af5d 100644 --- a/packages/service-library/src/servicelib/fastapi/cancellation_middleware.py +++ b/packages/service-library/src/servicelib/fastapi/cancellation_middleware.py @@ -20,7 +20,9 @@ async def _message_poller( while True: message = await receive() if message["type"] == "http.disconnect": - _logger.info("client disconnected, terminating request to %s!", request.url) + _logger.debug( + "client disconnected, terminating request to %s!", request.url + ) raise _TerminateTaskGroupError # Puts the message in the queue @@ -57,7 +59,6 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: # Let's make a shared queue for the request messages queue: asyncio.Queue[Message] = asyncio.Queue() - request = Request(scope) with log_context(_logger, logging.DEBUG, f"cancellable request {request.url}"): @@ -72,6 +73,8 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: await handler_task poller_task.cancel() except* _TerminateTaskGroupError: - _logger.info( - "The client disconnected. request to %s was cancelled.", request.url - ) + if not handler_task.done(): + _logger.info( + "The client disconnected. request to %s was cancelled.", + request.url, + ) diff --git a/services/agent/docker/boot.sh b/services/agent/docker/boot.sh index 259e1f54e71..74c058e55ec 100755 --- a/services/agent/docker/boot.sh +++ b/services/agent/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/agent - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/api-server/docker/boot.sh b/services/api-server/docker/boot.sh index 38322883120..2f545af3a59 100755 --- a/services/api-server/docker/boot.sh +++ b/services/api-server/docker/boot.sh @@ -19,7 +19,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/api-server - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -27,7 +27,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/autoscaling/docker/boot.sh b/services/autoscaling/docker/boot.sh index c97dedd7afa..a9adc4cda70 100755 --- a/services/autoscaling/docker/boot.sh +++ b/services/autoscaling/docker/boot.sh @@ -24,14 +24,14 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/autoscaling - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - uv pip list fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/catalog/docker/boot.sh b/services/catalog/docker/boot.sh index f7f01aec46e..ae506626b73 100755 --- a/services/catalog/docker/boot.sh +++ b/services/catalog/docker/boot.sh @@ -19,7 +19,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/catalog - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -27,7 +27,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/clusters-keeper/docker/boot.sh b/services/clusters-keeper/docker/boot.sh index 7623ff78e43..e25a2bb280c 100755 --- a/services/clusters-keeper/docker/boot.sh +++ b/services/clusters-keeper/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/clusters-keeper - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/dask-sidecar/docker/boot.sh b/services/dask-sidecar/docker/boot.sh index 2cfdbf349d7..89a4d14afc6 100755 --- a/services/dask-sidecar/docker/boot.sh +++ b/services/dask-sidecar/docker/boot.sh @@ -25,7 +25,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' cd services/dask-sidecar - uv pip sync --quiet --no-cache-dir requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - print_info "PIP :" uv pip list @@ -33,7 +33,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/datcore-adapter/docker/boot.sh b/services/datcore-adapter/docker/boot.sh index 74f8d84a7aa..848a70e4733 100755 --- a/services/datcore-adapter/docker/boot.sh +++ b/services/datcore-adapter/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/datcore-adapter - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/director-v2/docker/boot.sh b/services/director-v2/docker/boot.sh index a96161d9660..e8761c085a8 100755 --- a/services/director-v2/docker/boot.sh +++ b/services/director-v2/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/director-v2 - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/director/docker/boot.sh b/services/director/docker/boot.sh index eba9085247c..35c79fc1a7a 100755 --- a/services/director/docker/boot.sh +++ b/services/director/docker/boot.sh @@ -24,14 +24,14 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/director - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - uv pip list fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index 84a10a79930..a9928bc4cda 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -9,6 +9,7 @@ x-common-environment: &common-environment # Enforces app to boot debug mode (see docker/boot.sh). ONLY allowed in devel-mode! SC_BOOT_MODE : debug + services: api-server: environment: @@ -18,6 +19,7 @@ services: volumes: - ./api-server:/devel/services/api-server - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv autoscaling: environment: @@ -27,6 +29,7 @@ services: volumes: - ./autoscaling:/devel/services/autoscaling - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv invitations: environment: @@ -35,6 +38,7 @@ services: volumes: - ./invitations:/devel/services/invitations - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv payments: environment: @@ -43,6 +47,7 @@ services: volumes: - ./payments:/devel/services/payments - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv dynamic-schdlr: environment: @@ -52,6 +57,7 @@ services: volumes: - ./dynamic-scheduler:/devel/services/dynamic-scheduler - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv deploy: replicas: 1 @@ -64,6 +70,7 @@ services: volumes: - ./catalog:/devel/services/catalog - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv clusters-keeper: environment: @@ -72,6 +79,7 @@ services: volumes: - ./clusters-keeper:/devel/services/clusters-keeper - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv datcore-adapter: environment: @@ -80,6 +88,7 @@ services: volumes: - ./datcore-adapter:/devel/services/datcore-adapter - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv director: environment: @@ -88,7 +97,7 @@ services: volumes: - ./director:/devel/services/director - ../packages:/devel/packages - - ../api:/devel/services/api + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv director-v2: environment: @@ -100,6 +109,7 @@ services: volumes: - ./director-v2:/devel/services/director-v2 - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv efs-guardian: environment: @@ -108,6 +118,7 @@ services: volumes: - ./efs-guardian:/devel/services/efs-guardian - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv deploy: replicas: 0 @@ -122,6 +133,7 @@ services: volumes: &webserver_volumes_devel - ./web/server:/devel/services/web/server - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv environment: &webserver_environment_devel <<: *common-environment DEBUG: 1 # NOTE: gunicorn expects an int not a boolean @@ -151,6 +163,7 @@ services: &dask-sidecar_volumes_devel - ./dask-sidecar:/devel/services/dask-sidecar - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv - ${ETC_HOSTNAME:-/etc/hostname}:/home/scu/hostname:ro environment: @@ -179,11 +192,13 @@ services: volumes: - ./resource-usage-tracker:/devel/services/resource-usage-tracker - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv storage: volumes: - ./storage:/devel/services/storage - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv environment: <<: *common-environment STORAGE_PROFILING : ${STORAGE_PROFILING} @@ -193,6 +208,7 @@ services: volumes: - ./storage:/devel/services/storage - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv environment: <<: *common-environment STORAGE_PROFILING : ${STORAGE_PROFILING} @@ -205,3 +221,4 @@ services: volumes: - ./agent:/devel/services/agent - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv diff --git a/services/dynamic-scheduler/docker/boot.sh b/services/dynamic-scheduler/docker/boot.sh index a5a3ea811d5..c6c3cd4c849 100755 --- a/services/dynamic-scheduler/docker/boot.sh +++ b/services/dynamic-scheduler/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/dynamic-scheduler - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/dynamic-sidecar/docker/boot.sh b/services/dynamic-sidecar/docker/boot.sh index d21e87cb070..b68f0ae22ba 100755 --- a/services/dynamic-sidecar/docker/boot.sh +++ b/services/dynamic-sidecar/docker/boot.sh @@ -25,7 +25,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then # NOTE: uv does not like this requirement file... cd /devel/services/dynamic-sidecar - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" pip list | sed 's/^/ /' @@ -33,7 +33,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/efs-guardian/docker/boot.sh b/services/efs-guardian/docker/boot.sh index ad8de46bfda..e2452e13a6e 100755 --- a/services/efs-guardian/docker/boot.sh +++ b/services/efs-guardian/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/efs-guardian - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/invitations/docker/boot.sh b/services/invitations/docker/boot.sh index 7cbf4401013..cd8d839d053 100755 --- a/services/invitations/docker/boot.sh +++ b/services/invitations/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/invitations - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/payments/docker/boot.sh b/services/payments/docker/boot.sh index eb77d928809..fa3024433dc 100755 --- a/services/payments/docker/boot.sh +++ b/services/payments/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/payments - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/resource-usage-tracker/docker/boot.sh b/services/resource-usage-tracker/docker/boot.sh index 2b3c1bebc5f..ddfbfaf306e 100755 --- a/services/resource-usage-tracker/docker/boot.sh +++ b/services/resource-usage-tracker/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/resource-usage-tracker - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index dc913e99277..4cc5128b43a 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -24,7 +24,7 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then command -v python | sed 's/^/ /' cd services/storage - uv pip --quiet --no-cache-dir sync requirements/dev.txt + uv pip --quiet sync requirements/dev.txt cd - echo "$INFO" "PIP :" uv pip list @@ -32,7 +32,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi # diff --git a/services/web/server/docker/boot.sh b/services/web/server/docker/boot.sh index 08a812f7bb0..a65425c1155 100755 --- a/services/web/server/docker/boot.sh +++ b/services/web/server/docker/boot.sh @@ -31,7 +31,7 @@ fi if [ "${SC_BOOT_MODE}" = "debug" ]; then # NOTE: production does NOT pre-installs debugpy - uv pip install --no-cache-dir debugpy + uv pip install debugpy fi APP_LOG_LEVEL=${WEBSERVER_LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} diff --git a/services/web/server/tests/unit/with_dbs/docker-compose-devel.yml b/services/web/server/tests/unit/with_dbs/docker-compose-devel.yml index 053e270b29f..15c36b03136 100644 --- a/services/web/server/tests/unit/with_dbs/docker-compose-devel.yml +++ b/services/web/server/tests/unit/with_dbs/docker-compose-devel.yml @@ -34,6 +34,9 @@ services: - "log_min_duration_statement=500" - "-c" - "log_lock_waits=on" + # -c fsync=off is not recommended for production as this disable writing to disk https://pythonspeed.com/articles/faster-db-tests/ + - "-c" + - "fsync=off" adminer: image: adminer:4.8.1 init: true diff --git a/services/web/server/tests/unit/with_dbs/docker-compose.yml b/services/web/server/tests/unit/with_dbs/docker-compose.yml index 2ee36551d1e..f2cba772f46 100644 --- a/services/web/server/tests/unit/with_dbs/docker-compose.yml +++ b/services/web/server/tests/unit/with_dbs/docker-compose.yml @@ -29,6 +29,9 @@ services: - "tcp_keepalives_count=5" - "-c" - "log_lock_waits=on" + # -c fsync=off is not recommended for production as this disable writing to disk https://pythonspeed.com/articles/faster-db-tests/ + - "-c" + - "fsync=off" redis: image: "redis:6.2.6@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d" init: true