diff --git a/packages/service-library/requirements/_aiohttp.in b/packages/service-library/requirements/_aiohttp.in index d829728aa81..a01dcbcacd0 100644 --- a/packages/service-library/requirements/_aiohttp.in +++ b/packages/service-library/requirements/_aiohttp.in @@ -3,7 +3,7 @@ # # ---constraint ./_base.in +--constraint ./_base.txt aiohttp aiopg[sa] diff --git a/packages/service-library/requirements/_aiohttp.txt b/packages/service-library/requirements/_aiohttp.txt index e0134b56569..b48863ff5ca 100644 --- a/packages/service-library/requirements/_aiohttp.txt +++ b/packages/service-library/requirements/_aiohttp.txt @@ -1,52 +1,64 @@ aiohappyeyeballs==2.4.6 - # via aiohttp + # via + # -c requirements/./_base.txt + # aiohttp aiohttp==3.11.18 # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt + # -c requirements/./_base.txt # -r requirements/_aiohttp.in aiopg==1.4.0 # via -r requirements/_aiohttp.in aiosignal==1.3.2 - # via aiohttp + # via + # -c requirements/./_base.txt + # aiohttp async-timeout==4.0.3 # via aiopg attrs==25.1.0 # via + # -c requirements/./_base.txt # -r requirements/_aiohttp.in # aiohttp # jsonschema # referencing deprecated==1.2.18 # via + # -c requirements/./_base.txt # opentelemetry-api # opentelemetry-semantic-conventions frozenlist==1.5.0 # via + # -c requirements/./_base.txt # aiohttp # aiosignal greenlet==3.1.1 # via sqlalchemy idna==3.10 - # via yarl + # via + # -c requirements/./_base.txt + # yarl importlib-metadata==8.5.0 - # via opentelemetry-api + # via + # -c requirements/./_base.txt + # opentelemetry-api jsonschema==4.23.0 - # via -r requirements/_aiohttp.in + # via + # -c requirements/./_base.txt + # -r requirements/_aiohttp.in jsonschema-specifications==2024.10.1 - # via jsonschema + # via + # -c requirements/./_base.txt + # jsonschema markupsafe==3.0.2 # via werkzeug multidict==6.1.0 # via + # -c requirements/./_base.txt # aiohttp # yarl opentelemetry-api==1.30.0 # via + # -c requirements/./_base.txt # opentelemetry-instrumentation # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server @@ -55,6 +67,7 @@ opentelemetry-api==1.30.0 # opentelemetry-semantic-conventions opentelemetry-instrumentation==0.51b0 # via + # -c requirements/./_base.txt # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-aiopg @@ -69,20 +82,25 @@ opentelemetry-instrumentation-dbapi==0.51b0 # via opentelemetry-instrumentation-aiopg opentelemetry-semantic-conventions==0.51b0 # via + # -c requirements/./_base.txt # opentelemetry-instrumentation # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-dbapi opentelemetry-util-http==0.51b0 # via + # -c requirements/./_base.txt # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server packaging==24.2 - # via opentelemetry-instrumentation + # via + # -c requirements/./_base.txt + # opentelemetry-instrumentation prometheus-client==0.21.1 # via -r requirements/_aiohttp.in propcache==0.3.0 # via + # -c requirements/./_base.txt # aiohttp # yarl psycopg2-binary==2.9.10 @@ -91,31 +109,21 @@ psycopg2-binary==2.9.10 # sqlalchemy referencing==0.35.1 # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt + # -c requirements/./_base.txt # jsonschema # jsonschema-specifications rpds-py==0.23.1 # via + # -c requirements/./_base.txt # jsonschema # referencing sqlalchemy==1.4.54 - # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # aiopg + # via aiopg werkzeug==3.1.3 # via -r requirements/_aiohttp.in wrapt==1.17.2 # via + # -c requirements/./_base.txt # deprecated # opentelemetry-instrumentation # opentelemetry-instrumentation-aiohttp-client @@ -123,6 +131,10 @@ wrapt==1.17.2 # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-dbapi yarl==1.18.3 - # via aiohttp + # via + # -c requirements/./_base.txt + # aiohttp zipp==3.21.0 - # via importlib-metadata + # via + # -c requirements/./_base.txt + # importlib-metadata diff --git a/packages/service-library/requirements/_fastapi.in b/packages/service-library/requirements/_fastapi.in index 1ea0f1c0477..8d0c4f62782 100644 --- a/packages/service-library/requirements/_fastapi.in +++ b/packages/service-library/requirements/_fastapi.in @@ -3,7 +3,7 @@ # # ---constraint ./_base.in +--constraint ./_base.txt fastapi @@ -12,5 +12,4 @@ httpx opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-httpx prometheus-client -prometheus-fastapi-instrumentator uvicorn diff --git a/packages/service-library/requirements/_fastapi.txt b/packages/service-library/requirements/_fastapi.txt index 48615d9819a..e7b3ea8219a 100644 --- a/packages/service-library/requirements/_fastapi.txt +++ b/packages/service-library/requirements/_fastapi.txt @@ -1,25 +1,26 @@ annotated-types==0.7.0 - # via pydantic + # via + # -c requirements/./_base.txt + # pydantic anyio==4.8.0 # via + # -c requirements/./_base.txt # httpx # starlette asgiref==3.8.1 # via opentelemetry-instrumentation-asgi certifi==2025.1.31 # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt + # -c requirements/./_base.txt # httpcore # httpx click==8.1.8 - # via uvicorn + # via + # -c requirements/./_base.txt + # uvicorn deprecated==1.2.18 # via + # -c requirements/./_base.txt # opentelemetry-api # opentelemetry-semantic-conventions fastapi==0.115.11 @@ -35,22 +36,19 @@ h11==0.14.0 httpcore==1.0.7 # via httpx httpx==0.28.1 - # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_fastapi.in + # via -r requirements/_fastapi.in idna==3.10 # via + # -c requirements/./_base.txt # anyio # httpx importlib-metadata==8.5.0 - # via opentelemetry-api + # via + # -c requirements/./_base.txt + # opentelemetry-api opentelemetry-api==1.30.0 # via + # -c requirements/./_base.txt # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -58,6 +56,7 @@ opentelemetry-api==1.30.0 # opentelemetry-semantic-conventions opentelemetry-instrumentation==0.51b0 # via + # -c requirements/./_base.txt # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx @@ -69,48 +68,40 @@ opentelemetry-instrumentation-httpx==0.51b0 # via -r requirements/_fastapi.in opentelemetry-semantic-conventions==0.51b0 # via + # -c requirements/./_base.txt # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx opentelemetry-util-http==0.51b0 # via + # -c requirements/./_base.txt # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx packaging==24.2 - # via opentelemetry-instrumentation -prometheus-client==0.21.1 # via - # -r requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.2 + # -c requirements/./_base.txt + # opentelemetry-instrumentation +prometheus-client==0.21.1 # via -r requirements/_fastapi.in pydantic==2.10.6 # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt + # -c requirements/./_base.txt # fastapi pydantic-core==2.27.2 - # via pydantic + # via + # -c requirements/./_base.txt + # pydantic sniffio==1.3.1 - # via anyio -starlette==0.46.0 # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # fastapi - # prometheus-fastapi-instrumentator + # -c requirements/./_base.txt + # anyio +starlette==0.46.0 + # via fastapi typing-extensions==4.12.2 # via + # -c requirements/./_base.txt # anyio # fastapi # pydantic @@ -119,8 +110,11 @@ uvicorn==0.34.0 # via -r requirements/_fastapi.in wrapt==1.17.2 # via + # -c requirements/./_base.txt # deprecated # opentelemetry-instrumentation # opentelemetry-instrumentation-httpx zipp==3.21.0 - # via importlib-metadata + # via + # -c requirements/./_base.txt + # importlib-metadata diff --git a/packages/service-library/src/servicelib/aiohttp/monitoring.py b/packages/service-library/src/servicelib/aiohttp/monitoring.py index 51aba532721..b46dcde79c9 100644 --- a/packages/service-library/src/servicelib/aiohttp/monitoring.py +++ b/packages/service-library/src/servicelib/aiohttp/monitoring.py @@ -1,133 +1,47 @@ -""" Enables monitoring of some quantities needed for diagnostics - -""" +"""Enables monitoring of some quantities needed for diagnostics""" import asyncio import logging import time -from typing import Awaitable, Callable, cast +from collections.abc import Awaitable, Callable +from typing import Final -import prometheus_client from aiohttp import web -from prometheus_client import ( +from prometheus_client.openmetrics.exposition import ( CONTENT_TYPE_LATEST, - Counter, - Gauge, - GCCollector, - PlatformCollector, - ProcessCollector, - Summary, + generate_latest, ) from prometheus_client.registry import CollectorRegistry -from servicelib.aiohttp.typing_extension import Handler from ..common_headers import ( UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, X_SIMCORE_USER_AGENT, ) from ..logging_utils import log_catch +from ..prometheus_metrics import ( + PrometheusMetrics, + get_prometheus_metrics, + record_request_metrics, + record_response_metrics, +) +from .typing_extension import Handler log = logging.getLogger(__name__) - -# -# CAUTION CAUTION CAUTION NOTE: -# Be very careful with metrics. pay attention to metrics cardinatity. -# Each time series takes about 3kb of overhead in Prometheus -# -# CAUTION: every unique combination of key-value label pairs represents a new time series -# -# If a metrics is not needed, don't add it!! It will collapse the application AND prometheus -# -# references: -# https://prometheus.io/docs/practices/naming/ -# https://www.robustperception.io/cardinality-is-key -# https://www.robustperception.io/why-does-prometheus-use-so-much-ram -# https://promcon.io/2019-munich/slides/containing-your-cardinality.pdf -# https://grafana.com/docs/grafana-cloud/how-do-i/control-prometheus-metrics-usage/usage-analysis-explore/ -# - - -# This creates the following basic metrics: -# # HELP process_virtual_memory_bytes Virtual memory size in bytes. -# # TYPE process_virtual_memory_bytes gauge -# process_virtual_memory_bytes 8.12425216e+08 -# # HELP process_resident_memory_bytes Resident memory size in bytes. -# # TYPE process_resident_memory_bytes gauge -# process_resident_memory_bytes 1.2986368e+08 -# # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. -# # TYPE process_start_time_seconds gauge -# process_start_time_seconds 1.6418063518e+09 -# # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. -# # TYPE process_cpu_seconds_total counter -# process_cpu_seconds_total 9.049999999999999 -# # HELP process_open_fds Number of open file descriptors. -# # TYPE process_open_fds gauge -# process_open_fds 29.0 -# # HELP process_max_fds Maximum number of open file descriptors. -# # TYPE process_max_fds gauge -# process_max_fds 1.048576e+06 -# # HELP python_info Python platform information -# # TYPE python_info gauge -# python_info{implementation="CPython",major="3",minor="8",patchlevel="10",version="3.9.12"} 1.0 -# # HELP python_gc_objects_collected_total Objects collected during gc -# # TYPE python_gc_objects_collected_total counter -# python_gc_objects_collected_total{generation="0"} 7328.0 -# python_gc_objects_collected_total{generation="1"} 614.0 -# python_gc_objects_collected_total{generation="2"} 0.0 -# # HELP python_gc_objects_uncollectable_total Uncollectable object found during GC -# # TYPE python_gc_objects_uncollectable_total counter -# python_gc_objects_uncollectable_total{generation="0"} 0.0 -# python_gc_objects_uncollectable_total{generation="1"} 0.0 -# python_gc_objects_uncollectable_total{generation="2"} 0.0 -# # HELP python_gc_collections_total Number of times this generation was collected -# # TYPE python_gc_collections_total counter -# python_gc_collections_total{generation="0"} 628.0 -# python_gc_collections_total{generation="1"} 57.0 -# python_gc_collections_total{generation="2"} 5.0 -# # HELP http_requests_total Total requests count -# # TYPE http_requests_total counter -# http_requests_total{app_name="simcore_service_webserver",endpoint="/v0/",http_status="200",method="GET"} 15.0 -# # HELP http_requests_created Total requests count -# # TYPE http_requests_created gauge -# http_requests_created{app_name="simcore_service_webserver",endpoint="/v0/",http_status="200",method="GET"} 1.6418063614890063e+09 -# # HELP http_in_flight_requests Number of requests in process -# # TYPE http_in_flight_requests gauge -# http_in_flight_requests{app_name="simcore_service_webserver",endpoint="/v0/",method="GET"} 0.0 -# http_in_flight_requests{app_name="simcore_service_webserver",endpoint="/metrics",method="GET"} 1.0 -# # HELP http_request_latency_seconds Time processing a request -# # TYPE http_request_latency_seconds summary -# http_request_latency_seconds_count{app_name="simcore_service_webserver",endpoint="/v0/",method="GET"} 15.0 -# http_request_latency_seconds_sum{app_name="simcore_service_webserver",endpoint="/v0/",method="GET"} 0.007384857000033662 -# http_request_latency_seconds_count{app_name="simcore_service_webserver",endpoint="/metrics",method="GET"} 0.0 -# http_request_latency_seconds_sum{app_name="simcore_service_webserver",endpoint="/metrics",method="GET"} 0.0 -# # HELP http_request_latency_seconds_created Time processing a request -# # TYPE http_request_latency_seconds_created gauge -# http_request_latency_seconds_created{app_name="simcore_service_webserver",endpoint="/v0/",method="GET"} 1.6418063614873598e+09 -# http_request_latency_seconds_created{app_name="simcore_service_webserver",endpoint="/metrics",method="GET"} 1.641806371709292e+09 - - -kREQUEST_COUNT = f"{__name__}.request_count" # noqa: N816 -kINFLIGHTREQUESTS = f"{__name__}.in_flight_requests" # noqa: N816 -kRESPONSELATENCY = f"{__name__}.in_response_latency" # noqa: N816 - -kCOLLECTOR_REGISTRY = f"{__name__}.collector_registry" # noqa: N816 -kPROCESS_COLLECTOR = f"{__name__}.collector_process" # noqa: N816 -kPLATFORM_COLLECTOR = f"{__name__}.collector_platform" # noqa: N816 -kGC_COLLECTOR = f"{__name__}.collector_gc" # noqa: N816 +_PROMETHEUS_METRICS: Final[str] = f"{__name__}.prometheus_metrics" # noqa: N816 def get_collector_registry(app: web.Application) -> CollectorRegistry: - return cast(CollectorRegistry, app[kCOLLECTOR_REGISTRY]) + metrics = app[_PROMETHEUS_METRICS] + assert isinstance(metrics, PrometheusMetrics) # nosec + return metrics.registry async def metrics_handler(request: web.Request): registry = get_collector_registry(request.app) # NOTE: Cannot use ProcessPoolExecutor because registry is not pickable - result = await request.loop.run_in_executor( - None, prometheus_client.generate_latest, registry - ) + result = await request.loop.run_in_executor(None, generate_latest, registry) response = web.Response(body=result) response.content_type = CONTENT_TYPE_LATEST return response @@ -150,9 +64,6 @@ async def middleware_handler(request: web.Request, handler: Handler): resp: web.StreamResponse = web.HTTPInternalServerError( reason="Unexpected exception" ) - # NOTE: a canonical endpoint is `/v0/projects/{project_id}/node/{node_uuid}`` - # vs a resolved endpoint `/v0/projects/51e4bdf4-2cc7-43be-85a6-627a4c0afb77/nodes/51e4bdf4-2cc7-43be-85a6-627a4c0afb77` - # which would create way to many different endpoints for monitoring! canonical_endpoint = request.path if request.match_info.route.resource: canonical_endpoint = request.match_info.route.resource.canonical @@ -162,24 +73,19 @@ async def middleware_handler(request: web.Request, handler: Handler): with log_catch(logger=log, reraise=False): await enter_middleware_cb(request) - in_flight_gauge = request.app[kINFLIGHTREQUESTS] - response_summary = request.app[kRESPONSELATENCY] - - with in_flight_gauge.labels( - app_name, - request.method, - canonical_endpoint, - request.headers.get( - X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE - ), - ).track_inprogress(), response_summary.labels( - app_name, - request.method, - canonical_endpoint, - request.headers.get( - X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE - ), - ).time(): + metrics = request.app[_PROMETHEUS_METRICS] + assert isinstance(metrics, PrometheusMetrics) # nosec + + user_agent = request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ) + + with record_request_metrics( + metrics=metrics, + method=request.method, + endpoint=canonical_endpoint, + user_agent=user_agent, + ): resp = await handler(request) assert isinstance( # nosec @@ -187,37 +93,33 @@ async def middleware_handler(request: web.Request, handler: Handler): ), "Forgot envelope middleware?" except web.HTTPServerError as exc: - # Transforms exception into response object and log exception resp = exc log_exception = exc + raise resp from exc except web.HTTPException as exc: - # Transforms non-HTTPServerError exceptions into response object resp = exc log_exception = None + raise resp from exc except asyncio.CancelledError as exc: - # Mostly for logging resp = web.HTTPInternalServerError(reason=f"{exc}") log_exception = exc - raise + raise resp from exc except Exception as exc: # pylint: disable=broad-except - # Prevents issue #1025. resp = web.HTTPInternalServerError(reason=f"{exc}") resp.__cause__ = exc log_exception = exc + raise resp from exc finally: resp_time_secs: float = time.time() - start_time - # prometheus probes - request.app[kREQUEST_COUNT].labels( - app_name, - request.method, - canonical_endpoint, - resp.status, - request.headers.get( - X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE - ), - ).inc() + record_response_metrics( + metrics=metrics, + method=request.method, + endpoint=canonical_endpoint, + user_agent=user_agent, + http_status=resp.status, + ) if exit_middleware_cb: with log_catch(logger=log, reraise=False): @@ -239,7 +141,6 @@ async def middleware_handler(request: web.Request, handler: Handler): return resp - # adds identifier setattr( # noqa: B010 middleware_handler, "__middleware_name__", f"{__name__}.monitor_{app_name}" ) @@ -253,49 +154,8 @@ def setup_monitoring( *, enter_middleware_cb: EnterMiddlewareCB | None = None, exit_middleware_cb: ExitMiddlewareCB | None = None, - **app_info_kwargs, ): - # app-scope registry - target_info = {"application_name": app_name} - target_info.update(app_info_kwargs) - app[kCOLLECTOR_REGISTRY] = reg = CollectorRegistry( - auto_describe=False, target_info=target_info - ) - # automatically collects process metrics see [https://github.com/prometheus/client_python] - app[kPROCESS_COLLECTOR] = ProcessCollector(registry=reg) - # automatically collects python_info metrics see [https://github.com/prometheus/client_python] - app[kPLATFORM_COLLECTOR] = PlatformCollector(registry=reg) - # automatically collects python garbage collector metrics see [https://github.com/prometheus/client_python] - # prefixed with python_gc_ - app[kGC_COLLECTOR] = GCCollector(registry=reg) - - # Total number of requests processed - app[kREQUEST_COUNT] = Counter( - name="http_requests", - documentation="Total requests count", - labelnames=[ - "app_name", - "method", - "endpoint", - "http_status", - "simcore_user_agent", - ], - registry=reg, - ) - - app[kINFLIGHTREQUESTS] = Gauge( - name="http_in_flight_requests", - documentation="Number of requests in process", - labelnames=["app_name", "method", "endpoint", "simcore_user_agent"], - registry=reg, - ) - - app[kRESPONSELATENCY] = Summary( - name="http_request_latency_seconds", - documentation="Time processing a request", - labelnames=["app_name", "method", "endpoint", "simcore_user_agent"], - registry=reg, - ) + app[_PROMETHEUS_METRICS] = get_prometheus_metrics() # WARNING: ensure ERROR middleware is over this one # @@ -313,8 +173,7 @@ def setup_monitoring( # # ensures is first layer but cannot guarantee the order setup is applied - app.middlewares.insert( - 0, + app.middlewares.append( middleware_factory( app_name, enter_middleware_cb=enter_middleware_cb, diff --git a/packages/service-library/src/servicelib/fastapi/monitoring.py b/packages/service-library/src/servicelib/fastapi/monitoring.py new file mode 100644 index 00000000000..89a29b0642a --- /dev/null +++ b/packages/service-library/src/servicelib/fastapi/monitoring.py @@ -0,0 +1,136 @@ +# pylint: disable=protected-access + +import asyncio +import logging +from collections.abc import AsyncIterator +from typing import Final + +from fastapi import FastAPI, Request, Response, status +from fastapi_lifespan_manager import State +from prometheus_client import CollectorRegistry +from prometheus_client.openmetrics.exposition import ( + CONTENT_TYPE_LATEST, + generate_latest, +) +from servicelib.prometheus_metrics import ( + PrometheusMetrics, + get_prometheus_metrics, + record_request_metrics, + record_response_metrics, +) +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.types import ASGIApp + +from ..common_headers import ( + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + X_SIMCORE_USER_AGENT, +) + +_logger = logging.getLogger(__name__) +_PROMETHEUS_METRICS = "prometheus_metrics" + + +class PrometheusMiddleware(BaseHTTPMiddleware): + def __init__(self, app: ASGIApp, metrics: PrometheusMetrics): + super().__init__(app) + self.metrics = metrics + + async def dispatch( + self, request: Request, call_next: RequestResponseEndpoint + ) -> Response: + canonical_endpoint = request.url.path + user_agent = request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ) + + try: + with record_request_metrics( + metrics=self.metrics, + method=request.method, + endpoint=canonical_endpoint, + user_agent=user_agent, + ): + response = await call_next(request) + status_code = response.status_code + except Exception: # pylint: disable=broad-except + # NOTE: The prometheus metrics middleware should be "outside" exception handling + # middleware. See https://fastapi.tiangolo.com/advanced/middleware/#adding-asgi-middlewares + status_code = status.HTTP_500_INTERNAL_SERVER_ERROR + raise + finally: + record_response_metrics( + metrics=self.metrics, + method=request.method, + endpoint=canonical_endpoint, + user_agent=user_agent, + http_status=status_code, + ) + + return response + + +def initialize_prometheus_instrumentation(app: FastAPI) -> None: + metrics = get_prometheus_metrics() + app.state.prometheus_metrics = metrics + app.add_middleware(PrometheusMiddleware, metrics=metrics) + + +def _startup(app: FastAPI) -> None: + @app.get("/metrics", include_in_schema=False) + async def metrics_endpoint(request: Request) -> Response: + prometheus_metrics = request.app.state.prometheus_metrics + assert isinstance(prometheus_metrics, PrometheusMetrics) # nosec + + content = await asyncio.get_event_loop().run_in_executor( + None, generate_latest, prometheus_metrics.registry + ) + + return Response(content=content, headers={"Content-Type": CONTENT_TYPE_LATEST}) + + +def _shutdown(app: FastAPI) -> None: + prometheus_metrics = app.state.prometheus_metrics + assert isinstance(prometheus_metrics, PrometheusMetrics) # nosec + registry = prometheus_metrics.registry + for collector in list(registry._collector_to_names.keys()): # noqa: SLF001 + registry.unregister(collector) + + +def setup_prometheus_instrumentation(app: FastAPI) -> CollectorRegistry: + initialize_prometheus_instrumentation(app) + + async def _on_startup() -> None: + _startup(app) + + def _on_shutdown() -> None: + _shutdown(app) + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) + + prometheus_metrics = app.state.prometheus_metrics + assert isinstance(prometheus_metrics, PrometheusMetrics) # nosec + + return prometheus_metrics.registry + + +_PROMETHEUS_INSTRUMENTATION_ENABLED: Final[str] = "prometheus_instrumentation_enabled" + + +def create_prometheus_instrumentationmain_input_state(*, enabled: bool) -> State: + return {_PROMETHEUS_INSTRUMENTATION_ENABLED: enabled} + + +async def prometheus_instrumentation_lifespan( + app: FastAPI, state: State +) -> AsyncIterator[State]: + # NOTE: requires ``initialize_prometheus_instrumentation`` to be called before the + # lifespan of the applicaiton runs, usually rigth after the ``FastAPI`` instance is created + + instrumentaiton_enabled = state.get(_PROMETHEUS_INSTRUMENTATION_ENABLED, False) + if instrumentaiton_enabled: + + _startup(app) + yield {} + if instrumentaiton_enabled: + _shutdown(app) diff --git a/packages/service-library/src/servicelib/fastapi/prometheus_instrumentation.py b/packages/service-library/src/servicelib/fastapi/prometheus_instrumentation.py deleted file mode 100644 index 65464e6955f..00000000000 --- a/packages/service-library/src/servicelib/fastapi/prometheus_instrumentation.py +++ /dev/null @@ -1,76 +0,0 @@ -# pylint: disable=protected-access - -from collections.abc import AsyncIterator -from typing import Final - -from fastapi import FastAPI -from fastapi_lifespan_manager import State -from prometheus_client import CollectorRegistry -from prometheus_fastapi_instrumentator import Instrumentator - - -def initialize_prometheus_instrumentation(app: FastAPI) -> None: - # NOTE: this cannot be ran once the application is started - - # NOTE: use that registry to prevent having a global one - app.state.prometheus_registry = registry = CollectorRegistry(auto_describe=True) - app.state.prometheus_instrumentator = Instrumentator( - should_instrument_requests_inprogress=False, # bug in https://github.com/trallnag/prometheus-fastapi-instrumentator/issues/317 - inprogress_labels=False, - registry=registry, - ) - app.state.prometheus_instrumentator.instrument(app) - - -def _startup(app: FastAPI) -> None: - assert isinstance(app.state.prometheus_instrumentator, Instrumentator) # nosec - app.state.prometheus_instrumentator.expose(app, include_in_schema=False) - - -def _shutdown(app: FastAPI) -> None: - assert isinstance(app.state.prometheus_registry, CollectorRegistry) # nosec - registry = app.state.prometheus_registry - for collector in list(registry._collector_to_names.keys()): # noqa: SLF001 - registry.unregister(collector) - - -def get_prometheus_instrumentator(app: FastAPI) -> Instrumentator: - assert isinstance(app.state.prometheus_instrumentator, Instrumentator) # nosec - return app.state.prometheus_instrumentator - - -def setup_prometheus_instrumentation(app: FastAPI) -> Instrumentator: - initialize_prometheus_instrumentation(app) - - async def _on_startup() -> None: - _startup(app) - - def _on_shutdown() -> None: - _shutdown(app) - - app.add_event_handler("startup", _on_startup) - app.add_event_handler("shutdown", _on_shutdown) - - return get_prometheus_instrumentator(app) - - -_PROMETHEUS_INSTRUMENTATION_ENABLED: Final[str] = "prometheus_instrumentation_enabled" - - -def create_prometheus_instrumentationmain_input_state(*, enabled: bool) -> State: - return {_PROMETHEUS_INSTRUMENTATION_ENABLED: enabled} - - -async def prometheus_instrumentation_lifespan( - app: FastAPI, state: State -) -> AsyncIterator[State]: - # NOTE: requires ``initialize_prometheus_instrumentation`` to be called before the - # lifespan of the applicaiton runs, usually rigth after the ``FastAPI`` instance is created - - instrumentaiton_enabled = state.get(_PROMETHEUS_INSTRUMENTATION_ENABLED, False) - if instrumentaiton_enabled: - - _startup(app) - yield {} - if instrumentaiton_enabled: - _shutdown(app) diff --git a/packages/service-library/src/servicelib/prometheus_metrics.py b/packages/service-library/src/servicelib/prometheus_metrics.py new file mode 100644 index 00000000000..0ee74115b8e --- /dev/null +++ b/packages/service-library/src/servicelib/prometheus_metrics.py @@ -0,0 +1,161 @@ +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from time import perf_counter + +from opentelemetry import trace +from prometheus_client import ( + Counter, + Gauge, + GCCollector, + Histogram, + PlatformCollector, + ProcessCollector, +) +from prometheus_client.registry import CollectorRegistry + +# +# CAUTION CAUTION CAUTION NOTE: +# Be very careful with metrics. pay attention to metrics cardinatity. +# Each time series takes about 3kb of overhead in Prometheus +# +# CAUTION: every unique combination of key-value label pairs represents a new time series +# +# If a metrics is not needed, don't add it!! It will collapse the application AND prometheus +# +# references: +# https://prometheus.io/docs/practices/naming/ +# https://www.robustperception.io/cardinality-is-key +# https://www.robustperception.io/why-does-prometheus-use-so-much-ram +# https://promcon.io/2019-munich/slides/containing-your-cardinality.pdf +# https://grafana.com/docs/grafana-cloud/how-do-i/control-prometheus-metrics-usage/usage-analysis-explore/ +# + + +@dataclass +class PrometheusMetrics: + registry: CollectorRegistry + process_collector: ProcessCollector + platform_collector: PlatformCollector + gc_collector: GCCollector + request_count: Counter + in_flight_requests: Gauge + response_latency_with_labels: Histogram + response_latency_detailed_buckets: Histogram + + +def _get_exemplar() -> dict[str, str] | None: + current_span = trace.get_current_span() + if not current_span.is_recording(): + return None + trace_id = trace.format_trace_id(current_span.get_span_context().trace_id) + return {"TraceID": trace_id} + + +def get_prometheus_metrics() -> PrometheusMetrics: + # app-scope registry + registry = CollectorRegistry(auto_describe=False) + + # automatically collects process metrics + process_collector = ProcessCollector(registry=registry) + # automatically collects python_info metrics + platform_collector = PlatformCollector(registry=registry) + # automatically collects python garbage collector metrics + gc_collector = GCCollector(registry=registry) + + # Total number of requests processed + request_count = Counter( + name="http_requests", + documentation="Total requests count", + labelnames=[ + "method", + "endpoint", + "http_status", + "simcore_user_agent", + ], + registry=registry, + ) + + in_flight_requests = Gauge( + name="http_in_flight_requests", + documentation="Number of requests in process", + labelnames=["method", "endpoint", "simcore_user_agent"], + registry=registry, + ) + + response_latency_with_labels = Histogram( + name="http_request_latency_seconds_with_labels", + documentation="Time processing a request with detailed labels", + labelnames=["method", "endpoint", "simcore_user_agent"], + registry=registry, + buckets=(0.1, 0.5, 1), + ) + + response_latency_detailed_buckets = Histogram( + name="http_request_latency_seconds_detailed_buckets", + documentation="Time processing a request with detailed buckets but no labels", + registry=registry, + buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10), + ) + + return PrometheusMetrics( + registry=registry, + process_collector=process_collector, + platform_collector=platform_collector, + gc_collector=gc_collector, + request_count=request_count, + in_flight_requests=in_flight_requests, + response_latency_with_labels=response_latency_with_labels, + response_latency_detailed_buckets=response_latency_detailed_buckets, + ) + + +@contextmanager +def record_request_metrics( + *, + metrics: PrometheusMetrics, + method: str, + endpoint: str, + user_agent: str, +) -> Iterator[None]: + """ + Context manager to record Prometheus metrics for a request. + + Args: + metrics (PrometheusMetrics): The Prometheus metrics instance. + app_name (str): The application name. + method (str): The HTTP method. + endpoint (str): The canonical endpoint. + user_agent (str): The user agent header value. + """ + + with metrics.in_flight_requests.labels( + method, endpoint, user_agent + ).track_inprogress(): + + start = perf_counter() + + yield + + amount = perf_counter() - start + exemplar = _get_exemplar() + metrics.response_latency_with_labels.labels( + method, endpoint, user_agent + ).observe(amount=amount, exemplar=exemplar) + metrics.response_latency_detailed_buckets.observe( + amount=amount, exemplar=exemplar + ) + + +def record_response_metrics( + *, + metrics: PrometheusMetrics, + method: str, + endpoint: str, + user_agent: str, + http_status: int, +) -> None: + exemplar = _get_exemplar() + metrics.request_count.labels(method, endpoint, http_status, user_agent).inc( + exemplar=exemplar + ) diff --git a/packages/service-library/tests/aiohttp/test_monitoring.py b/packages/service-library/tests/aiohttp/test_monitoring.py index 9c38cb4a3ec..e4691c61c9e 100644 --- a/packages/service-library/tests/aiohttp/test_monitoring.py +++ b/packages/service-library/tests/aiohttp/test_monitoring.py @@ -4,13 +4,14 @@ from asyncio import AbstractEventLoop -from typing import Any, Callable +from collections.abc import Callable +from typing import Any import pytest from aiohttp import web from aiohttp.test_utils import TestClient from faker import Faker -from prometheus_client.parser import text_string_to_metric_families +from prometheus_client.openmetrics.parser import text_string_to_metric_families from servicelib.aiohttp import status from servicelib.aiohttp.monitoring import setup_monitoring from servicelib.common_headers import ( @@ -37,7 +38,7 @@ async def monitored_request(request: web.Request) -> web.Response: for resource in app.router.resources(): print(resource) - setup_monitoring(app, app_name="pytest_app", version="0.0.1") + setup_monitoring(app, app_name="pytest_app") return event_loop.run_until_complete( aiohttp_client(app, server_kwargs={"port": ports[0]}) @@ -93,7 +94,6 @@ async def test_setup_monitoring(client: TestClient): metric_name="http_requests", sample_name="http_requests_total", labels={ - "app_name": "pytest_app", "endpoint": "/monitored_request", "http_status": "200", "method": "GET", @@ -107,7 +107,6 @@ async def test_setup_monitoring(client: TestClient): metric_name="http_requests", sample_name="http_requests_total", labels={ - "app_name": "pytest_app", "endpoint": "/metrics", "http_status": "200", "method": "GET", @@ -133,7 +132,6 @@ async def test_request_with_simcore_user_agent(client: TestClient, faker: Faker) metric_name="http_requests", sample_name="http_requests_total", labels={ - "app_name": "pytest_app", "endpoint": "/monitored_request", "http_status": "200", "method": "GET", diff --git a/packages/service-library/tests/aiohttp/test_monitoring_middleware.py b/packages/service-library/tests/aiohttp/test_monitoring_middleware.py new file mode 100644 index 00000000000..7c081a508ab --- /dev/null +++ b/packages/service-library/tests/aiohttp/test_monitoring_middleware.py @@ -0,0 +1,32 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +import pytest +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer +from prometheus_client.openmetrics.exposition import ( + CONTENT_TYPE_LATEST, +) +from servicelib.aiohttp.monitoring import setup_monitoring + + +@pytest.fixture +def aiohttp_app_with_monitoring(): + app = web.Application() + setup_monitoring(app, app_name="test_app") + return app + + +@pytest.fixture +async def client(aiohttp_app_with_monitoring): + async with TestServer(aiohttp_app_with_monitoring) as server: + async with TestClient(server) as client: + yield client + + +async def test_metrics_endpoint(client): + response = await client.get("/metrics") + assert response.status == 200 + assert response.headers["Content-Type"] == CONTENT_TYPE_LATEST + body = await response.text() + assert "# HELP" in body # Check for Prometheus metrics format diff --git a/packages/service-library/tests/fastapi/test_prometheus_middleware.py b/packages/service-library/tests/fastapi/test_prometheus_middleware.py new file mode 100644 index 00000000000..9d67cc8ee0d --- /dev/null +++ b/packages/service-library/tests/fastapi/test_prometheus_middleware.py @@ -0,0 +1,31 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +from collections.abc import AsyncIterable + +import pytest +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from httpx import AsyncClient +from prometheus_client.openmetrics.exposition import CONTENT_TYPE_LATEST +from servicelib.fastapi.monitoring import setup_prometheus_instrumentation + + +@pytest.fixture +async def app(app: FastAPI) -> AsyncIterable[FastAPI]: + """ + Fixture that sets up the Prometheus middleware in the FastAPI app. + """ + setup_prometheus_instrumentation(app) + async with LifespanManager(app): + yield app + + +async def test_metrics_endpoint(client: AsyncClient, app: FastAPI): + """ + Test that the /metrics endpoint is available and returns Prometheus metrics. + """ + response = await client.get("/metrics") + assert response.status_code == 200 + assert response.headers["Content-Type"] == CONTENT_TYPE_LATEST + assert "# HELP" in response.text + assert "# TYPE" in response.text diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index 11c43e63206..0ba4f5c3625 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -249,10 +249,6 @@ packaging==24.2 pamqp==3.3.0 # via aiormq prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.0 # via @@ -412,7 +408,6 @@ starlette==0.46.0 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via -r requirements/../../../packages/service-library/requirements/_base.in tenacity==9.0.0 diff --git a/services/agent/src/simcore_service_agent/services/instrumentation/_setup.py b/services/agent/src/simcore_service_agent/services/instrumentation/_setup.py index ad4e2f3cf2e..0f57e5be288 100644 --- a/services/agent/src/simcore_service_agent/services/instrumentation/_setup.py +++ b/services/agent/src/simcore_service_agent/services/instrumentation/_setup.py @@ -1,5 +1,5 @@ from fastapi import FastAPI -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation, ) from simcore_service_agent.core.settings import ApplicationSettings @@ -12,12 +12,10 @@ def setup_instrumentation(app: FastAPI) -> None: if not settings.AGENT_PROMETHEUS_INSTRUMENTATION_ENABLED: return - instrumentator = setup_prometheus_instrumentation(app) + registry = setup_prometheus_instrumentation(app) async def on_startup() -> None: - app.state.instrumentation = AgentInstrumentation( - registry=instrumentator.registry - ) + app.state.instrumentation = AgentInstrumentation(registry=registry) app.add_event_handler("startup", on_startup) diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index 5a7695ce29a..bc6e9f8d491 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -491,10 +491,6 @@ pint==0.24.4 platformdirs==4.3.6 # via pint prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.2.1 # via @@ -811,7 +807,6 @@ starlette==0.41.3 # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via # -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/api-server/src/simcore_service_api_server/core/_prometheus_instrumentation.py b/services/api-server/src/simcore_service_api_server/core/_prometheus_instrumentation.py index 4085561cdfc..f19bac34a76 100644 --- a/services/api-server/src/simcore_service_api_server/core/_prometheus_instrumentation.py +++ b/services/api-server/src/simcore_service_api_server/core/_prometheus_instrumentation.py @@ -9,7 +9,7 @@ from pydantic import PositiveInt from servicelib.async_utils import cancel_wait_task from servicelib.background_task import create_periodic_task -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation as setup_rest_instrumentation, ) from servicelib.logging_utils import log_catch @@ -66,11 +66,11 @@ async def _collect_prometheus_metrics_task(app: FastAPI): def setup_prometheus_instrumentation(app: FastAPI): - instrumentator = setup_rest_instrumentation(app) + registry = setup_rest_instrumentation(app) async def on_startup() -> None: app.state.instrumentation = ApiServerPrometheusInstrumentation( - registry=instrumentator.registry + registry=registry ) await wait_till_log_distributor_ready(app) app.state.instrumentation_task = create_periodic_task( diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index 9bee8dd0ac4..bc319bb09a0 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -87,6 +87,9 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI: setup_rabbitmq(app) + if app.state.settings.API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED: + setup_prometheus_instrumentation(app) + if settings.API_SERVER_TRACING: initialize_tracing(app, settings.API_SERVER_TRACING, APP_NAME) @@ -115,15 +118,12 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI: app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) - exceptions.setup_exception_handlers( - app, is_debug=settings.SC_BOOT_MODE == BootModeEnum.DEBUG - ) - if settings.API_SERVER_PROFILING: initialize_profiler(app) - if app.state.settings.API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED: - setup_prometheus_instrumentation(app) + exceptions.setup_exception_handlers( + app, is_debug=settings.SC_BOOT_MODE == BootModeEnum.DEBUG + ) # routing diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index 3e76117d00d..f676fe27b72 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -437,10 +437,6 @@ partd==1.4.2 # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.2.1 # via @@ -719,7 +715,6 @@ starlette==0.41.3 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/instrumentation/_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/instrumentation/_core.py index e3bc20ef518..9de65aac078 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/instrumentation/_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/instrumentation/_core.py @@ -1,7 +1,7 @@ from typing import cast from fastapi import FastAPI -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation as setup_rest_instrumentation, ) @@ -16,7 +16,7 @@ def setup(app: FastAPI) -> None: return # NOTE: this must be setup before application startup - instrumentator = setup_rest_instrumentation(app) + registry = setup_rest_instrumentation(app) async def on_startup() -> None: metrics_subsystem = ( @@ -24,12 +24,11 @@ async def on_startup() -> None: ) app.state.instrumentation = ( AutoscalingInstrumentation( # pylint: disable=unexpected-keyword-arg - registry=instrumentator.registry, subsystem=metrics_subsystem + registry=registry, subsystem=metrics_subsystem ) ) - async def on_shutdown() -> None: - ... + async def on_shutdown() -> None: ... app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index 1ec6f75dac7..94e60b6dc27 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -324,10 +324,6 @@ packaging==24.2 pamqp==3.3.0 # via aiormq prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.0 # via @@ -535,7 +531,6 @@ starlette==0.46.1 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via -r requirements/../../../packages/service-library/requirements/_base.in tenacity==9.0.0 diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index f7161d6ce87..0ff4deb86d8 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -4,10 +4,10 @@ from fastapi.middleware.gzip import GZipMiddleware from models_library.basic_types import BootModeEnum from servicelib.fastapi import timing_middleware -from servicelib.fastapi.openapi import override_fastapi_openapi_method -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( initialize_prometheus_instrumentation, ) +from servicelib.fastapi.openapi import override_fastapi_openapi_method from servicelib.fastapi.tracing import initialize_tracing from starlette.middleware.base import BaseHTTPMiddleware @@ -63,11 +63,10 @@ def create_app() -> FastAPI: app.state.settings = settings # MIDDLEWARES - if settings.CATALOG_TRACING: - initialize_tracing(app, settings.CATALOG_TRACING, APP_NAME) - if settings.CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED: initialize_prometheus_instrumentation(app) + if settings.CATALOG_TRACING: + initialize_tracing(app, settings.CATALOG_TRACING, APP_NAME) if settings.SC_BOOT_MODE != BootModeEnum.PRODUCTION: # middleware to time requests (ONLY for development) diff --git a/services/catalog/src/simcore_service_catalog/core/events.py b/services/catalog/src/simcore_service_catalog/core/events.py index 57089d9fc5e..8695b10f15e 100644 --- a/services/catalog/src/simcore_service_catalog/core/events.py +++ b/services/catalog/src/simcore_service_catalog/core/events.py @@ -3,13 +3,13 @@ from fastapi import FastAPI from fastapi_lifespan_manager import LifespanManager, State -from servicelib.fastapi.postgres_lifespan import ( - create_postgres_database_input_state, -) -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( create_prometheus_instrumentationmain_input_state, prometheus_instrumentation_lifespan, ) +from servicelib.fastapi.postgres_lifespan import ( + create_postgres_database_input_state, +) from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..api.rpc.events import rpc_api_lifespan diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index fbdebe2d6fb..b8d25923fc3 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -435,10 +435,6 @@ partd==1.4.2 # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.2.1 # via @@ -717,7 +713,6 @@ starlette==0.41.3 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py index ad354a2c8b1..120bfc873cb 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py @@ -1,7 +1,7 @@ import logging from fastapi import FastAPI -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation, ) from servicelib.fastapi.tracing import initialize_tracing diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index fe75179d845..352e42abae5 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -100,7 +100,6 @@ fastapi==0.115.5 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # fastapi-lifespan-manager - # prometheus-fastapi-instrumentator fastapi-lifespan-manager==0.1.4 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in fastapi-pagination==0.12.31 @@ -267,10 +266,6 @@ orjson==3.10.0 pamqp==3.3.0 # via aiormq prometheus-client==0.20.0 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.1 # via diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py index 5ecac86b882..fab6b65d746 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py @@ -6,10 +6,10 @@ from fastapi_pagination import add_pagination from servicelib.fastapi import timing_middleware from servicelib.fastapi.http_error import set_app_default_http_error_handlers -from servicelib.fastapi.openapi import override_fastapi_openapi_method -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation, ) +from servicelib.fastapi.openapi import override_fastapi_openapi_method from servicelib.fastapi.tracing import initialize_tracing from starlette.middleware.base import BaseHTTPMiddleware diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index f048b5f73af..ec073478903 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -556,10 +556,6 @@ pint==0.24.4 platformdirs==4.3.6 # via pint prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.0 # via @@ -973,7 +969,6 @@ starlette==0.41.3 # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via # -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 17835676cad..dd04fb355d3 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -145,6 +145,9 @@ def init_app(settings: AppSettings | None = None) -> FastAPI: substitutions.setup(app) + if settings.DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED: + instrumentation.setup(app) + if settings.DIRECTOR_V2_TRACING: initialize_tracing(app, settings.DIRECTOR_V2_TRACING, APP_NAME) @@ -202,9 +205,6 @@ def init_app(settings: AppSettings | None = None) -> FastAPI: resource_usage_tracker_client.setup(app) - if settings.DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED: - instrumentation.setup(app) - if settings.DIRECTOR_V2_PROFILING: initialize_profiler(app) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_setup.py b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_setup.py index 889cb39a460..bfc02835eac 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_setup.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_setup.py @@ -1,7 +1,7 @@ from typing import cast from fastapi import FastAPI -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation, ) @@ -10,12 +10,10 @@ def setup(app: FastAPI) -> None: - instrumentator = setup_prometheus_instrumentation(app) + registry = setup_prometheus_instrumentation(app) async def on_startup() -> None: - app.state.instrumentation = DirectorV2Instrumentation( - registry=instrumentator.registry - ) + app.state.instrumentation = DirectorV2Instrumentation(registry=registry) app.add_event_handler("startup", on_startup) diff --git a/services/director/requirements/_base.txt b/services/director/requirements/_base.txt index 92c6bdb2a1e..1b6c62bc819 100644 --- a/services/director/requirements/_base.txt +++ b/services/director/requirements/_base.txt @@ -96,7 +96,6 @@ fastapi==0.115.5 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # fastapi-lifespan-manager - # prometheus-fastapi-instrumentator fastapi-cli==0.0.5 # via fastapi fastapi-lifespan-manager==0.1.4 @@ -286,9 +285,6 @@ prometheus-client==0.21.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==6.1.0 - # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.2.0 # via # aiohttp diff --git a/services/director/src/simcore_service_director/core/application.py b/services/director/src/simcore_service_director/core/application.py index 24716583b92..f9ff1a0391d 100644 --- a/services/director/src/simcore_service_director/core/application.py +++ b/services/director/src/simcore_service_director/core/application.py @@ -50,6 +50,8 @@ def create_app(settings: ApplicationSettings) -> FastAPI: # PLUGINS SETUP setup_api_routes(app) + setup_instrumentation(app) + if app.state.settings.DIRECTOR_TRACING: initialize_tracing(app, app.state.settings.DIRECTOR_TRACING, APP_NAME) @@ -60,8 +62,6 @@ def create_app(settings: ApplicationSettings) -> FastAPI: ) setup_registry(app) - setup_instrumentation(app) - # ERROR HANDLERS # EVENTS diff --git a/services/director/src/simcore_service_director/instrumentation.py b/services/director/src/simcore_service_director/instrumentation.py index cb63d5f35f5..40ee74d831a 100644 --- a/services/director/src/simcore_service_director/instrumentation.py +++ b/services/director/src/simcore_service_director/instrumentation.py @@ -3,7 +3,7 @@ from fastapi import FastAPI from prometheus_client import CollectorRegistry, Counter -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation, ) from servicelib.instrumentation import MetricsBase, get_metrics_namespace @@ -59,16 +59,15 @@ def setup(app: FastAPI) -> None: return # NOTE: this must be setup before application startup - instrumentator = setup_prometheus_instrumentation(app) + registry = setup_prometheus_instrumentation(app) async def on_startup() -> None: metrics_subsystem = "" app.state.instrumentation = DirectorV0Instrumentation( - registry=instrumentator.registry, subsystem=metrics_subsystem + registry=registry, subsystem=metrics_subsystem ) - async def on_shutdown() -> None: - ... + async def on_shutdown() -> None: ... app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 6b3c54c64f8..c45cc6acbdd 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -331,10 +331,6 @@ packaging==24.2 pamqp==3.3.0 # via aiormq prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.0 # via @@ -547,7 +543,6 @@ starlette==0.46.0 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via -r requirements/../../../packages/service-library/requirements/_base.in tenacity==9.0.0 diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py index e3996390eba..83d361a2155 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py @@ -1,9 +1,9 @@ from fastapi import FastAPI -from servicelib.fastapi.openapi import override_fastapi_openapi_method -from servicelib.fastapi.profiler import initialize_profiler -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( initialize_prometheus_instrumentation, ) +from servicelib.fastapi.openapi import override_fastapi_openapi_method +from servicelib.fastapi.profiler import initialize_profiler from servicelib.fastapi.tracing import initialize_tracing from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py index 341b82eeae4..14745c70731 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py @@ -6,13 +6,13 @@ create_remote_docker_client_input_state, remote_docker_client_lifespan, ) -from servicelib.fastapi.postgres_lifespan import ( - create_postgres_database_input_state, -) -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( create_prometheus_instrumentationmain_input_state, prometheus_instrumentation_lifespan, ) +from servicelib.fastapi.postgres_lifespan import ( + create_postgres_database_input_state, +) from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..api.rpc.routes import rpc_api_routes_lifespan diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index c74f2aa223e..15d632d8ed3 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -417,10 +417,6 @@ pint==0.24.4 platformdirs==4.3.6 # via pint prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.0 # via @@ -760,7 +756,6 @@ starlette==0.46.0 # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via # -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index d266090b7c7..e1a3efbccee 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -155,7 +155,6 @@ fastapi==0.115.5 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # fastapi-lifespan-manager - # prometheus-fastapi-instrumentator fastapi-lifespan-manager==0.1.4 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in faststream==0.5.31 @@ -412,10 +411,6 @@ packaging==24.1 pamqp==3.3.0 # via aiormq prometheus-client==0.21.0 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.1 # via diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt index 7782b5b9799..159a328732c 100644 --- a/services/invitations/requirements/_base.txt +++ b/services/invitations/requirements/_base.txt @@ -267,10 +267,6 @@ packaging==24.2 pamqp==3.3.0 # via aiormq prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.0 # via @@ -434,7 +430,6 @@ starlette==0.46.1 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via -r requirements/../../../packages/service-library/requirements/_base.in tenacity==9.0.0 diff --git a/services/invitations/src/simcore_service_invitations/core/application.py b/services/invitations/src/simcore_service_invitations/core/application.py index b3f96c07a82..02e0cac84d4 100644 --- a/services/invitations/src/simcore_service_invitations/core/application.py +++ b/services/invitations/src/simcore_service_invitations/core/application.py @@ -1,8 +1,8 @@ from fastapi import FastAPI -from servicelib.fastapi.openapi import override_fastapi_openapi_method -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation, ) +from servicelib.fastapi.openapi import override_fastapi_openapi_method from servicelib.fastapi.tracing import initialize_tracing from .._meta import ( diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index 8b48b47bfb6..ba3090c531e 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -286,10 +286,6 @@ packaging==24.2 pamqp==3.3.0 # via aiormq prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.1 # via @@ -484,7 +480,6 @@ starlette==0.46.1 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via -r requirements/../../../packages/service-library/requirements/_base.in tenacity==9.0.0 diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index 3b7cfe92c26..9c1465c9a02 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -1,13 +1,13 @@ import logging from fastapi import FastAPI +from servicelib.fastapi.monitoring import ( + initialize_prometheus_instrumentation, +) from servicelib.fastapi.openapi import ( get_common_oas_options, override_fastapi_openapi_method, ) -from servicelib.fastapi.prometheus_instrumentation import ( - initialize_prometheus_instrumentation, -) from servicelib.fastapi.tracing import initialize_tracing from servicelib.logging_utils import config_all_loggers diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index 65a828744e5..879582575c0 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -2,14 +2,14 @@ from fastapi import FastAPI from fastapi_lifespan_manager import LifespanManager, State +from servicelib.fastapi.monitoring import ( + create_prometheus_instrumentationmain_input_state, + prometheus_instrumentation_lifespan, +) from servicelib.fastapi.postgres_lifespan import ( create_postgres_database_input_state, postgres_database_lifespan, ) -from servicelib.fastapi.prometheus_instrumentation import ( - create_prometheus_instrumentationmain_input_state, - prometheus_instrumentation_lifespan, -) from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..api.rpc.routing import rpc_api_routes_lifespan diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index db10f20ca98..a8e58f5f76a 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -335,10 +335,6 @@ packaging==24.2 pamqp==3.3.0 # via aiormq prometheus-client==0.21.0 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.2.0 # via @@ -572,7 +568,6 @@ starlette==0.41.3 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via -r requirements/../../../packages/service-library/requirements/_base.in tenacity==9.0.0 diff --git a/services/payments/src/simcore_service_payments/core/application.py b/services/payments/src/simcore_service_payments/core/application.py index 6bfaf9b2dec..eb8169f2a80 100644 --- a/services/payments/src/simcore_service_payments/core/application.py +++ b/services/payments/src/simcore_service_payments/core/application.py @@ -1,8 +1,8 @@ from fastapi import FastAPI -from servicelib.fastapi.openapi import override_fastapi_openapi_method -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation, ) +from servicelib.fastapi.openapi import override_fastapi_openapi_method from servicelib.fastapi.tracing import initialize_tracing from .._meta import ( diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index 3b266c6cbe7..d37393ff3f3 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -163,7 +163,6 @@ fastapi==0.115.5 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # fastapi-lifespan-manager - # prometheus-fastapi-instrumentator fastapi-lifespan-manager==0.1.4 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in faststream==0.5.31 @@ -444,10 +443,6 @@ pillow==10.2.0 prometheus-api-client==0.5.5 # via -r requirements/_base.in prometheus-client==0.20.0 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in propcache==0.3.1 # via diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index fd9e7abdca0..c220d9d81fb 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -488,10 +488,6 @@ packaging==24.2 pamqp==3.3.0 # via aiormq prometheus-client==0.21.1 - # via - # -r requirements/../../../packages/service-library/requirements/_fastapi.in - # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==7.0.2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in prompt-toolkit==3.0.50 # via click-repl @@ -827,7 +823,6 @@ starlette==0.45.3 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi - # prometheus-fastapi-instrumentator stream-zip==0.0.83 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index b70c2cef302..4ef671e2a13 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -12,11 +12,11 @@ from servicelib.fastapi import timing_middleware from servicelib.fastapi.cancellation_middleware import RequestCancellationMiddleware from servicelib.fastapi.client_session import setup_client_session -from servicelib.fastapi.openapi import override_fastapi_openapi_method -from servicelib.fastapi.profiler import ProfilerMiddleware -from servicelib.fastapi.prometheus_instrumentation import ( +from servicelib.fastapi.monitoring import ( setup_prometheus_instrumentation, ) +from servicelib.fastapi.openapi import override_fastapi_openapi_method +from servicelib.fastapi.profiler import ProfilerMiddleware from servicelib.fastapi.tracing import initialize_tracing from starlette.middleware.base import BaseHTTPMiddleware @@ -111,10 +111,10 @@ def create_app(settings: ApplicationSettings) -> FastAPI: # noqa: C901 app.add_middleware(RequestCancellationMiddleware) - if settings.STORAGE_TRACING: - initialize_tracing(app, settings.STORAGE_TRACING, APP_NAME) if settings.STORAGE_MONITORING_ENABLED: setup_prometheus_instrumentation(app) + if settings.STORAGE_TRACING: + initialize_tracing(app, settings.STORAGE_TRACING, APP_NAME) async def _on_startup() -> None: if settings.STORAGE_WORKER_MODE: diff --git a/services/web/server/src/simcore_service_webserver/diagnostics/_monitoring.py b/services/web/server/src/simcore_service_webserver/diagnostics/_monitoring.py index 6ab317e73d3..dd7d169462d 100644 --- a/services/web/server/src/simcore_service_webserver/diagnostics/_monitoring.py +++ b/services/web/server/src/simcore_service_webserver/diagnostics/_monitoring.py @@ -1,6 +1,5 @@ -""" Enables monitoring of some quantities needed for diagnostics +"""Enables monitoring of some quantities needed for diagnostics""" -""" import logging import time @@ -53,7 +52,6 @@ def setup_monitoring(app: web.Application): _meta.APP_NAME, enter_middleware_cb=enter_middleware_cb, exit_middleware_cb=exit_middleware_cb, - version=f"{_meta.VERSION}", ) monitor_services.add_instrumentation( diff --git a/services/web/server/tests/unit/isolated/test_tracing.py b/services/web/server/tests/unit/isolated/test_tracing.py index c356a31053c..88eec9626a0 100644 --- a/services/web/server/tests/unit/isolated/test_tracing.py +++ b/services/web/server/tests/unit/isolated/test_tracing.py @@ -35,8 +35,4 @@ def test_middleware_restrictions_opentelemetry_is_second_middleware( app = create_application() assert app.middlewares - assert ( - app.middlewares[0].__middleware_name__ - == "servicelib.aiohttp.monitoring.monitor_simcore_service_webserver" - ) - assert app.middlewares[1] is aiohttp_opentelemetry_middleware + assert app.middlewares[0] is aiohttp_opentelemetry_middleware