Logged unhealth decision

Author: Pedro Crespo

services/web/server/src/simcore_service_webserver/diagnostics.py

@@ -1,32 +1,34 @@
 
+import logging
 import os
-from typing import List
+from typing import List, Optional
 
 import attr
 from aiohttp import web
-import logging
 
 from servicelib import monitor_slow_callbacks
 
 log = logging.getLogger(__name__)
 
 INCIDENTS_REGISTRY_KEY = f"{__name__}.registry"
 
-AIODEBUG_SLOW_DURATION_SECS = float(os.environ.get("AIODEBUG_SLOW_DURATION_SECS", 0.1))
+AIODEBUG_SLOW_DURATION_SECS = float(os.environ.get("AIODEBUG_SLOW_DURATION_SECS", 0.2))
 MAX_DELAY_SECS_ALLOWED = 300 * AIODEBUG_SLOW_DURATION_SECS
 
 @attr.s(auto_attribs=True)
 class IncidentsRegistry:
+    # FIXME: this needs a limit to keep worst cases?
     slow_callbaks: List[monitor_slow_callbacks.Incident]
 
-    @property
-    def max_delay(self) -> float:
+    def eval_max_delay(self) -> float:
         return max( incident.delay_secs for incident in self.slow_callbaks )
 
 
 
-def setup_diagnostics(app: web.Application):
+def setup_diagnostics(app: web.Application, *, max_delay_allowed: Optional[float]=None):
     # NOTE: Every task blocking > AIODEBUG_SLOW_DURATION_SECS secs is considered slow and logged as warning
-    incidents = monitor_slow_callbacks.enable(MAX_DELAY_SECS_ALLOWED)
+    if max_delay_allowed is None:
+        max_delay_allowed = MAX_DELAY_SECS_ALLOWED
+    incidents = monitor_slow_callbacks.enable(max_delay_allowed)
 
     app[INCIDENTS_REGISTRY_KEY] = IncidentsRegistry(incidents)

services/web/server/src/simcore_service_webserver/rest_handlers.py

@@ -4,6 +4,7 @@
 """
 import asyncio
 from typing import Optional
+import logging
 
 from aiohttp import web
 
@@ -20,12 +21,21 @@
 from .utils import get_task_info, get_tracemalloc_info
 
 
+log = logging.getLogger(__name__)
+
+
 async def check_health(request: web.Request):
 
     # diagnostics of incidents
     incidents: Optional[IncidentsRegistry] = request.app.get(INCIDENTS_REGISTRY_KEY)
     if incidents:
-        if incidents.max_delay > MAX_DELAY_SECS_ALLOWED:
+        max_delay: float = incidents.eval_max_delay()
+        if max_delay > MAX_DELAY_SECS_ALLOWED:
+            log.error(
+                "Unhealthy service: %s secs delay [%s secs allowed]",
+                max_delay,
+                MAX_DELAY_SECS_ALLOWED,
+            )
             raise web.HTTPServiceUnavailable()
 
     data = {