⚗️ Added debug only env var to sleep after removing user services ⚠️ (#6042)

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

.env-devel

@@ -82,6 +82,7 @@ COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=PRESIGNED
 COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=PRESIGNED
 DIRECTOR_V2_DEV_FEATURES_ENABLED=0
 DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED=1
+DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL=0
 DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 DIRECTOR_V2_HOST=director-v2
 DIRECTOR_V2_LOGLEVEL=WARNING

services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py

@@ -30,7 +30,9 @@ class DynamicServicesSchedulerSettings(BaseCustomSettings):
     )
 
     DYNAMIC_SIDECAR_DOCKER_COMPOSE_VERSION: str = Field(
-        "3.8", description="docker-compose spec version used in the compose-specs", deprecated=True
+        "3.8",
+        description="docker-compose spec version used in the compose-specs",
+        deprecated=True,
     )
 
     DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS: bool = Field(
@@ -61,7 +63,9 @@ class DynamicServicesSchedulerSettings(BaseCustomSettings):
         description="Prometheus will scrape service placed on these networks",
     )
 
-    DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED: bool = Field(
+    DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED: (
+        bool
+    ) = Field(
         default=True,
         description=(
             "when the message indicating there are no more credits left in a wallet "
@@ -149,3 +153,11 @@ class DynamicServicesSchedulerSettings(BaseCustomSettings):
             "allow for some time to pass before declaring it failed."
         ),
     )
+
+    #
+    # DEBUG
+    #
+
+    DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL: timedelta = Field(
+        timedelta(0), description="time to sleep before removing a container"
+    )

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py

@@ -1,5 +1,6 @@
 # pylint: disable=relative-beyond-top-level
 
+import asyncio
 import json
 import logging
 from typing import Any, cast
@@ -276,6 +277,11 @@ async def attempt_pod_removal_and_data_saving(
 
     await service_remove_containers(app, scheduler_data.node_uuid, sidecars_client)
 
+    # used for debuug, normally sleeps 0
+    await asyncio.sleep(
+        settings.DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL.total_seconds()
+    )
+
     # only try to save the status if :
     # - it is requested to save the state
     # - the dynamic-sidecar has finished booting correctly

services/docker-compose.yml

@@ -281,6 +281,7 @@ services:
       DIRECTOR_V2_PUBLIC_API_BASE_URL: ${DIRECTOR_V2_PUBLIC_API_BASE_URL}
       DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: ${DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS}
       DIRECTOR_V2_PROFILING: ${DIRECTOR_V2_PROFILING}
+      DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL: ${DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL}
 
       DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED: ${DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED}
       DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS: ${DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS}