From a4270170a0452101b468641f9c5ce6e206f8fa9c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 1 Dec 2020 17:46:42 +0100 Subject: [PATCH 1/5] GC will shuffle data before emoving it --- .../resource_manager/garbage_collector.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py index 2420d0f54f7..2baca32ac48 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py @@ -1,5 +1,6 @@ import asyncio import logging +import random from contextlib import suppress from itertools import chain from typing import Dict, List, Tuple @@ -195,7 +196,11 @@ async def remove_disconnected_user_resources( dead_key_resources, ) - for resource_name, resource_value in dead_key_resources.items(): + # shuffle order of guest users to allow for random errors to not stop + # the removal of "not erroring" services + dead_key_resources_items = dead_key_resources.items() + random.shuffle(dead_key_resources_items) + for resource_name, resource_value in dead_key_resources_items: # Releasing a resource consists of two steps # - (1) release actual resource (e.g. stop service, close project, deallocate memory, etc) @@ -287,6 +292,9 @@ async def remove_users_manually_marked_as_guests( guest_users: List[Tuple[int, str]] = await get_guest_user_ids_and_names(app) logger.info("GUEST user candidates to clean %s", guest_users) + # shuffle order of guest users to allow for random errors to not stop + # the removal of "not erroring" services + random.shuffle(guest_users) for guest_user_id, guest_user_name in guest_users: if guest_user_id in user_ids_to_ignore: logger.info( From e749338c3b21631f3a46840e76a71b0bf3da75a5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 1 Dec 2020 17:57:47 +0100 Subject: [PATCH 2/5] trying to unstuck CI --- .../resource_manager/garbage_collector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py index 2baca32ac48..90cb5e9f8c9 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py @@ -610,3 +610,4 @@ async def remove_user(app: web.Application, user_id: int) -> None: logger.warning( "User '%s' still has some projects, could not be deleted", user_id ) + From b4188e16dcf06252ff232e4cb2fd281fd36a364c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 1 Dec 2020 17:57:58 +0100 Subject: [PATCH 3/5] reverting --- .../resource_manager/garbage_collector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py index 90cb5e9f8c9..2baca32ac48 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py @@ -610,4 +610,3 @@ async def remove_user(app: web.Application, user_id: int) -> None: logger.warning( "User '%s' still has some projects, could not be deleted", user_id ) - From a214a6d33c3f5952c136a88a8f716e7ea15c5c48 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 3 Dec 2020 15:17:33 +0100 Subject: [PATCH 4/5] instead of shuffling the error is trapped and logged --- .../resource_manager/garbage_collector.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py index 2baca32ac48..a19cc9a86e2 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py @@ -196,11 +196,7 @@ async def remove_disconnected_user_resources( dead_key_resources, ) - # shuffle order of guest users to allow for random errors to not stop - # the removal of "not erroring" services - dead_key_resources_items = dead_key_resources.items() - random.shuffle(dead_key_resources_items) - for resource_name, resource_value in dead_key_resources_items: + for resource_name, resource_value in dead_key_resources.items(): # Releasing a resource consists of two steps # - (1) release actual resource (e.g. stop service, close project, deallocate memory, etc) @@ -292,9 +288,6 @@ async def remove_users_manually_marked_as_guests( guest_users: List[Tuple[int, str]] = await get_guest_user_ids_and_names(app) logger.info("GUEST user candidates to clean %s", guest_users) - # shuffle order of guest users to allow for random errors to not stop - # the removal of "not erroring" services - random.shuffle(guest_users) for guest_user_id, guest_user_name in guest_users: if guest_user_id in user_ids_to_ignore: logger.info( @@ -380,8 +373,15 @@ async def remove_guest_user_with_all_its_resources( logger.debug("User is not GUEST, skipping cleanup") return - await remove_all_projects_for_user(app=app, user_id=user_id) - await remove_user(app=app, user_id=user_id) + try: + await remove_all_projects_for_user(app=app, user_id=user_id) + await remove_user(app=app, user_id=user_id) + except Exception as e: # pylint: disable=broad-except + logger.warning("%s", e) + logger.warning( + "Could not remove GUEST with id=%s. Check the logs above for details", + user_id, + ) async def remove_all_projects_for_user(app: web.Application, user_id: int) -> None: From 31414d7b3183734db5c038179a1bd3481a7c9df1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 3 Dec 2020 15:18:07 +0100 Subject: [PATCH 5/5] removing unused import --- .../resource_manager/garbage_collector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py index a19cc9a86e2..b9051470faa 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py @@ -1,6 +1,5 @@ import asyncio import logging -import random from contextlib import suppress from itertools import chain from typing import Dict, List, Tuple