♻️ Autoscaling: refactor before 1st draft of computational autoscaling (⚠️ devops) (#4874)

Author: sanderegg

.coveragerc

@@ -24,7 +24,7 @@ exclude_lines =
     if __name__ == .__main__.:
 
     # Don't complain about abstract methods, they aren't run:
-    @(abc\.)?abstractmethod
+    @(abc\.)?abstract(((class|static)?method)|property)
 
 ignore_errors = True
 show_missing = True

packages/pytest-simcore/src/pytest_simcore/dask_gateway.py

@@ -10,6 +10,7 @@
 from dask_gateway_server.app import DaskGateway
 from dask_gateway_server.backends.local import UnsafeLocalBackend
 from distributed import Client
+from faker import Faker
 
 
 @pytest.fixture
@@ -70,17 +71,31 @@ async def local_dask_gateway_server(
     print("...done")
 
 
+@pytest.fixture
+def gateway_username(faker: Faker) -> str:
+    return faker.user_name()
+
+
+@pytest.fixture
+def gateway_auth(
+    local_dask_gateway_server: DaskGatewayServer, gateway_username: str
+) -> auth.BasicAuth:
+    return auth.BasicAuth(gateway_username, local_dask_gateway_server.password)
+
+
 @pytest.fixture
 async def dask_gateway(
-    local_dask_gateway_server: DaskGatewayServer,
+    local_dask_gateway_server: DaskGatewayServer, gateway_auth: auth.BasicAuth
 ) -> Gateway:
     async with Gateway(
         local_dask_gateway_server.address,
         local_dask_gateway_server.proxy_address,
         asynchronous=True,
-        auth=auth.BasicAuth("pytest_user", local_dask_gateway_server.password),
+        auth=gateway_auth,
     ) as gateway:
-        print(f"--> {gateway=} created")
+        print(
+            f"--> {gateway=} created, with {gateway_auth.username=}/{gateway_auth.password=}"
+        )
         cluster_options = await gateway.cluster_options()
         gateway_versions = await gateway.get_versions()
         clusters_list = await gateway.list_clusters()

packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py

@@ -3,8 +3,8 @@
 # pylint: disable=unused-variable
 
 
-from collections.abc import AsyncIterable, Callable
-from typing import Any, AsyncIterator
+from collections.abc import AsyncIterable, AsyncIterator, Callable
+from typing import Any
 
 import distributed
 import pytest

packages/service-library/src/servicelib/utils_formatting.py

@@ -0,0 +1,10 @@
+import datetime
+
+_TIME_FORMAT = "{:02d}:{:02d}"  # format for minutes:seconds
+
+
+def timedelta_as_minute_second(delta: datetime.timedelta) -> str:
+    total_seconds = round(delta.total_seconds())
+    minutes, seconds = divmod(abs(total_seconds), 60)
+    sign = "-" if total_seconds < 0 else ""
+    return f"{sign}{_TIME_FORMAT.format(minutes, seconds)}"

packages/service-library/tests/test_utils_formatting.py

@@ -0,0 +1,20 @@
+from datetime import timedelta
+
+import pytest
+from servicelib.utils_formatting import timedelta_as_minute_second
+
+
+@pytest.mark.parametrize(
+    "input_timedelta, expected_formatting",
+    [
+        (timedelta(), "00:00"),
+        (timedelta(seconds=23), "00:23"),
+        (timedelta(days=2, seconds=23), f"{2*24*60}:23"),
+        (timedelta(seconds=-23), "-00:23"),
+        (timedelta(seconds=-83), "-01:23"),
+    ],
+)
+def test_timedelta_as_minute_second(
+    input_timedelta: timedelta, expected_formatting: str
+):
+    assert timedelta_as_minute_second(input_timedelta) == expected_formatting

services/autoscaling/requirements/_test.in

@@ -26,4 +26,5 @@ pytest-cov
 pytest-mock
 pytest-runner
 python-dotenv
+pytest-icdiff
 respx

services/autoscaling/requirements/_test.txt

@@ -106,6 +106,8 @@ httpx==0.24.0
     #   -c requirements/_base.txt
     #   -r requirements/_test.in
     #   respx
+icdiff==2.0.7
+    # via pytest-icdiff
 idna==3.4
     # via
     #   -c requirements/_base.txt
@@ -174,6 +176,8 @@ pbr==5.11.1
     #   sarif-om
 pluggy==1.3.0
     # via pytest
+pprintpp==0.4.0
+    # via pytest-icdiff
 psutil==5.9.5
     # via -r requirements/_test.in
 py-partiql-parser==0.3.6
@@ -200,11 +204,14 @@ pytest==7.4.2
     #   -r requirements/_test.in
     #   pytest-asyncio
     #   pytest-cov
+    #   pytest-icdiff
     #   pytest-mock
 pytest-asyncio==0.21.1
     # via -r requirements/_test.in
 pytest-cov==4.1.0
     # via -r requirements/_test.in
+pytest-icdiff==0.8
+    # via -r requirements/_test.in
 pytest-mock==3.11.1
     # via -r requirements/_test.in
 pytest-runner==6.0.0
@@ -218,7 +225,9 @@ python-dateutil==2.8.2
 python-dotenv==1.0.0
     # via -r requirements/_test.in
 python-jose==3.3.0
-    # via moto
+    # via
+    #   moto
+    #   python-jose
 pyyaml==6.0.1
     # via
     #   -c requirements/../../../requirements/constraints.txt

services/autoscaling/src/simcore_service_autoscaling/__init__.py

@@ -1 +0,0 @@
-from ._meta import __version__

services/autoscaling/src/simcore_service_autoscaling/_meta.py

@@ -1,35 +1,17 @@
-""" Application's metadata
-
-"""
-from contextlib import suppress
 from typing import Final
 
-import pkg_resources
+from models_library.basic_types import VersionTag
 from packaging.version import Version
+from servicelib.utils_meta import PackageInfo
 
-_current_distribution = pkg_resources.get_distribution("simcore-service-autoscaling")
-
-__version__: str = _current_distribution.version
-
-
-APP_NAME: Final[str] = _current_distribution.project_name
-API_VERSION: Final[str] = __version__
-VERSION: Final[Version] = Version(__version__)
-API_VTAG: Final[str] = f"v{VERSION.major}"
-
-
-def get_summary() -> str:
-    with suppress(Exception):
-        try:
-            metadata = _current_distribution.get_metadata_lines("METADATA")
-        except FileNotFoundError:
-            metadata = _current_distribution.get_metadata_lines("PKG-INFO")
+info: Final = PackageInfo(package_name="simcore-service-autoscaling")
+__version__: Final[str] = info.__version__
 
-        return next(x.split(":") for x in metadata if x.startswith("Summary:"))[-1]
-    return ""  # pragma: no cover
-
-
-SUMMARY: Final[str] = get_summary()
+APP_NAME: Final[str] = info.project_name
+API_VERSION: Final[str] = info.__version__
+VERSION: Final[Version] = info.version
+API_VTAG: Final[VersionTag] = VersionTag(info.api_prefix_path_tag)
+SUMMARY: Final[str] = info.get_summary()
 
 
 # https://patorjk.com/software/taag/#p=testall&f=Avatar&t=Autoscaling
@@ -47,6 +29,24 @@ def get_summary() -> str:
 )
 
 
-APP_FINISHED_BANNER_MSG = "{:=^100}".format(
-    f"🎉 App {APP_NAME}=={__version__} shutdown completed 🎉"
-)
+APP_STARTED_DYNAMIC_BANNER_MSG = r"""
+      _                                  _
+     | |                                (_)
+   __| | _   _  _ __    __ _  _ __ ___   _   ___
+  / _` || | | || '_ \  / _` || '_ ` _ \ | | / __|
+ | (_| || |_| || | | || (_| || | | | | || || (__
+  \__,_| \__, ||_| |_| \__,_||_| |_| |_||_| \___|
+          __/ |
+         |___/
+"""
+
+APP_STARTED_DISABLED_BANNER_MSG = r"""
+      _  _              _      _            _
+     | |(_)            | |    | |          | |
+   __| | _  ___   __ _ | |__  | |  ___   __| |
+  / _` || |/ __| / _` || '_ \ | | / _ \ / _` |
+ | (_| || |\__ \| (_| || |_) || ||  __/| (_| |
+  \__,_||_||___/ \__,_||_.__/ |_| \___| \__,_|
+"""
+
+APP_FINISHED_BANNER_MSG = info.get_finished_banner()

services/autoscaling/src/simcore_service_autoscaling/core/application.py

@@ -8,6 +8,8 @@
     APP_FINISHED_BANNER_MSG,
     APP_NAME,
     APP_STARTED_BANNER_MSG,
+    APP_STARTED_DISABLED_BANNER_MSG,
+    APP_STARTED_DYNAMIC_BANNER_MSG,
 )
 from ..api.routes import setup_api_routes
 from ..modules.auto_scaling_task import setup as setup_background_task
@@ -49,10 +51,14 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
 
     # EVENTS
     async def _on_startup() -> None:
-        print(APP_STARTED_BANNER_MSG, flush=True)
+        print(APP_STARTED_BANNER_MSG, flush=True)  # noqa: T201
+        if settings.AUTOSCALING_NODES_MONITORING:
+            print(APP_STARTED_DYNAMIC_BANNER_MSG, flush=True)  # noqa: T201
+        else:
+            print(APP_STARTED_DISABLED_BANNER_MSG, flush=True)  # noqa: T201
 
     async def _on_shutdown() -> None:
-        print(APP_FINISHED_BANNER_MSG, flush=True)
+        print(APP_FINISHED_BANNER_MSG, flush=True)  # noqa: T201
 
     app.add_event_handler("startup", _on_startup)
     app.add_event_handler("shutdown", _on_shutdown)

services/autoscaling/src/simcore_service_autoscaling/core/settings.py

@@ -119,17 +119,17 @@ def check_valid_intance_names(cls, value):
 
 class NodesMonitoringSettings(BaseCustomSettings):
     NODES_MONITORING_NODE_LABELS: list[DockerLabelKey] = Field(
-        default_factory=list,
+        ...,
         description="autoscaling will only monitor nodes with the given labels (if empty all nodes will be monitored), these labels will be added to the new created nodes by default",
     )
 
     NODES_MONITORING_SERVICE_LABELS: list[DockerLabelKey] = Field(
-        default_factory=list,
+        ...,
         description="autoscaling will only monitor services with the given labels (if empty all services will be monitored)",
     )
 
     NODES_MONITORING_NEW_NODES_LABELS: list[DockerLabelKey] = Field(
-        default=["io.simcore.autoscaled-node"],
+        ...,
         description="autoscaling will add these labels to any new node it creates (additional to the ones in NODES_MONITORING_NODE_LABELS",
     )
 

services/autoscaling/src/simcore_service_autoscaling/models.py

@@ -21,15 +21,19 @@ def __add__(self, other: "Resources") -> "Resources":
         return Resources.construct(
             **{
                 key: a + b
-                for (key, a), b in zip(self.dict().items(), other.dict().values())
+                for (key, a), b in zip(
+                    self.dict().items(), other.dict().values(), strict=True
+                )
             }
         )
 
     def __sub__(self, other: "Resources") -> "Resources":
         return Resources.construct(
             **{
                 key: a - b
-                for (key, a), b in zip(self.dict().items(), other.dict().values())
+                for (key, a), b in zip(
+                    self.dict().items(), other.dict().values(), strict=True
+                )
             }
         )