✨Maintenance CLI: introduce option to terminate dynamic instances (#7630)

Author: sanderegg

scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py

@@ -209,5 +209,18 @@ def check_database_connection() -> None:
     asyncio.run(api.check_database_connection(state))
 
 
+@app.command()
+def terminate_dynamic_instances(
+    user_id: Annotated[int | None, typer.Option(help="the user ID")] = None,
+    instance_id: Annotated[str | None, typer.Option(help="the instance ID")] = None,
+    *,
+    force: Annotated[bool, typer.Option(help="will not ask for confirmation")] = False,
+) -> None:
+    """this will terminate the instance(s) used for the given user or instance ID."""
+    asyncio.run(
+        api.terminate_dynamic_instances(state, user_id, instance_id, force=force)
+    )
+
+
 if __name__ == "__main__":
     app()

scripts/maintenance/computational-clusters/autoscaled_monitor/core.py

@@ -491,7 +491,10 @@ async def summary(
     # get all the running instances
     assert state.ec2_resource_autoscaling
     dynamic_instances = await ec2.list_dynamic_instances_from_ec2(
-        state, user_id, wallet_id
+        state,
+        filter_by_user_id=user_id,
+        filter_by_wallet_id=wallet_id,
+        filter_by_instance_id=None,
     )
     dynamic_autoscaled_instances = await _parse_dynamic_instances(
         state, dynamic_instances, state.ssh_key_path, user_id, wallet_id
@@ -776,3 +779,48 @@ async def trigger_cluster_termination(
 
 async def check_database_connection(state: AppState) -> None:
     await db.check_db_connection(state)
+
+
+async def terminate_dynamic_instances(
+    state: AppState,
+    user_id: int | None,
+    instance_id: str | None,
+    *,
+    force: bool,
+) -> None:
+    if not user_id and not instance_id:
+        rich.print("either define user_id or instance_id!")
+        raise typer.Exit(2)
+    dynamic_instances = await ec2.list_dynamic_instances_from_ec2(
+        state,
+        filter_by_user_id=None,
+        filter_by_wallet_id=None,
+        filter_by_instance_id=instance_id,
+    )
+
+    dynamic_autoscaled_instances = await _parse_dynamic_instances(
+        state, dynamic_instances, state.ssh_key_path, user_id, None
+    )
+
+    if not dynamic_autoscaled_instances:
+        rich.print("no instances found")
+        raise typer.Exit(1)
+
+    _print_dynamic_instances(
+        dynamic_autoscaled_instances,
+        state.environment,
+        state.ec2_resource_autoscaling.meta.client.meta.region_name,
+        output=None,
+    )
+
+    for instance in dynamic_autoscaled_instances:
+        rich.print(
+            f"terminating instance {instance.ec2_instance.instance_id} with name {utils.get_instance_name(instance.ec2_instance)}"
+        )
+        if force is True or typer.confirm(
+            f"Are you sure you want to terminate instance {instance.ec2_instance.instance_id}?"
+        ):
+            instance.ec2_instance.terminate()
+            rich.print(f"terminated instance {instance.ec2_instance.instance_id}")
+        else:
+            rich.print("not terminating anything")

scripts/maintenance/computational-clusters/autoscaled_monitor/ec2.py

@@ -18,6 +18,7 @@ def _list_running_ec2_instances(
     custom_tags: dict[str, str],
     user_id: int | None,
     wallet_id: int | None,
+    instance_id: str | None,
 ) -> ServiceResourceInstancesCollection:
     # get all the running instances
 
@@ -37,7 +38,8 @@ def _list_running_ec2_instances(
         ec2_filters.append({"Name": "tag:user_id", "Values": [f"{user_id}"]})
     if wallet_id:
         ec2_filters.append({"Name": "tag:wallet_id", "Values": [f"{wallet_id}"]})
-
+    if instance_id:
+        ec2_filters.append({"Name": "instance-id", "Values": [f"{instance_id}"]})
     return ec2_resource.instances.filter(Filters=ec2_filters)
 
 
@@ -66,13 +68,16 @@ async def list_computational_instances_from_ec2(
         custom_tags,
         user_id,
         wallet_id,
+        None,
     )
 
 
 async def list_dynamic_instances_from_ec2(
     state: AppState,
-    user_id: int | None,
-    wallet_id: int | None,
+    *,
+    filter_by_user_id: int | None,
+    filter_by_wallet_id: int | None,
+    filter_by_instance_id: str | None,
 ) -> ServiceResourceInstancesCollection:
     assert state.environment["EC2_INSTANCES_KEY_NAME"]
     custom_tags = {}
@@ -83,8 +88,9 @@ async def list_dynamic_instances_from_ec2(
         state.ec2_resource_autoscaling,
         state.environment["EC2_INSTANCES_KEY_NAME"],
         custom_tags,
-        user_id,
-        wallet_id,
+        filter_by_user_id,
+        filter_by_wallet_id,
+        filter_by_instance_id,
     )
 
 
@@ -101,6 +107,7 @@ async def get_computational_bastion_instance(state: AppState) -> Instance:
         {},
         None,
         None,
+        None,
     )
 
     possible_bastions = list(
@@ -120,6 +127,7 @@ async def get_dynamic_bastion_instance(state: AppState) -> Instance:
         {},
         None,
         None,
+        None,
     )
 
     possible_bastions = list(