chore: improve logging in scale down lambda (#3954)

These changes make it easier to debug the scale down lambda to diagnose
problems

Signed-off-by: Brend Smits <[email protected]>

Author: Brend-Smits

lambdas/functions/control-plane/src/aws/runners.ts

@@ -94,6 +94,7 @@ function getRunnerInfo(runningInstances: DescribeInstancesResult) {
 }
 
 export async function terminateRunner(instanceId: string): Promise<void> {
+  logger.info(`Runner '${instanceId}' will be terminated.`);
   const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION }));
   await ec2.send(new TerminateInstancesCommand({ InstanceIds: [instanceId] }));
   logger.info(`Runner ${instanceId} has been terminated.`);

lambdas/functions/control-plane/src/scale-runners/scale-down.ts

@@ -88,7 +88,8 @@ async function listGitHubRunners(runner: RunnerInfo): Promise<GhRunners> {
           per_page: 100,
         });
   githubCache.runners.set(key, runners);
-
+  logger.debug(`[listGithubRunners] Cache set for ${key}`);
+  logger.debug(`[listGithubRunners] Runners: ${JSON.stringify(runners)}`);
   return runners;
 }
 
@@ -156,18 +157,25 @@ async function evaluateAndRemoveRunners(
       .filter((runner) => runner.owner === ownerTag)
       .sort(evictionStrategy === 'oldest_first' ? oldestFirstStrategy : newestFirstStrategy);
     logger.debug(`Found: '${ec2RunnersFiltered.length}' active GitHub runners with owner tag: '${ownerTag}'`);
+    logger.debug(`Active GitHub runners with owner tag: '${ownerTag}': ${JSON.stringify(ec2RunnersFiltered)}`);
     for (const ec2Runner of ec2RunnersFiltered) {
       const ghRunners = await listGitHubRunners(ec2Runner);
       const ghRunnersFiltered = ghRunners.filter((runner: { name: string }) =>
         runner.name.endsWith(ec2Runner.instanceId),
       );
+      logger.debug(
+        `Found: '${ghRunnersFiltered.length}' GitHub runners for AWS runner instance: '${ec2Runner.instanceId}'`,
+      );
+      logger.debug(
+        `GitHub runners for AWS runner instance: '${ec2Runner.instanceId}': ${JSON.stringify(ghRunnersFiltered)}`,
+      );
       if (ghRunnersFiltered.length) {
         if (runnerMinimumTimeExceeded(ec2Runner)) {
           if (idleCounter > 0) {
             idleCounter--;
             logger.info(`Runner '${ec2Runner.instanceId}' will be kept idle.`);
           } else {
-            logger.info(`Runner '${ec2Runner.instanceId}' will be terminated.`);
+            logger.info(`Will try to terminate runners that are not busy`);
             await removeRunner(
               ec2Runner,
               ghRunnersFiltered.map((runner: { id: number }) => runner.id),
@@ -224,6 +232,7 @@ export async function scaleDown(): Promise<void> {
   const ec2Runners = await listRunners(environment);
   const activeEc2RunnersCount = ec2Runners.length;
   logger.info(`Found: '${activeEc2RunnersCount}' active GitHub EC2 runner instances before clean-up.`);
+  logger.debug(`Active GitHub EC2 runner instances: ${JSON.stringify(ec2Runners)}`);
 
   if (activeEc2RunnersCount === 0) {
     logger.debug(`No active runners found for environment: '${environment}'`);