feat(runners): Add delay to prevent ssm rate limits using setTimeout (#2823)

* ISSUE-1841: Fix ssm rate limiting issue when instance count is greater than SSM parameter store max throughput

* ISSUE-1841: Add jest test for creating 40 runners at once to cover ssm rate limits condition

* ISSUE-1841: Fix linter and update test to check the actual time delay for creating 40+ runners

* ISSUE-1841: Fix require statement for perf_hooks

* ISSUE-1841: Fix formatting

Author: devsh4

modules/runners/lambdas/runners/src/aws/runners.test.ts

@@ -1,4 +1,5 @@
 import { EC2 } from 'aws-sdk';
+import { performance } from 'perf_hooks';
 
 import ScaleError from './../scale-runners/ScaleError';
 import { RunnerInfo, RunnerInputParameters, createRunner, listEC2Runners, terminateRunner } from './runners';
@@ -224,6 +225,75 @@ describe('create runner', () => {
     expect(mockSSM.putParameter).toBeCalledTimes(1);
   });
 
+  it('calls create fleet of 40 instances (ssm rate limit condition) to test time delay ', async () => {
+    const startTime = performance.now();
+    const instances = [
+      {
+        InstanceIds: [
+          'i-1234',
+          'i-5678',
+          'i-5567',
+          'i-5569',
+          'i-5561',
+          'i-5560',
+          'i-5566',
+          'i-5536',
+          'i-5526',
+          'i-5516',
+          'i-122',
+          'i-123',
+          'i-124',
+          'i-125',
+          'i-126',
+          'i-127',
+          'i-128',
+          'i-129',
+          'i-130',
+          'i-131',
+          'i-132',
+          'i-133',
+          'i-134',
+          'i-135',
+          'i-136',
+          'i-137',
+          'i-138',
+          'i-139',
+          'i-140',
+          'i-141',
+          'i-142',
+          'i-143',
+          'i-144',
+          'i-145',
+          'i-146',
+          'i-147',
+          'i-148',
+          'i-149',
+          'i-150',
+          'i-151',
+        ],
+      },
+    ];
+    mockCreateFleet.promise.mockReturnValue({
+      Instances: instances,
+    });
+
+    await createRunner({ ...createRunnerConfig(defaultRunnerConfig), numberOfRunners: 40 });
+    const endTime = performance.now();
+
+    expect(endTime - startTime).toBeGreaterThan(1000);
+    expect(mockEC2.createFleet).toBeCalledWith(
+      expectedCreateFleetRequest({ ...defaultExpectedFleetRequestValues, totalTargetCapacity: 40 }),
+    );
+    expect(mockSSM.putParameter).toBeCalledTimes(40);
+    for (const instance of instances[0].InstanceIds) {
+      expect(mockSSM.putParameter).toBeCalledWith({
+        Name: `${SSM_TOKEN_PATH}/${instance}`,
+        Type: 'SecureString',
+        Value: '--token foo --url http://github.com',
+      });
+    }
+  });
+
   it('calls create fleet of 1 instance with the on-demand capacity', async () => {
     await createRunner(createRunnerConfig({ ...defaultRunnerConfig, capacityType: 'on-demand' }));
     expect(mockEC2.createFleet).toBeCalledWith(

modules/runners/lambdas/runners/src/aws/runners.ts

@@ -256,6 +256,10 @@ export async function createRunner(runnerParameters: RunnerInputParameters): Pro
 
   logger.info('Created instance(s): ', instances.join(','), LogFields.print());
 
+  const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+  const ssmParameterStoreMaxThroughput = 40;
+  const isDelay = instances.length >= ssmParameterStoreMaxThroughput ? true : false;
+
   for (const instance of instances) {
     await ssm
       .putParameter({
@@ -264,5 +268,10 @@ export async function createRunner(runnerParameters: RunnerInputParameters): Pro
         Type: 'SecureString',
       })
       .promise();
+
+    if (isDelay) {
+      // Delay to prevent AWS ssm rate limits by being within the max throughput limit
+      await delay(25);
+    }
   }
 }