Skip to content

Commit ddeecaa

Browse files
authored
Merge pull request #1220 from philips-labs/develop
release
2 parents d180f00 + c8b8139 commit ddeecaa

File tree

23 files changed

+5035
-7724
lines changed

23 files changed

+5035
-7724
lines changed

.release/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@
1515
"conventional-changelog-conventionalcommits": "^4.5.0",
1616
"semantic-release": "^17.3.9"
1717
}
18-
}
18+
}

.release/yarn.lock

+1,258-2,279
Large diffs are not rendered by default.

README.md

+80-81
Large diffs are not rendered by default.

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json

+5-5
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,17 @@
2222
"@types/request": "^2.48.4",
2323
"@typescript-eslint/eslint-plugin": "^4.29.2",
2424
"@typescript-eslint/parser": "^4.29.2",
25-
"@vercel/ncc": "^0.29.1",
25+
"@vercel/ncc": "^0.31.1",
2626
"aws-sdk": "^2.970.0",
2727
"eslint": "^7.32.0",
2828
"eslint-plugin-prettier": "3.4.0",
29-
"jest": "^26.6.3",
30-
"prettier": "2.3.2",
31-
"ts-jest": "^26.5.5",
29+
"jest": "^27.2.4",
30+
"prettier": "2.4.1",
31+
"ts-jest": "^27.0.5",
3232
"ts-node-dev": "^1.1.6",
3333
"typescript": "^4.3.4"
3434
},
3535
"dependencies": {
3636
"node-fetch": "^2.6.1"
3737
}
38-
}
38+
}

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock

+880-1,776
Large diffs are not rendered by default.

modules/runners/README.md

+9-8
Original file line numberDiff line numberDiff line change
@@ -87,15 +87,15 @@ No Modules.
8787

8888
| Name | Description | Type | Default | Required |
8989
|------|-------------|------|---------|:--------:|
90-
| ami\_filter | List of maps used to create the AMI filter for the action runner AMI. | `map(list(string))` | <pre>{<br> "name": [<br> "amzn2-ami-hvm-2.*-x86_64-ebs"<br> ]<br>}</pre> | no |
90+
| ami\_filter | Map of lists used to create the AMI filter for the action runner AMI. | `map(list(string))` | <pre>{<br> "name": [<br> "amzn2-ami-hvm-2.*-x86_64-ebs"<br> ]<br>}</pre> | no |
9191
| ami\_owners | The list of owners used to select the AMI of action runner instances. | `list(string)` | <pre>[<br> "amazon"<br>]</pre> | no |
9292
| aws\_region | AWS region. | `string` | n/a | yes |
9393
| block\_device\_mappings | The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops` | `map(string)` | `{}` | no |
9494
| cloudwatch\_config | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. | `string` | `null` | no |
95-
| create\_service\_linked\_role\_spot | (optional) create the serviced linked role for spot instances that is required by the scale-up lambda. | `bool` | `false` | no |
95+
| create\_service\_linked\_role\_spot | (optional) create the service linked role for spot instances that is required by the scale-up lambda. | `bool` | `false` | no |
9696
| enable\_cloudwatch\_agent | Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`. | `bool` | `true` | no |
9797
| enable\_organization\_runners | n/a | `bool` | n/a | yes |
98-
| enable\_ssm\_on\_runners | Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances. | `bool` | n/a | yes |
98+
| enable\_ssm\_on\_runners | Enable to allow access to the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances. | `bool` | n/a | yes |
9999
| environment | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes |
100100
| ghes\_url | GitHub Enterprise Server URL. DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no |
101101
| github\_app\_parameters | Parameter Store for GitHub App Parameters. | <pre>object({<br> key_base64 = map(string)<br> id = map(string)<br> client_id = map(string)<br> client_secret = map(string)<br> })</pre> | n/a | yes |
@@ -114,16 +114,17 @@ No Modules.
114114
| logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
115115
| market\_options | Market options for the action runner instances. | `string` | `"spot"` | no |
116116
| minimum\_running\_time\_in\_minutes | The time an ec2 action runner should be running at minimum before terminated if non busy. | `number` | `5` | no |
117-
| overrides | This maps provides the possibility to override some defaults. The following attributes are supported: `name_sg` overwrite the `Name` tag for all security groups created by this module. `name_runner_agent_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` override the `Name` tag spot instances created by the runner agent. | `map(string)` | <pre>{<br> "name_runner": "",<br> "name_sg": ""<br>}</pre> | no |
118-
| role\_path | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
117+
| overrides | This map provides the possibility to override some defaults. The following attributes are supported: `name_sg` overrides the `Name` tag for all security groups created by this module. `name_runner_agent_instance` overrides the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` overrides the `Name` tag spot instances created by the runner agent. | `map(string)` | <pre>{<br> "name_runner": "",<br> "name_sg": ""<br>}</pre> | no |
118+
| role\_path | The path that will be added to the role; if not set, the environment name will be used. | `string` | `null` | no |
119119
| role\_permissions\_boundary | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
120120
| runner\_additional\_security\_group\_ids | (optional) List of additional security groups IDs to apply to the runner | `list(string)` | `[]` | no |
121121
| runner\_architecture | The platform architecture of the runner instance\_type. | `string` | `"x64"` | no |
122122
| runner\_as\_root | Run the action runner under the root user. | `bool` | `false` | no |
123+
| runner\_boot\_time\_in\_minutes | The minimum time for an EC2 runner to boot and register as a runner. | `number` | `5` | no |
123124
| runner\_extra\_labels | Extra labels for the runners (GitHub). Separate each label by a comma | `string` | `""` | no |
124125
| runner\_group\_name | Name of the runner group. | `string` | `"Default"` | no |
125126
| runner\_iam\_role\_managed\_policy\_arns | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no |
126-
| runner\_log\_files | (optional) List of logfiles to send to cloudwatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/<var.environment>`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | <pre>list(object({<br> log_group_name = string<br> prefix_log_group = bool<br> file_path = string<br> log_stream_name = string<br> }))</pre> | <pre>[<br> {<br> "file_path": "/var/log/messages",<br> "log_group_name": "messages",<br> "log_stream_name": "{instance_id}",<br> "prefix_log_group": true<br> },<br> {<br> "file_path": "/var/log/user-data.log",<br> "log_group_name": "user_data",<br> "log_stream_name": "{instance_id}",<br> "prefix_log_group": true<br> },<br> {<br> "file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",<br> "log_group_name": "runner",<br> "log_stream_name": "{instance_id}",<br> "prefix_log_group": true<br> }<br>]</pre> | no |
127+
| runner\_log\_files | (optional) List of logfiles to send to CloudWatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/<var.environment>`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | <pre>list(object({<br> log_group_name = string<br> prefix_log_group = bool<br> file_path = string<br> log_stream_name = string<br> }))</pre> | <pre>[<br> {<br> "file_path": "/var/log/messages",<br> "log_group_name": "messages",<br> "log_stream_name": "{instance_id}",<br> "prefix_log_group": true<br> },<br> {<br> "file_path": "/var/log/user-data.log",<br> "log_group_name": "user_data",<br> "log_stream_name": "{instance_id}",<br> "prefix_log_group": true<br> },<br> {<br> "file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",<br> "log_group_name": "runner",<br> "log_stream_name": "{instance_id}",<br> "prefix_log_group": true<br> }<br>]</pre> | no |
127128
| runners\_lambda\_s3\_key | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
128129
| runners\_lambda\_s3\_object\_version | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
129130
| runners\_maximum\_count | The maximum number of runners that will be created. | `number` | `3` | no |
@@ -133,8 +134,8 @@ No Modules.
133134
| sqs\_build\_queue | SQS queue to consume accepted build events. | <pre>object({<br> arn = string<br> })</pre> | n/a | yes |
134135
| subnet\_ids | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | n/a | yes |
135136
| tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
136-
| userdata\_post\_install | User-data script snippet to insert after GitHub acton runner install | `string` | `""` | no |
137-
| userdata\_pre\_install | User-data script snippet to insert before GitHub acton runner install | `string` | `""` | no |
137+
| userdata\_post\_install | User-data script snippet to insert after GitHub action runner install | `string` | `""` | no |
138+
| userdata\_pre\_install | User-data script snippet to insert before GitHub action runner install | `string` | `""` | no |
138139
| userdata\_template | Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no |
139140
| volume\_size | Size of runner volume | `number` | `30` | no |
140141
| vpc\_id | The VPC for the security groups. | `string` | n/a | yes |

modules/runners/lambdas/runners/package.json

+6-6
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@
2121
"@types/jest": "^27.0.1",
2222
"@typescript-eslint/eslint-plugin": "^4.29.2",
2323
"@typescript-eslint/parser": "^4.29.2",
24-
"@vercel/ncc": "^0.29.1",
24+
"@vercel/ncc": "^0.31.1",
2525
"eslint": "^7.32.0",
26-
"eslint-plugin-prettier": "3.4.0",
27-
"jest": "27.0.6",
26+
"eslint-plugin-prettier": "4.0.0",
27+
"jest": "27.2.4",
2828
"jest-mock-extended": "^2.0.1",
2929
"moment-timezone": "^0.5.33",
3030
"nock": "^13.0.11",
31-
"prettier": "2.3.2",
31+
"prettier": "2.4.1",
3232
"ts-jest": "^27.0.5",
3333
"ts-node": "^10.2.0",
3434
"ts-node-dev": "^1.1.6"
@@ -42,7 +42,7 @@
4242
"@types/express": "^4.17.11",
4343
"@types/node": "^16.6.1",
4444
"aws-sdk": "^2.970.0",
45-
"cron-parser": "^3.3.0",
45+
"cron-parser": "^4.0.0",
4646
"typescript": "^4.2.3"
4747
}
48-
}
48+
}

modules/runners/lambdas/runners/src/lambda.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { scaleDown as scaleDownAction } from './scale-runners/scale-down';
33
import { SQSEvent, ScheduledEvent, Context } from 'aws-lambda';
44

55
export const scaleUp = async (event: SQSEvent, context: Context, callback: any): Promise<void> => {
6-
console.dir(event, { depth: 5 });
6+
console.debug(JSON.stringify(event));
77
try {
88
for (const e of event.Records) {
99
await scaleUpAction(e.eventSource, JSON.parse(e.body));
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { Octokit } from '@octokit/rest';
2+
3+
export type UnboxPromise<T> = T extends Promise<infer U> ? U : T;
4+
5+
export type GhRunners = UnboxPromise<ReturnType<Octokit['actions']['listSelfHostedRunnersForRepo']>>['data']['runners'];
6+
7+
export class githubCache {
8+
static clients: Map<string, Octokit> = new Map();
9+
static runners: Map<string, GhRunners> = new Map();
10+
11+
public static reset(): void {
12+
githubCache.clients.clear();
13+
githubCache.runners.clear();
14+
}
15+
}

modules/runners/lambdas/runners/src/scale-runners/runners.test.ts

+26-20
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { listRunners, createRunner, terminateRunner, RunnerInfo } from './runners';
1+
import { listEC2Runners, createRunner, terminateRunner, RunnerInfo } from './runners';
22

33
const mockEC2 = { describeInstances: jest.fn(), runInstances: jest.fn(), terminateInstances: jest.fn() };
44
const mockSSM = { putParameter: jest.fn() };
@@ -25,17 +25,17 @@ describe('list instances', () => {
2525
LaunchTime: new Date('2020-10-10T14:48:00.000+09:00'),
2626
InstanceId: 'i-1234',
2727
Tags: [
28-
{ Key: 'Repo', Value: 'CoderToCat/hello-world' },
29-
{ Key: 'Org', Value: 'CoderToCat' },
3028
{ Key: 'Application', Value: 'github-action-runner' },
29+
{ Key: 'Type', Value: 'Org' },
30+
{ Key: 'Owner', Value: 'CoderToCat' },
3131
],
3232
},
3333
{
3434
LaunchTime: new Date('2020-10-11T14:48:00.000+09:00'),
3535
InstanceId: 'i-5678',
3636
Tags: [
37-
{ Key: 'Repo', Value: REPO_NAME },
38-
{ Key: 'Org', Value: ORG_NAME },
37+
{ Key: 'Owner', Value: REPO_NAME },
38+
{ Key: 'Type', Value: 'Repo' },
3939
{ Key: 'Application', Value: 'github-action-runner' },
4040
],
4141
},
@@ -47,51 +47,53 @@ describe('list instances', () => {
4747
});
4848

4949
it('returns a list of instances', async () => {
50-
const resp = await listRunners();
50+
const resp = await listEC2Runners();
5151
expect(resp.length).toBe(2);
5252
expect(resp).toContainEqual({
5353
instanceId: 'i-1234',
5454
launchTime: new Date('2020-10-10T14:48:00.000+09:00'),
55-
repo: 'CoderToCat/hello-world',
56-
org: 'CoderToCat',
55+
type: 'Org',
56+
owner: 'CoderToCat',
5757
});
5858
expect(resp).toContainEqual({
5959
instanceId: 'i-5678',
6060
launchTime: new Date('2020-10-11T14:48:00.000+09:00'),
61-
repo: REPO_NAME,
62-
org: ORG_NAME,
61+
type: 'Repo',
62+
owner: REPO_NAME,
6363
});
6464
});
6565

6666
it('calls EC2 describe instances', async () => {
67-
await listRunners();
67+
await listEC2Runners();
6868
expect(mockEC2.describeInstances).toBeCalled();
6969
});
7070

7171
it('filters instances on repo name', async () => {
72-
await listRunners({ runnerType: 'Repo', runnerOwner: REPO_NAME, environment: undefined });
72+
await listEC2Runners({ runnerType: 'Repo', runnerOwner: REPO_NAME, environment: undefined });
7373
expect(mockEC2.describeInstances).toBeCalledWith({
7474
Filters: [
7575
{ Name: 'tag:Application', Values: ['github-action-runner'] },
7676
{ Name: 'instance-state-name', Values: ['running', 'pending'] },
77-
{ Name: 'tag:Repo', Values: [REPO_NAME] },
77+
{ Name: 'tag:Type', Values: ['Repo'] },
78+
{ Name: 'tag:Owner', Values: [REPO_NAME] },
7879
],
7980
});
8081
});
8182

8283
it('filters instances on org name', async () => {
83-
await listRunners({ runnerType: 'Org', runnerOwner: ORG_NAME, environment: undefined });
84+
await listEC2Runners({ runnerType: 'Org', runnerOwner: ORG_NAME, environment: undefined });
8485
expect(mockEC2.describeInstances).toBeCalledWith({
8586
Filters: [
8687
{ Name: 'tag:Application', Values: ['github-action-runner'] },
8788
{ Name: 'instance-state-name', Values: ['running', 'pending'] },
88-
{ Name: 'tag:Org', Values: [ORG_NAME] },
89+
{ Name: 'tag:Type', Values: ['Org'] },
90+
{ Name: 'tag:Owner', Values: [ORG_NAME] },
8991
],
9092
});
9193
});
9294

93-
it('filters instances on org name', async () => {
94-
await listRunners({ environment: ENVIRONMENT });
95+
it('filters instances on environment', async () => {
96+
await listEC2Runners({ environment: ENVIRONMENT });
9597
expect(mockEC2.describeInstances).toBeCalledWith({
9698
Filters: [
9799
{ Name: 'tag:Application', Values: ['github-action-runner'] },
@@ -112,8 +114,10 @@ describe('terminate runner', () => {
112114
it('calls terminate instances with the right instance ids', async () => {
113115
const runner: RunnerInfo = {
114116
instanceId: 'instance-2',
117+
owner: 'owner-2',
118+
type: 'Repo',
115119
};
116-
await terminateRunner(runner);
120+
await terminateRunner(runner.instanceId);
117121

118122
expect(mockEC2.terminateInstances).toBeCalledWith({ InstanceIds: [runner.instanceId] });
119123
});
@@ -156,7 +160,8 @@ describe('create runner', () => {
156160
ResourceType: 'instance',
157161
Tags: [
158162
{ Key: 'Application', Value: 'github-action-runner' },
159-
{ Key: 'Repo', Value: REPO_NAME },
163+
{ Key: 'Type', Value: 'Repo' },
164+
{ Key: 'Owner', Value: REPO_NAME },
160165
],
161166
},
162167
],
@@ -183,7 +188,8 @@ describe('create runner', () => {
183188
ResourceType: 'instance',
184189
Tags: [
185190
{ Key: 'Application', Value: 'github-action-runner' },
186-
{ Key: 'Org', Value: ORG_NAME },
191+
{ Key: 'Type', Value: 'Org' },
192+
{ Key: 'Owner', Value: ORG_NAME },
187193
],
188194
},
189195
],

0 commit comments

Comments
 (0)