Skip to content

Commit f438c8d

Browse files
biffgautbiffgaut
authored
Merge pull request #642 from awslabs/bump/2.5.0
chore(release): 2.5.0
2 parents f65d5fe + 8ec92d9 commit f438c8d

24 files changed

+1152
-115
lines changed

‎CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,14 @@
22

33
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
44

5+
## [1.148.0](https://github.com/awslabs/aws-solutions-constructs/compare/v1.147.0...v1.148.0) (2022-03-30)
6+
7+
* Upgraded all patterns to CDK v1.148.0
8+
9+
## [1.147.0](https://github.com/awslabs/aws-solutions-constructs/compare/v2.4.0...v1.147.0) (2022-03-29)
10+
11+
* Upgraded all patterns to CDK v1.147.0
12+
513
## [1.146.0](https://github.com/awslabs/aws-solutions-constructs/compare/v1.145.0...v1.146.0) (2022-03-02)
614

715
* Upgraded all patterns to CDK v1.146.0

‎CHANGELOG.v2.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,17 @@
22

33
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
44

5+
## [2.5.0](https://github.com/awslabs/aws-solutions-constructs/compare/v2.4.0...v2.5.0) (2022-03-30)
6+
7+
### Features
8+
* Includes all functionality of V1.148.0
9+
* Built upon underlying CDK version V2.15.0
10+
511
## [2.4.0](https://github.com/awslabs/aws-solutions-constructs/compare/v2.3.0...v2.4.0) (2022-03-29)
612

713
### Features
814
* Includes all functionality of V1.146.0
9-
* Built upon underlying CDK version V2.10.0
15+
* Built upon underlying CDK version V2.15.0
1016

1117
## [2.3.0](https://github.com/awslabs/aws-solutions-constructs/compare/v2.2.0...v2.3.0) (2022-02-24)
1218

‎deployment/v2/align-version.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ const findVersion = process.argv[2];
1010
const replaceVersion = process.argv[3];
1111

1212
// these versions need to be sourced from a config file
13-
const awsCdkLibVersion = '2.10.0';
13+
const awsCdkLibVersion = '2.15.0';
1414
const constructsVersion = '10.0.0';
1515
const MODULE_EXEMPTIONS = new Set([
1616
'@aws-cdk/cloudformation-diff',

‎source/lerna.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@
66
"./patterns/@aws-solutions-constructs/*"
77
],
88
"rejectCycles": "true",
9-
"version": "1.146.0"
9+
"version": "1.148.0"
1010
}

‎source/lerna.v2.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@
66
"./patterns/@aws-solutions-constructs/*"
77
],
88
"rejectCycles": "true",
9-
"version": "2.4.0"
9+
"version": "2.5.0"
1010
}

‎source/patterns/@aws-solutions-constructs/aws-iot-lambda-dynamodb/test/integ.iot-lambda-dynamodb.expected.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,8 @@
8080
"dynamodb:BatchWriteItem",
8181
"dynamodb:PutItem",
8282
"dynamodb:UpdateItem",
83-
"dynamodb:DeleteItem"
83+
"dynamodb:DeleteItem",
84+
"dynamodb:DescribeTable"
8485
],
8586
"Effect": "Allow",
8687
"Resource": [

‎source/patterns/@aws-solutions-constructs/aws-iot-lambda-dynamodb/test/iot-lambda-dynamodb.test.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,8 @@ test('check lambda function policy ', () => {
223223
"dynamodb:BatchWriteItem",
224224
"dynamodb:PutItem",
225225
"dynamodb:UpdateItem",
226-
"dynamodb:DeleteItem"
226+
"dynamodb:DeleteItem",
227+
"dynamodb:DescribeTable"
227228
],
228229
Effect: "Allow",
229230
Resource: [

‎source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/README.md

Lines changed: 36 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -30,44 +30,44 @@ This AWS Solutions Construct deploys a Kinesis Stream and configures a AWS Glue
3030
Here is a minimal deployable pattern definition in Typescript:
3131

3232
```javascript
33-
import * as glue from '@aws-cdk/aws-glue';
34-
import * as s3assets from '@aws-cdk/aws-s3-assets';
35-
import {KinesisstreamsToGluejob} from '@aws-solutions-constructs/aws-kinesisstreams-gluejob';
33+
import * as glue from "@aws-cdk/aws-glue";
34+
import * as s3assets from "@aws-cdk/aws-s3-assets";
35+
import { KinesisstreamsToGluejob } from "@aws-solutions-constructs/aws-kinesisstreams-gluejob";
3636

3737
const fieldSchema: glue.CfnTable.ColumnProperty[] = [
38-
{
39-
name: 'id',
40-
type: 'int',
41-
comment: 'Identifier for the record',
42-
},
43-
{
44-
name: 'name',
45-
type: 'string',
46-
comment: 'Name for the record',
47-
},
48-
{
49-
name: 'address',
50-
type: 'string',
51-
comment: 'Address for the record',
52-
},
53-
{
54-
name: 'value',
55-
type: 'int',
56-
comment: 'Value for the record',
57-
},
38+
{
39+
name: "id",
40+
type: "int",
41+
comment: "Identifier for the record",
42+
},
43+
{
44+
name: "name",
45+
type: "string",
46+
comment: "Name for the record",
47+
},
48+
{
49+
name: "address",
50+
type: "string",
51+
comment: "Address for the record",
52+
},
53+
{
54+
name: "value",
55+
type: "int",
56+
comment: "Value for the record",
57+
},
5858
];
5959

60-
const customEtlJob = new KinesisstreamsToGluejob(this, 'CustomETL', {
61-
glueJobProps: {
62-
command: {
63-
name: 'gluestreaming',
64-
pythonVersion: '3',
65-
scriptLocation: new s3assets.Asset(this, 'ScriptLocation', {
66-
path: `${__dirname}/../etl/transform.py`,
67-
}).s3ObjectUrl,
68-
},
60+
const customEtlJob = new KinesisstreamsToGluejob(this, "CustomETL", {
61+
glueJobProps: {
62+
command: {
63+
name: "gluestreaming",
64+
pythonVersion: "3",
6965
},
70-
fieldSchema: fieldSchema,
66+
},
67+
fieldSchema: fieldSchema,
68+
etlCodeAsset: new s3assets.Asset(this, "ScriptLocation", {
69+
path: `${__dirname}/../etl/transform.py`,
70+
}),
7171
});
7272
```
7373

@@ -87,17 +87,18 @@ _Parameters_
8787

8888
| **Name** | **Type** | **Description** |
8989
| :------------------ | :---------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
90-
| existingStreamObj? | [`kinesis.Stream`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesis.Stream.html) | Existing instance of Kinesis Stream, providing both this and `kinesisStreamProps` will cause an error. |
90+
| existingStreamObj? | [`kinesis.Stream`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesis.Stream.html) | Existing instance of Kinesis Stream, providing both this and `kinesisStreamProps` will cause an error. |
9191
| kinesisStreamProps? | [`kinesis.StreamProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesis.StreamProps.html) | Optional user-provided props to override the default props for the Kinesis stream. |
9292
| glueJobProps? | [`cfnJob.CfnJobProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnJobProps.html) | User provided props to override the default props for the AWS Glue Job. |
93-
| existingGlueJob? | [`cfnJob.CfnJob`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnJob.html) | Existing instance of AWS Glue Job, providing both this and `glueJobProps` will cause an error. |
93+
| existingGlueJob? | [`cfnJob.CfnJob`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnJob.html) | Existing instance of AWS Glue Job, providing both this and `glueJobProps` will cause an error. |
9494
| fieldSchema? | [`CfnTable.ColumnProperty[]`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnTable.ColumnProperty.html) | User provided schema structure to create an AWS Glue Table. |
9595
| existingTable? | [`CfnTable`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnTable.html) | Existing instance of AWS Glue Table. If this is set, tableProps and fieldSchema are ignored. |
9696
| tableProps? | [`CfnTableProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.TableProps.html) | User provided AWS Glue Table props to override default props used to create a Glue Table. |
9797
| existingDatabase? | [`CfnDatabase`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnDatabase.html) | Existing instance of AWS Glue Database. If this is set, then databaseProps is ignored. |
9898
| databaseProps? | [`CfnDatabaseProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnDatabaseProps.html) | User provided Glue Database Props to override the default props used to create the Glue Database. |
9999
| outputDataStore? | [`SinkDataStoreProps`](#sinkdatastoreprops) | User provided properties for S3 bucket that stores Glue Job output. Current datastore types suported is only S3. |
100100
|createCloudWatchAlarms?|`boolean`|Whether to create recommended CloudWatch alarms for Kinesis Data Stream. Default value is set to `true`.|
101+
| etlCodeAsset? | [s3assets.Asset](https://docs.aws.amazon.com/cdk/api/v1/docs/@aws-cdk_aws-s3-assets.Asset.html) | User provided instance of the Asset class that represents the ETL code on the local filesytem |
101102

102103
### SinkDataStoreProps
103104

‎source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/lib/index.ts

Lines changed: 93 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,14 @@
1111
* and limitations under the License.
1212
*/
1313

14-
import * as glue from '@aws-cdk/aws-glue';
15-
import { Effect, IRole, Policy, PolicyStatement } from '@aws-cdk/aws-iam';
16-
import { Stream, StreamProps } from '@aws-cdk/aws-kinesis';
17-
import { Bucket } from '@aws-cdk/aws-s3';
18-
import { Aws, Construct } from '@aws-cdk/core';
19-
import * as cloudwatch from '@aws-cdk/aws-cloudwatch';
20-
import * as defaults from '@aws-solutions-constructs/core';
14+
import * as glue from "@aws-cdk/aws-glue";
15+
import * as s3assets from "@aws-cdk/aws-s3-assets";
16+
import { Effect, IRole, Policy, PolicyStatement } from "@aws-cdk/aws-iam";
17+
import { Stream, StreamProps } from "@aws-cdk/aws-kinesis";
18+
import { Bucket } from "@aws-cdk/aws-s3";
19+
import { Aws, Construct } from "@aws-cdk/core";
20+
import * as cloudwatch from "@aws-cdk/aws-cloudwatch";
21+
import * as defaults from "@aws-solutions-constructs/core";
2122

2223
export interface KinesisstreamsToGluejobProps {
2324
/**
@@ -45,7 +46,19 @@ export interface KinesisstreamsToGluejobProps {
4546
*/
4647
readonly glueJobProps?: glue.CfnJobProps | any;
4748
/**
48-
* Existing GlueJob configuration. If this property is provided, any properties provided through @glueJobProps is ignored
49+
* Existing GlueJob configuration. If this property is provided, any properties provided through @glueJobProps is ignored.
50+
* The ETL script can be provided either under glue.CfnJob.JobCommandProperty or set as an Asset instance under
51+
* @KinesisstreamsToGluejobProps.etlCodeAsset.
52+
*
53+
* If an S3 location is know and exists, provide the S3 url in the `scriptLocation` attribute in glue.CfnJob.JobCommandProperty as an
54+
* S3 format URL (example: `s3://bucketname/keyprefix.py`)
55+
*
56+
* If the ETL script exists as a local files or directories, create an instance of the Asset (aws-cdk-lib » aws_s3_assets) class
57+
* set the @KinesisstreamsToGluejobProps.etlCodeAsset.
58+
*
59+
* In case both (JobCommandProperty.scriptLocation and @KinesisstreamsToGluejobProps.etlCodeAsset) are set,
60+
* @KinesisstreamsToGluejobProps.etlCodeAsset will take higher precedence and override the JobCommandProperty.scriptLocation
61+
*
4962
*/
5063
readonly existingGlueJob?: glue.CfnJob;
5164
/**
@@ -71,7 +84,7 @@ export interface KinesisstreamsToGluejobProps {
7184
*
7285
* @default - None
7386
*/
74-
readonly fieldSchema?: glue.CfnTable.ColumnProperty [];
87+
readonly fieldSchema?: glue.CfnTable.ColumnProperty[];
7588
/**
7689
* Glue Table for this construct, If not provided the construct will create a new Table in the
7790
* database. This table should define the schema for the records in the Kinesis Data Streams.
@@ -107,6 +120,17 @@ export interface KinesisstreamsToGluejobProps {
107120
* @default - Alarms are created
108121
*/
109122
readonly createCloudWatchAlarms?: boolean;
123+
/**
124+
* Provide Asset instance corresponding to the code in the local filesystem, responsible for
125+
* performing the Glue Job transformation. This property will override any S3 locations provided
126+
* under glue.CfnJob.JobCommandProperty
127+
*
128+
* As of CDK V2, all ETL scripts sourced from local code should explicitly create an asset and provide
129+
* that asset through this attribute.
130+
*
131+
* @default - None
132+
*/
133+
readonly etlCodeAsset?: s3assets.Asset;
110134
}
111135

112136
/**
@@ -145,6 +169,28 @@ export class KinesisstreamsToGluejob extends Construct {
145169
super(scope, id);
146170
defaults.CheckProps(props);
147171

172+
// custom props check
173+
if (props.existingGlueJob && props.glueJobProps) {
174+
throw Error("Either existingGlueJob instance or glueJobProps should be set, but found both");
175+
}
176+
177+
if (!props.existingGlueJob) {
178+
if (!props.glueJobProps.command.scriptLocation && !props.etlCodeAsset) {
179+
throw Error('Either one of CfnJob.JobCommandProperty.scriptLocation or KinesisstreamsToGluejobProps.etlCodeAsset has ' +
180+
'to be provided. If the ETL Job code file exists in a local filesystem, please set ' +
181+
'KinesisstreamsToGluejobProps.etlCodeAsset. If the ETL Job is available in an S3 bucket, set the ' +
182+
'CfnJob.JobCommandProperty.scriptLocation property');
183+
}
184+
185+
if (!props.etlCodeAsset) {
186+
const s3Url: string = props.glueJobProps.command.scriptLocation;
187+
const found = s3Url.match(/^s3:\/\/\S+\/\S+/g);
188+
if (!(found && found.length > 0 && found[0].length === s3Url.length)) {
189+
throw Error("Invalid S3 URL provided");
190+
}
191+
}
192+
}
193+
148194
this.kinesisStream = defaults.buildKinesisStream(this, {
149195
existingStreamObj: props.existingStreamObj,
150196
kinesisStreamProps: props.kinesisStreamProps,
@@ -153,23 +199,24 @@ export class KinesisstreamsToGluejob extends Construct {
153199
this.database = props.existingDatabase !== undefined ? props.existingDatabase : defaults.createGlueDatabase(scope, props.databaseProps);
154200

155201
if (props.fieldSchema === undefined && props.existingTable === undefined && props.tableProps === undefined) {
156-
throw Error('Either fieldSchema or table property has to be set, both cannot be optional');
202+
throw Error("Either fieldSchema or table property has to be set, both cannot be optional");
157203
}
158204

159205
if (props.existingTable !== undefined) {
160206
this.table = props.existingTable;
161207
} else {
162-
this.table = defaults.createGlueTable(scope, this.database, props.tableProps, props.fieldSchema, 'kinesis', {
163-
STREAM_NAME: this.kinesisStream.streamName
208+
this.table = defaults.createGlueTable(scope, this.database, props.tableProps, props.fieldSchema, "kinesis", {
209+
STREAM_NAME: this.kinesisStream.streamName,
164210
});
165211
}
166212

167-
[ this.glueJob, this.glueJobRole, this.outputBucket ] = defaults.buildGlueJob(this, {
213+
[this.glueJob, this.glueJobRole, this.outputBucket] = defaults.buildGlueJob(this, {
168214
existingCfnJob: props.existingGlueJob,
169215
glueJobProps: props.glueJobProps,
170216
table: this.table!,
171217
database: this.database!,
172-
outputDataStore: props.outputDataStore!
218+
outputDataStore: props.outputDataStore!,
219+
etlCodeAsset: props.etlCodeAsset
173220
});
174221

175222
this.glueJobRole = this.buildRolePolicy(scope, id, this.database, this.table, this.glueJob, this.glueJobRole);
@@ -194,47 +241,54 @@ export class KinesisstreamsToGluejob extends Construct {
194241
const _glueJobPolicy = new Policy(scope, `${id}GlueJobPolicy`, {
195242
statements: [ new PolicyStatement({
196243
effect: Effect.ALLOW,
197-
actions: [ 'glue:GetJob' ],
198-
resources: [ `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:job/${glueJob.ref}` ]
199-
}), new PolicyStatement({
244+
actions: ["glue:GetJob"],
245+
resources: [
246+
`arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:job/${glueJob.ref}`,
247+
],
248+
}),
249+
new PolicyStatement({
200250
effect: Effect.ALLOW,
201-
actions: [ 'glue:GetSecurityConfiguration' ],
202-
resources: [ '*' ] // Security Configurations have no resource ARNs
203-
}), new PolicyStatement({
251+
actions: ["glue:GetSecurityConfiguration"],
252+
resources: ["*"], // Security Configurations have no resource ARNs
253+
}),
254+
new PolicyStatement({
204255
effect: Effect.ALLOW,
205-
actions: [ 'glue:GetTable' ],
206-
resources: [ `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:table/${glueDatabase.ref}/${glueTable.ref}`,
256+
actions: ["glue:GetTable"],
257+
resources: [
258+
`arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:table/${glueDatabase.ref}/${glueTable.ref}`,
207259
`arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:database/${glueDatabase.ref}`,
208-
`arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:catalog`
209-
]
210-
}), new PolicyStatement({
260+
`arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:catalog`,
261+
],
262+
}),
263+
new PolicyStatement({
211264
effect: Effect.ALLOW,
212-
actions: [ 'cloudwatch:PutMetricData' ],
213-
resources: [ '*' ], // Metrics do not have resource ARN and hence added conditions
265+
actions: ["cloudwatch:PutMetricData"],
266+
resources: ["*"], // Metrics do not have resource ARN and hence added conditions
214267
conditions: {
215268
StringEquals: {
216-
"cloudwatch:namespace": "Glue"
269+
"cloudwatch:namespace": "Glue",
217270
},
218271
Bool: {
219-
"aws:SecureTransport": "true"
220-
}
221-
}
222-
}), new PolicyStatement({
272+
"aws:SecureTransport": "true",
273+
},
274+
},
275+
}),
276+
new PolicyStatement({
223277
effect: Effect.ALLOW,
224-
actions: [ 'kinesis:DescribeStream', 'kinesis:DescribeStreamSummary', 'kinesis:GetRecords',
225-
'kinesis:GetShardIterator', 'kinesis:ListShards', 'kinesis:SubscribeToShard' ],
226-
resources: [ this.kinesisStream.streamArn ]
227-
})]
278+
actions: [ "kinesis:DescribeStream", "kinesis:DescribeStreamSummary", "kinesis:GetRecords",
279+
"kinesis:GetShardIterator", "kinesis:ListShards", "kinesis:SubscribeToShard" ],
280+
resources: [this.kinesisStream.streamArn],
281+
})],
228282
});
229283

230284
defaults.addCfnSuppressRules(_glueJobPolicy, [
231285
{
232-
id: 'W12',
233-
reason: "Glue Security Configuration does not have an ARN, and the policy only allows reading the configuration. CloudWatch metrics also do not have an ARN but adding a namespace condition to the policy to allow it to publish metrics only for AWS Glue"
286+
id: "W12",
287+
reason: "Glue Security Configuration does not have an ARN, and the policy only allows reading the configuration. CloudWatch metrics also do not have an ARN but adding a namespace condition to the policy to allow it to publish metrics only for AWS Glue",
234288
},
235289
]);
236290

237291
role.attachInlinePolicy(_glueJobPolicy);
238292
return role;
239293
}
240-
}
294+
}

‎source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/test/fakefile.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# empty python file for testing

0 commit comments

Comments
 (0)