Fix(Set outputBucket property on aws-kinesisstreams-gluejob): Issue #448 to include S3 bucket for Glue Job that the consturct creates (#452)

* update glue max allocation units to 2

* update glue max allocation units to 2

* resolving workerType and maxCapacity config related rules

* resolving workerType and maxCapacity config related rules

* resolving workerType and maxCapacity config related rules

* removing additional lines from the file

* fix snapshots for glue helper

* fix snapshots for glue helper

* fix snapshots for glue helper

* fix snapshots for glue helper

* fix snapshots for glue helper

* fix snapshots for glue helper

* documentation rephrase

* new construct first commit with README and architecture

* adding minimal deployable code to README

* adding minimal deployable code to README

* fix for github issue #448

Co-authored-by: nihitkas <[email protected]>
Co-authored-by: biffgaut <[email protected]>

Author: knihit

source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/lib/index.ts

@@ -126,6 +126,11 @@ export class KinesisstreamsToGluejob extends Construct {
   public readonly glueJobRole: IRole;
   public readonly database: glue.CfnDatabase;
   public readonly table: glue.CfnTable;
+  /**
+   * This property is only set if the Glue Job is created by the construct. If an exisiting Glue Job
+   * configuraton is supplied, the construct does not create an S3 bucket and hence the @outputBucket
+   * property is undefined
+   */
   public readonly outputBucket?: [Bucket, (Bucket | undefined)?];
   public readonly cloudwatchAlarms?: cloudwatch.Alarm[];
 
@@ -159,7 +164,7 @@ export class KinesisstreamsToGluejob extends Construct {
       });
     }
 
-    [ this.glueJob, this.glueJobRole ] = defaults.buildGlueJob(this, {
+    [ this.glueJob, this.glueJobRole, this.outputBucket ] = defaults.buildGlueJob(this, {
       existingCfnJob: props.existingGlueJob,
       glueJobProps: props.glueJobProps,
       table: this.table!,

source/patterns/@aws-solutions-constructs/core/lib/glue-job-helper.ts

@@ -78,7 +78,7 @@ export interface BuildGlueJobProps {
   readonly outputDataStore?: SinkDataStoreProps
 }
 
-export function buildGlueJob(scope: Construct, props: BuildGlueJobProps): [glue.CfnJob, IRole] {
+export function buildGlueJob(scope: Construct, props: BuildGlueJobProps): [glue.CfnJob, IRole, [Bucket, (Bucket | undefined)?]?] {
   if (!props.existingCfnJob) {
     if (props.glueJobProps) {
       if (props.glueJobProps.glueVersion === '2.0' && props.glueJobProps.maxCapacity) {
@@ -101,7 +101,7 @@ export function buildGlueJob(scope: Construct, props: BuildGlueJobProps): [glue.
 }
 
 export function deployGlueJob(scope: Construct, glueJobProps: glue.CfnJobProps, database: glue.CfnDatabase, table: glue.CfnTable,
-  outputDataStore: SinkDataStoreProps): [glue.CfnJob, IRole] {
+  outputDataStore: SinkDataStoreProps): [glue.CfnJob, IRole, [Bucket, (Bucket | undefined)?]] {
 
   let _glueSecurityConfigName: string;
 
@@ -183,7 +183,7 @@ export function deployGlueJob(scope: Construct, glueJobProps: glue.CfnJobProps,
   _scriptBucketLocation.grantRead(_jobRole);
 
   const _glueJob: glue.CfnJob = new glue.CfnJob(scope, 'KinesisETLJob', _newGlueJobProps);
-  return [_glueJob, _jobRole];
+  return [_glueJob, _jobRole, _outputLocation];
 }
 
 /**

source/patterns/@aws-solutions-constructs/core/test/glue-job-helper.test.ts

@@ -42,7 +42,7 @@ test('Test deployment with role creation', () => {
 
   const _database = defaults.createGlueDatabase(stack, defaults.DefaultGlueDatabaseProps());
 
-  defaults.buildGlueJob(stack, {
+  const _glueJob = defaults.buildGlueJob(stack, {
     glueJobProps: cfnJobProps,
     database: _database,
     table: defaults.createGlueTable(stack, _database, undefined, [{
@@ -52,6 +52,8 @@ test('Test deployment with role creation', () => {
     }], 'kinesis', {STREAM_NAME: 'testStream'})
   });
 
+  expect(_glueJob[2]?.[0]).toBeDefined();
+  expect(_glueJob[2]?.[0]).toBeInstanceOf(Bucket);
   expect(stack).toHaveResourceLike('AWS::Glue::Job', {
     Type: "AWS::Glue::Job",
     Properties: {
@@ -99,7 +101,7 @@ test('Create a Glue Job outside the construct', () => {
 
   const _database = defaults.createGlueDatabase(stack, defaults.DefaultGlueDatabaseProps());
 
-  defaults.buildGlueJob(stack, {
+  const _glueJob = defaults.buildGlueJob(stack, {
     existingCfnJob: _existingCfnJob,
     outputDataStore: {
       datastoreType: defaults.SinkStoreType.S3
@@ -111,6 +113,8 @@ test('Create a Glue Job outside the construct', () => {
       comment: ""
     }], 'kinesis', {STREAM_NAME: 'testStream'})
   });
+
+  expect(_glueJob[2]).not.toBeDefined();
   expect(stack).toHaveResourceLike('AWS::Glue::Job', {
     Type: "AWS::Glue::Job",
     Properties: {