From 64f256aaed6b08f7285f4abd92a1a5e2694aad32 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Wed, 24 Jan 2024 22:37:25 -0800 Subject: [PATCH 1/6] Add S3 Batch Operations 2.0 Schema Fields --- events/s3_batch_job.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/events/s3_batch_job.go b/events/s3_batch_job.go index f2626edd..d7ed142a 100644 --- a/events/s3_batch_job.go +++ b/events/s3_batch_job.go @@ -12,7 +12,8 @@ type S3BatchJobEvent struct { // S3BatchJob whichs have the job id type S3BatchJob struct { - ID string `json:"id"` + ID string `json:"id"` + UserArguments map[string]string `json:"userArguments"` } // S3BatchJobTask represents one task in the s3 batch job and have all task details @@ -20,6 +21,7 @@ type S3BatchJobTask struct { TaskID string `json:"taskId"` S3Key string `json:"s3Key"` S3VersionID string `json:"s3VersionId"` + S3Bucket string `json:"s3Bucket"` S3BucketARN string `json:"s3BucketArn"` } From 4b16f6a1aae113a5bf28373edd587ea373cf673a Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Wed, 24 Jan 2024 22:38:20 -0800 Subject: [PATCH 2/6] Create s3-batch-job-event-request-2.0.json --- .../s3-batch-job-event-request-2.0.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 events/testdata/s3-batch-job-event-request-2.0.json diff --git a/events/testdata/s3-batch-job-event-request-2.0.json b/events/testdata/s3-batch-job-event-request-2.0.json new file mode 100644 index 00000000..014c4da6 --- /dev/null +++ b/events/testdata/s3-batch-job-event-request-2.0.json @@ -0,0 +1,19 @@ +{ + "invocationSchemaVersion": "2.0", + "invocationId": "YXNkbGZqYWRmaiBhc2RmdW9hZHNmZGpmaGFzbGtkaGZza2RmaAo", + "job": { + "id": "f3cc4f60-61f6-4a2b-8a21-d07600c373ce", + "userArguments": { + "k1": "v1", + "k2": "v2" + } + }, + "tasks": [ + { + "taskId": "dGFza2lkZ29lc2hlcmUK", + "s3Key": "prefix/dataset/dataset.20231222.json.gz", + "s3VersionId": null, + "s3Bucket": "powertools-dataset" + } + ] +} From 83ff1fffa1c256256016a30f557650f781cfe8e9 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Wed, 24 Jan 2024 22:39:49 -0800 Subject: [PATCH 3/6] Update s3_batch_job_test.go --- events/s3_batch_job_test.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/events/s3_batch_job_test.go b/events/s3_batch_job_test.go index 802d38b2..fd14ddd4 100644 --- a/events/s3_batch_job_test.go +++ b/events/s3_batch_job_test.go @@ -10,10 +10,30 @@ import ( "github.com/stretchr/testify/assert" ) -func TestS3BatchJobEventMarshaling(t *testing.T) { +func TestS3BatchJobEventMarshalingSchema1(t *testing.T) { // 1. read JSON from file - inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-batch-job-event-request.json") + inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-batch-job-event-request-1.0.json") + + // 2. de-serialize into Go object + var inputEvent S3BatchJobEvent + if err := json.Unmarshal(inputJSON, &inputEvent); err != nil { + t.Errorf("could not unmarshal event. details: %v", err) + } + + // 3. serialize to JSON + outputJSON, err := json.Marshal(inputEvent) + if err != nil { + t.Errorf("could not marshal event. details: %v", err) + } + + // 4. check result + assert.JSONEq(t, string(inputJSON), string(outputJSON)) +} + +func TestS3BatchJobEventMarshalingSchema2(t *testing.T) { + // 1. read JSON from file + inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-batch-job-event-request-2.0.json") // 2. de-serialize into Go object var inputEvent S3BatchJobEvent From d42db67feed2a4f81ff6bc2f89c2267d56b2e0d0 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Wed, 24 Jan 2024 22:40:05 -0800 Subject: [PATCH 4/6] Rename s3-batch-job-event-request.json to s3-batch-job-event-request-1.0.json --- ...b-event-request.json => s3-batch-job-event-request-1.0.json} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename events/testdata/{s3-batch-job-event-request.json => s3-batch-job-event-request-1.0.json} (99%) diff --git a/events/testdata/s3-batch-job-event-request.json b/events/testdata/s3-batch-job-event-request-1.0.json similarity index 99% rename from events/testdata/s3-batch-job-event-request.json rename to events/testdata/s3-batch-job-event-request-1.0.json index 210173f4..4c3038ec 100644 --- a/events/testdata/s3-batch-job-event-request.json +++ b/events/testdata/s3-batch-job-event-request-1.0.json @@ -12,4 +12,4 @@ "s3BucketArn": "arn:aws:s3:us-east-1:0123456788:awsexamplebucket" } ] -} \ No newline at end of file +} From 64a0c9ebc78cd18cb9a714d5fb670dccdcb50547 Mon Sep 17 00:00:00 2001 From: Luke Young Date: Wed, 24 Jan 2024 22:50:20 -0800 Subject: [PATCH 5/6] Fix tests --- events/s3_batch_job.go | 6 +++--- events/testdata/s3-batch-job-event-request-2.0.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/events/s3_batch_job.go b/events/s3_batch_job.go index d7ed142a..c84f45a2 100644 --- a/events/s3_batch_job.go +++ b/events/s3_batch_job.go @@ -13,7 +13,7 @@ type S3BatchJobEvent struct { // S3BatchJob whichs have the job id type S3BatchJob struct { ID string `json:"id"` - UserArguments map[string]string `json:"userArguments"` + UserArguments map[string]string `json:"userArguments,omitempty"` } // S3BatchJobTask represents one task in the s3 batch job and have all task details @@ -21,8 +21,8 @@ type S3BatchJobTask struct { TaskID string `json:"taskId"` S3Key string `json:"s3Key"` S3VersionID string `json:"s3VersionId"` - S3Bucket string `json:"s3Bucket"` - S3BucketARN string `json:"s3BucketArn"` + S3Bucket string `json:"s3Bucket,omitempty"` + S3BucketARN string `json:"s3BucketArn,omitempty"` } // S3BatchJobResponse is the response of a iven s3 batch job with the results diff --git a/events/testdata/s3-batch-job-event-request-2.0.json b/events/testdata/s3-batch-job-event-request-2.0.json index 014c4da6..3e2c3d7a 100644 --- a/events/testdata/s3-batch-job-event-request-2.0.json +++ b/events/testdata/s3-batch-job-event-request-2.0.json @@ -12,7 +12,7 @@ { "taskId": "dGFza2lkZ29lc2hlcmUK", "s3Key": "prefix/dataset/dataset.20231222.json.gz", - "s3VersionId": null, + "s3VersionId": "1", "s3Bucket": "powertools-dataset" } ] From b94adbed0e47c0679922332463322e2872fcc0a6 Mon Sep 17 00:00:00 2001 From: Luke Young Date: Fri, 26 Jan 2024 19:53:45 -0800 Subject: [PATCH 6/6] Split into S3BatchJobV2 --- events/s3_batch_job.go | 28 ++++++++++++++++--- events/s3_batch_job_test.go | 6 ++-- .../s3-batch-job-event-request-2.0.json | 6 ++-- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/events/s3_batch_job.go b/events/s3_batch_job.go index c84f45a2..183ec907 100644 --- a/events/s3_batch_job.go +++ b/events/s3_batch_job.go @@ -12,8 +12,7 @@ type S3BatchJobEvent struct { // S3BatchJob whichs have the job id type S3BatchJob struct { - ID string `json:"id"` - UserArguments map[string]string `json:"userArguments,omitempty"` + ID string `json:"id"` } // S3BatchJobTask represents one task in the s3 batch job and have all task details @@ -21,8 +20,29 @@ type S3BatchJobTask struct { TaskID string `json:"taskId"` S3Key string `json:"s3Key"` S3VersionID string `json:"s3VersionId"` - S3Bucket string `json:"s3Bucket,omitempty"` - S3BucketARN string `json:"s3BucketArn,omitempty"` + S3BucketARN string `json:"s3BucketArn"` +} + +// S3BatchJobEventV2 encapsulates the detail of a s3 batch job +type S3BatchJobEventV2 struct { + InvocationSchemaVersion string `json:"invocationSchemaVersion"` + InvocationID string `json:"invocationId"` + Job S3BatchJobV2 `json:"job"` + Tasks []S3BatchJobTaskV2 `json:"tasks"` +} + +// S3BatchJobV2 whichs have the job id +type S3BatchJobV2 struct { + ID string `json:"id"` + UserArguments map[string]string `json:"userArguments"` +} + +// S3BatchJobTaskV2 represents one task in the s3 batch job and have all task details +type S3BatchJobTaskV2 struct { + TaskID string `json:"taskId"` + S3Key string `json:"s3Key"` + S3VersionID string `json:"s3VersionId"` + S3Bucket string `json:"s3Bucket"` } // S3BatchJobResponse is the response of a iven s3 batch job with the results diff --git a/events/s3_batch_job_test.go b/events/s3_batch_job_test.go index fd14ddd4..3067c256 100644 --- a/events/s3_batch_job_test.go +++ b/events/s3_batch_job_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestS3BatchJobEventMarshalingSchema1(t *testing.T) { +func TestS3BatchJobEventMarshaling(t *testing.T) { // 1. read JSON from file inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-batch-job-event-request-1.0.json") @@ -31,12 +31,12 @@ func TestS3BatchJobEventMarshalingSchema1(t *testing.T) { assert.JSONEq(t, string(inputJSON), string(outputJSON)) } -func TestS3BatchJobEventMarshalingSchema2(t *testing.T) { +func TestS3BatchJobEventV2Marshaling(t *testing.T) { // 1. read JSON from file inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-batch-job-event-request-2.0.json") // 2. de-serialize into Go object - var inputEvent S3BatchJobEvent + var inputEvent S3BatchJobEventV2 if err := json.Unmarshal(inputJSON, &inputEvent); err != nil { t.Errorf("could not unmarshal event. details: %v", err) } diff --git a/events/testdata/s3-batch-job-event-request-2.0.json b/events/testdata/s3-batch-job-event-request-2.0.json index 3e2c3d7a..738339cc 100644 --- a/events/testdata/s3-batch-job-event-request-2.0.json +++ b/events/testdata/s3-batch-job-event-request-2.0.json @@ -11,9 +11,9 @@ "tasks": [ { "taskId": "dGFza2lkZ29lc2hlcmUK", - "s3Key": "prefix/dataset/dataset.20231222.json.gz", - "s3VersionId": "1", - "s3Bucket": "powertools-dataset" + "s3Key": "customerImage1.jpg", + "s3VersionId": "jbo9_jhdPEyB4RrmOxWS0kU0EoNrU_oI", + "s3Bucket": "awsexamplebucket" } ] }