Skip to content

Commit 4780720

Browse files
authored
Pipeline unit tests (#181)
* initial pipeline resource * check for last-modified time in update and re-order generator-file * add tags for integration test * disable pipelinedefinitionS3location * use create model step: * add unit-tests for pipeline
1 parent c0e559a commit 4780720

17 files changed

+808
-0
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package pipeline
15+
16+
import (
17+
"errors"
18+
"fmt"
19+
20+
"path/filepath"
21+
"testing"
22+
23+
ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1"
24+
ackmetrics "github.com/aws-controllers-k8s/runtime/pkg/metrics"
25+
acktypes "github.com/aws-controllers-k8s/runtime/pkg/types"
26+
svcapitypes "github.com/aws-controllers-k8s/sagemaker-controller/apis/v1alpha1"
27+
"github.com/aws-controllers-k8s/sagemaker-controller/pkg/testutil"
28+
mocksvcsdkapi "github.com/aws-controllers-k8s/sagemaker-controller/test/mocks/aws-sdk-go/sagemaker"
29+
svcsdk "github.com/aws/aws-sdk-go/service/sagemaker"
30+
"github.com/google/go-cmp/cmp"
31+
"github.com/google/go-cmp/cmp/cmpopts"
32+
"go.uber.org/zap/zapcore"
33+
ctrlrtzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
34+
)
35+
36+
// provideResourceManagerWithMockSDKAPI accepts MockSageMakerAPI and returns pointer to resourceManager
37+
// the returned resourceManager is configured to use mockapi api.
38+
func provideResourceManagerWithMockSDKAPI(mockSageMakerAPI *mocksvcsdkapi.SageMakerAPI) *resourceManager {
39+
zapOptions := ctrlrtzap.Options{
40+
Development: true,
41+
Level: zapcore.InfoLevel,
42+
}
43+
fakeLogger := ctrlrtzap.New(ctrlrtzap.UseFlagOptions(&zapOptions))
44+
return &resourceManager{
45+
rr: nil,
46+
awsAccountID: "",
47+
awsRegion: "",
48+
sess: nil,
49+
sdkapi: mockSageMakerAPI,
50+
log: fakeLogger,
51+
metrics: ackmetrics.NewMetrics("sagemaker"),
52+
}
53+
}
54+
55+
// TestPipelineTestSuite runs the test suite for pipeline
56+
func TestPipelineTestSuite(t *testing.T) {
57+
var ts = testutil.TestSuite{}
58+
testutil.LoadFromFixture(filepath.Join("testdata", "test_suite.yaml"), &ts)
59+
var delegate = testRunnerDelegate{t: t}
60+
var runner = testutil.TestSuiteRunner{TestSuite: &ts, Delegate: &delegate}
61+
runner.RunTests()
62+
}
63+
64+
// testRunnerDelegate implements testutil.TestRunnerDelegate
65+
type testRunnerDelegate struct {
66+
t *testing.T
67+
}
68+
69+
func (d *testRunnerDelegate) ResourceDescriptor() acktypes.AWSResourceDescriptor {
70+
return &resourceDescriptor{}
71+
}
72+
73+
func (d *testRunnerDelegate) ResourceManager(mocksdkapi *mocksvcsdkapi.SageMakerAPI) acktypes.AWSResourceManager {
74+
return provideResourceManagerWithMockSDKAPI(mocksdkapi)
75+
}
76+
77+
func (d *testRunnerDelegate) GoTestRunner() *testing.T {
78+
return d.t
79+
}
80+
81+
func (d *testRunnerDelegate) EmptyServiceAPIOutput(apiName string) (interface{}, error) {
82+
if apiName == "" {
83+
return nil, errors.New("no API name specified")
84+
}
85+
//TODO: use reflection, template to auto generate this block/method.
86+
switch apiName {
87+
case "CreatePipelineWithContext":
88+
var output svcsdk.CreatePipelineOutput
89+
return &output, nil
90+
case "DescribePipelineWithContext":
91+
var output svcsdk.DescribePipelineOutput
92+
return &output, nil
93+
case "DeletePipelineWithContext":
94+
var output svcsdk.DeletePipelineOutput
95+
return &output, nil
96+
case "UpdatePipelineWithContext":
97+
var output svcsdk.UpdatePipelineOutput
98+
return &output, nil
99+
}
100+
return nil, errors.New(fmt.Sprintf("no matching API name found for: %s", apiName))
101+
}
102+
103+
func (d *testRunnerDelegate) Equal(a acktypes.AWSResource, b acktypes.AWSResource) bool {
104+
ac := a.(*resource)
105+
bc := b.(*resource)
106+
// Ignore LastTransitionTime since it gets updated each run.
107+
opts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(ackv1alpha1.Condition{}, "LastTransitionTime"),
108+
cmpopts.IgnoreFields(svcapitypes.PipelineStatus{}, "CreationTime"),
109+
cmpopts.IgnoreFields(svcapitypes.PipelineStatus{}, "LastModifiedTime")}
110+
111+
var specMatch = false
112+
if cmp.Equal(ac.ko.Spec, bc.ko.Spec, opts...) {
113+
specMatch = true
114+
} else {
115+
fmt.Printf("Difference ko.Spec (-expected +actual):\n\n")
116+
fmt.Println(cmp.Diff(ac.ko.Spec, bc.ko.Spec, opts...))
117+
specMatch = false
118+
}
119+
120+
var statusMatch = false
121+
if cmp.Equal(ac.ko.Status, bc.ko.Status, opts...) {
122+
statusMatch = true
123+
} else {
124+
fmt.Printf("Difference ko.Status (-expected +actual):\n\n")
125+
fmt.Println(cmp.Diff(ac.ko.Status, bc.ko.Status, opts...))
126+
statusMatch = false
127+
}
128+
return statusMatch && specMatch
129+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"PipelineArn": "arn:aws:sagemaker:us-west-2:123456789012:pipeline/test-pipeline"
3+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"PipelineArn": "arn:aws:sagemaker:us-west-2:123456789:pipeline/test-pipeline",
3+
"PipelineName": "test-pipeline",
4+
"PipelineDisplayName": "test-pipeline",
5+
"PipelineDefinition": "{\"Version\": \"2020-12-01\", \"Metadata\": {}, \"Parameters\": [{\"Name\": \"ProcessingInstanceType\", \"Type\": \"String\", \"DefaultValue\": \"ml.m5.xlarge\"}, {\"Name\": \"ProcessingInstanceCount\", \"Type\": \"Integer\", \"DefaultValue\": 1}, {\"Name\": \"InputData\", \"Type\": \"String\", \"DefaultValue\": \"s3://sagemaker-us-west-2-123456789/pipeline-model-example/data/raw\"}], \"PipelineExperimentConfig\": {\"ExperimentName\": {\"Get\": \"Execution.PipelineName\"}, \"TrialName\": {\"Get\": \"Execution.PipelineExecutionId\"}}, \"Steps\": [{\"Name\": \"PreprocessData\", \"Type\": \"Processing\", \"Arguments\": {\"ProcessingResources\": {\"ClusterConfig\": {\"InstanceType\": \"ml.m5.large\", \"InstanceCount\": {\"Get\": \"Parameters.ProcessingInstanceCount\"}, \"VolumeSizeInGB\": 30}}, \"AppSpecification\": {\"ImageUri\": \"257758044811.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:0.23-1-cpu-py3\", \"ContainerEntrypoint\": [\"python3\", \"/opt/ml/processing/input/code/preprocess.py\"]}, \"RoleArn\": \"arn:aws:iam::123456789:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole\", \"ProcessingInputs\": [{\"InputName\": \"input-1\", \"AppManaged\": false, \"S3Input\": {\"S3Uri\": {\"Get\": \"Parameters.InputData\"}, \"LocalPath\": \"/opt/ml/processing/input\", \"S3DataType\": \"S3Prefix\", \"S3InputMode\": \"File\", \"S3DataDistributionType\": \"FullyReplicated\", \"S3CompressionType\": \"None\"}}, {\"InputName\": \"code\", \"AppManaged\": false, \"S3Input\": {\"S3Uri\": \"s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-21-21-843/input/code/preprocess.py\", \"LocalPath\": \"/opt/ml/processing/input/code\", \"S3DataType\": \"S3Prefix\", \"S3InputMode\": \"File\", \"S3DataDistributionType\": \"FullyReplicated\", \"S3CompressionType\": \"None\"}}], \"ProcessingOutputConfig\": {\"Outputs\": [{\"OutputName\": \"scaler_model\", \"AppManaged\": false, \"S3Output\": {\"S3Uri\": \"s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-21-21-843/output/scaler_model\", \"LocalPath\": \"/opt/ml/processing/scaler_model\", \"S3UploadMode\": \"EndOfJob\"}}, {\"OutputName\": \"train\", \"AppManaged\": false, \"S3Output\": {\"S3Uri\": \"s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-21-21-843/output/train\", \"LocalPath\": \"/opt/ml/processing/train\", \"S3UploadMode\": \"EndOfJob\"}}, {\"OutputName\": \"test\", \"AppManaged\": false, \"S3Output\": {\"S3Uri\": \"s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-21-21-843/output/test\", \"LocalPath\": \"/opt/ml/processing/test\", \"S3UploadMode\": \"EndOfJob\"}}]}}}]}",
6+
"RoleArn": "arn:aws:iam::123456789:role/ack-sagemaker-execution-role-123456789",
7+
"PipelineStatus": "Active",
8+
"CreationTime": "2022-11-21T18:02:48.242000-08:00",
9+
"LastModifiedTime": "2022-11-21T18:02:48.242000-08:00",
10+
"CreatedBy": {},
11+
"LastModifiedBy": {},
12+
"ParallelismConfiguration": {
13+
"MaxParallelExecutionSteps": 2
14+
}
15+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
tests:
2+
- name: "Pipeline create tests"
3+
description: "Part of Pipeline CRD tests."
4+
scenarios:
5+
- name: "Create=InvalidInput"
6+
description: "Given one of the parameters is invalid, Status shows a terminal condition"
7+
given:
8+
desired_state: "v1alpha1/create/desired/invalid_before_create.yaml"
9+
svc_api:
10+
- operation: CreatePipelineWithContext
11+
error:
12+
code: InvalidParameterValue
13+
message: "The Pipeline name must not include a special character."
14+
invoke: Create
15+
expect:
16+
latest_state: "v1alpha1/create/observed/invalid_create_attempted.yaml"
17+
error: resource is in terminal condition
18+
- name: "Create=Valid"
19+
description: "Create a new Pipeline successfully (ARN in status)."
20+
given:
21+
desired_state: "v1alpha1/create/desired/success_before_create.yaml"
22+
svc_api:
23+
- operation: CreatePipelineWithContext
24+
output_fixture: "sdkapi/create/create_success.json"
25+
invoke: Create
26+
expect:
27+
latest_state: "v1alpha1/create/observed/success_after_create.yaml"
28+
error: nil
29+
- name: "Pipeline readOne tests"
30+
description: "Testing the readOne operation"
31+
scenarios:
32+
- name: "ReadOne=MissingRequiredField"
33+
description: "Testing readOne when required field is missing. No API call is made and returns error."
34+
given:
35+
desired_state: "v1alpha1/readone/desired/missing_required_field.yaml"
36+
invoke: ReadOne
37+
expect:
38+
error: "resource not found"
39+
- name: "ReadOne=NotFound"
40+
description: "Testing readOne when Describe fails to find the resource on SageMaker"
41+
given:
42+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
43+
svc_api:
44+
- operation: DescribePipelineWithContext
45+
error:
46+
code: ResourceNotFound
47+
message: "does not exist"
48+
invoke: ReadOne
49+
expect:
50+
error: "resource is in terminal condition"
51+
- name: "ReadOne=Fail"
52+
description: "This test checks if the condition is updated if describe fails and readOne returns error"
53+
given:
54+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
55+
svc_api:
56+
- operation: DescribePipelineWithContext
57+
error:
58+
code: ServiceUnavailable
59+
message: "Server is down"
60+
invoke: ReadOne
61+
expect:
62+
latest_state: "v1alpha1/readone/observed/error_on_describe.yaml"
63+
error: "ServiceUnavailable: Server is down\n\tstatus code: 0, request id: "
64+
- name: "ReadOne=AfterCreate"
65+
description: "Testing readOne after create, the status should have ARN."
66+
given:
67+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
68+
svc_api:
69+
- operation: DescribePipelineWithContext
70+
output_fixture: "sdkapi/describe/describe_success.json"
71+
invoke: ReadOne
72+
expect:
73+
latest_state: "v1alpha1/readone/observed/created.yaml"
74+
- name: "ReadOne=SuccessClearsConditions"
75+
description: "Testing a successful reconciliation clears conditions if terminal/recoverable condition were already set to true"
76+
given:
77+
desired_state: "v1alpha1/readone/desired/error_conditions_true.yaml"
78+
svc_api:
79+
- operation: DescribePipelineWithContext
80+
output_fixture: "sdkapi/describe/describe_success.json"
81+
invoke: ReadOne
82+
expect:
83+
latest_state: "v1alpha1/readone/observed/conditions_clear_on_success.yaml"
84+
- name: "Pipeline update tests"
85+
description: "Testing the Update operation"
86+
scenarios:
87+
- name: "Update=Success"
88+
description: "This test checks if the Pipeline is updated sucessfully"
89+
given:
90+
desired_state: "v1alpha1/update/desired/updated.yaml"
91+
latest_state: "v1alpha1/readone/observed/created.yaml"
92+
svc_api:
93+
- operation: UpdatePipelineWithContext
94+
output_fixture: "sdkapi/create/create_success.json"
95+
invoke: Update
96+
expect:
97+
latest_state: "v1alpha1/update/observed/update_pipeline.yaml"
98+
error: nil
99+
- name: "Pipeline delete tests"
100+
description: "Testing the delete operation"
101+
scenarios:
102+
- name: "Delete=Fail"
103+
description: "This test checks if the condition is updated if delete fails and returns error"
104+
given:
105+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
106+
svc_api:
107+
- operation: DeletePipelineWithContext
108+
error:
109+
code: ServiceUnavailable
110+
message: "Server is down"
111+
invoke: Delete
112+
expect:
113+
latest_state: "v1alpha1/delete/observed/error_on_delete.yaml"
114+
error: "ServiceUnavailable: Server is down\n\tstatus code: 0, request id: "
115+
- name: "Delete=FailNotFound"
116+
description: "This test checks if delete fails and returns error not found"
117+
given:
118+
desired_state: "v1alpha1/readone/desired/error_conditions_true.yaml"
119+
svc_api:
120+
- operation: DeletePipelineWithContext
121+
error:
122+
code: ResourceNotFound
123+
message: "does not exist."
124+
invoke: Delete
125+
expect:
126+
latest_state: "v1alpha1/delete/observed/not_found_on_delete.yaml"
127+
- name: "Delete=Successful"
128+
description: "This test checks if the Pipeline is deleted successfully"
129+
given:
130+
desired_state: "v1alpha1/create/observed/success_after_create.yaml"
131+
svc_api:
132+
- operation: DeletePipelineWithContext
133+
- operation: DescribePipelineWithContext
134+
error:
135+
code: ResourceNotFound
136+
message: "does not exist."
137+
invoke: Delete
138+
expect:
139+
error: nil
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: sagemaker.services.k8s.aws/v1alpha1
2+
kind: Pipeline
3+
metadata:
4+
name: test-pipeline
5+
spec:
6+
pipelineDisplayName: test-pipeline
7+
pipelineName: test-pipeline
8+
pipelineDefinition: '{"Version": "2020-12-01", "Metadata": {}, "Parameters": [{"Name": "ProcessingInstanceType", "Type": "String", "DefaultValue": "ml.m5.xlarge"}, {"Name": "ProcessingInstanceCount", "Type": "Integer", "DefaultValue": 1}, {"Name": "InputData", "Type": "String", "DefaultValue": "s3://sagemaker-us-west-2-123456789/pipeline-model-example/data/raw"}], "PipelineExperimentConfig": {"ExperimentName": {"Get": "Execution.PipelineName"}, "TrialName": {"Get": "Execution.PipelineExecutionId"}}, "Steps": [{"Name": "PreprocessData", "Type": "Processing", "Arguments": {"ProcessingResources": {"ClusterConfig": {"InstanceType": "ml.m5.large", "InstanceCount": {"Get": "Parameters.ProcessingInstanceCount"}, "VolumeSizeInGB": 30}}, "AppSpecification": {"ImageUri": "246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:0.23-1-cpu-py3", "ContainerEntrypoint": ["python3", "/opt/ml/processing/input/code/preprocess.py"]},
9+
"RoleArn": "arn:aws:iam::123456789:role/ack-sagemaker-execution-role-123456789", "ProcessingInputs": [{"InputName": "input-1", "AppManaged": false, "S3Input": {"S3Uri": {"Get": "Parameters.InputData"}, "LocalPath": "/opt/ml/processing/input", "S3DataType": "S3Prefix", "S3InputMode": "File", "S3DataDistributionType": "FullyReplicated", "S3CompressionType": "None"}}, {"InputName": "code", "AppManaged": false, "S3Input": {"S3Uri": "s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-38-12-974/input/code/preprocess.py", "LocalPath": "/opt/ml/processing/input/code", "S3DataType": "S3Prefix", "S3InputMode": "File", "S3DataDistributionType": "FullyReplicated", "S3CompressionType": "None"}}], "ProcessingOutputConfig": {"Outputs": [{"OutputName": "scaler_model", "AppManaged": false, "S3Output": {"S3Uri": "s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-38-12-974/output/scaler_model", "LocalPath": "/opt/ml/processing/scaler_model", "S3UploadMode": "EndOfJob"}}, {"OutputName": "train", "AppManaged": false, "S3Output": {"S3Uri": "s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-38-12-974/output/train", "LocalPath": "/opt/ml/processing/train", "S3UploadMode": "EndOfJob"}}, {"OutputName": "test", "AppManaged": false, "S3Output": {"S3Uri": "s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-38-12-974/output/test", "LocalPath": "/opt/ml/processing/test", "S3UploadMode": "EndOfJob"}}]}}}]}'
10+
roleARN: arn:aws:iam::123456789:role/ack-sagemaker-execution-role-123456789
11+
parallelismConfiguration:
12+
maxParallelExecutionSteps: 2
13+
tags:
14+
- key: environment
15+
value: testing
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: sagemaker.services.k8s.aws/v1alpha1
2+
kind: Pipeline
3+
metadata:
4+
name: test-pipeline
5+
spec:
6+
pipelineDisplayName: test-pipeline
7+
pipelineName: test-pipeline
8+
pipelineDefinition: '{"Version": "2020-12-01", "Metadata": {}, "Parameters": [{"Name": "ProcessingInstanceType", "Type": "String", "DefaultValue": "ml.m5.xlarge"}, {"Name": "ProcessingInstanceCount", "Type": "Integer", "DefaultValue": 1}, {"Name": "InputData", "Type": "String", "DefaultValue": "s3://sagemaker-us-west-2-123456789/pipeline-model-example/data/raw"}], "PipelineExperimentConfig": {"ExperimentName": {"Get": "Execution.PipelineName"}, "TrialName": {"Get": "Execution.PipelineExecutionId"}}, "Steps": [{"Name": "PreprocessData", "Type": "Processing", "Arguments": {"ProcessingResources": {"ClusterConfig": {"InstanceType": "ml.m5.large", "InstanceCount": {"Get": "Parameters.ProcessingInstanceCount"}, "VolumeSizeInGB": 30}}, "AppSpecification": {"ImageUri": "246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:0.23-1-cpu-py3", "ContainerEntrypoint": ["python3", "/opt/ml/processing/input/code/preprocess.py"]},
9+
"RoleArn": "arn:aws:iam::123456789:role/ack-sagemaker-execution-role-123456789", "ProcessingInputs": [{"InputName": "input-1", "AppManaged": false, "S3Input": {"S3Uri": {"Get": "Parameters.InputData"}, "LocalPath": "/opt/ml/processing/input", "S3DataType": "S3Prefix", "S3InputMode": "File", "S3DataDistributionType": "FullyReplicated", "S3CompressionType": "None"}}, {"InputName": "code", "AppManaged": false, "S3Input": {"S3Uri": "s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-38-12-974/input/code/preprocess.py", "LocalPath": "/opt/ml/processing/input/code", "S3DataType": "S3Prefix", "S3InputMode": "File", "S3DataDistributionType": "FullyReplicated", "S3CompressionType": "None"}}], "ProcessingOutputConfig": {"Outputs": [{"OutputName": "scaler_model", "AppManaged": false, "S3Output": {"S3Uri": "s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-38-12-974/output/scaler_model", "LocalPath": "/opt/ml/processing/scaler_model", "S3UploadMode": "EndOfJob"}}, {"OutputName": "train", "AppManaged": false, "S3Output": {"S3Uri": "s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-38-12-974/output/train", "LocalPath": "/opt/ml/processing/train", "S3UploadMode": "EndOfJob"}}, {"OutputName": "test", "AppManaged": false, "S3Output": {"S3Uri": "s3://sagemaker-us-west-2-123456789/sklearn-housing-data-process-2022-11-03-17-38-12-974/output/test", "LocalPath": "/opt/ml/processing/test", "S3UploadMode": "EndOfJob"}}]}}}]}'
10+
roleARN: arn:aws:iam::123456789:role/ack-sagemaker-execution-role-123456789
11+
parallelismConfiguration:
12+
maxParallelExecutionSteps: 2
13+
tags:
14+
- key: environment
15+
value: testing

0 commit comments

Comments
 (0)