Skip to content

Commit 4c366b1

Browse files
rd-pongrd-pong
authored
[Cherry-pick] Create shallow canary marker and fix "No S3 objects found under S3" (#242)
 - Create shallow canary marker for eu-south-2, ap-southeast-3, me-central-1, eu-central-2, since some registries are not available in those regions. - Add training and hosting to shallow canary - Add `--region ` for `aws s3 sync` to prevent error when downloading s3 bucket data to temp directory. - Remove dependency on `s3://sagemaker-sample-data-$AWS_REGION `
1 parent dca96d4 commit 4c366b1

8 files changed

+55
-2
lines changed

‎test/canary/scripts/run_test.sh

+2
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@ pushd $E2E_DIR
9797
if [[ $SERVICE_REGION =~ ^(eu-north-1|eu-west-3)$ ]]; then
9898
# If select_regions_1 true we run the notebook_instance test
9999
pytest_args+=(-m "canary or select_regions_1")
100+
elif [[ $SERVICE_REGION =~ ^(eu-south-2|ap-southeast-3|me-central-1|eu-central-2)$ ]]; then
101+
pytest_args+=(-m "shallow_canary")
100102
else
101103
pytest_args+=(-m "canary")
102104
fi

‎test/e2e/conftest.py

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ def pytest_configure(config):
3030
)
3131
config.addinivalue_line("markers", "slow: mark test as slow to run")
3232
config.addinivalue_line("markers", "select_regions_1: mark test to only run if in select region")
33+
config.addinivalue_line("markers", "shallow_canary: mark test to run in shallow canary tests")
3334

3435

3536
def pytest_collection_modifyitems(config, items):

‎test/e2e/replacement_values.py

+46
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from e2e.bootstrap_resources import get_bootstrap_resources
1919

2020
# Taken from the SageMaker Python SDK
21+
# https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/image_uri_config/xgboost.json
2122
# Rather than including the entire SDK
2223
XGBOOST_IMAGE_URIS = {
2324
"us-west-1": "746614075791.dkr.ecr.us-west-1.amazonaws.com",
@@ -40,8 +41,16 @@
4041
"eu-west-3": "659782779980.dkr.ecr.eu-west-3.amazonaws.com",
4142
"me-south-1": "801668240914.dkr.ecr.me-south-1.amazonaws.com",
4243
"sa-east-1": "737474898029.dkr.ecr.sa-east-1.amazonaws.com",
44+
"eu-south-1": "978288397137.dkr.ecr.eu-south-1.amazonaws.com",
45+
"ap-northeast-3": "867004704886.dkr.ecr.ap-northeast-3.amazonaws.com",
46+
"eu-south-2": "104374241257.dkr.ecr.eu-south-2.amazonaws.com",
47+
"ap-southeast-3": "951798379941.dkr.ecr.ap-southeast-3.amazonaws.com",
48+
"me-central-1": "272398656194.dkr.ecr.me-central-1.amazonaws.com",
49+
"eu-central-2": "680994064768.dkr.ecr.eu-central-2.amazonaws.com",
50+
"af-south-1": "510948584623.dkr.ecr.af-south-1.amazonaws.com",
4351
}
4452

53+
# https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/image_uri_config/debugger.json
4554
DEBUGGER_IMAGE_URIS = {
4655
"us-west-1": "685455198987.dkr.ecr.us-west-1.amazonaws.com",
4756
"us-west-2": "895741380848.dkr.ecr.us-west-2.amazonaws.com",
@@ -63,6 +72,9 @@
6372
"eu-west-3": "447278800020.dkr.ecr.eu-west-3.amazonaws.com",
6473
"me-south-1": "986000313247.dkr.ecr.me-south-1.amazonaws.com",
6574
"sa-east-1": "818342061345.dkr.ecr.sa-east-1.amazonaws.com",
75+
"eu-south-1": "563282790590.dkr.ecr.eu-south-1.amazonaws.com",
76+
"ap-northeast-3": "479947661362.dkr.ecr.ap-northeast-3.amazonaws.com",
77+
"af-south-1": "314341159256.dkr.ecr.af-south-1.amazonaws.com",
6678
}
6779

6880
# https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html
@@ -87,6 +99,13 @@
8799
"eu-west-3": "749696950732.dkr.ecr.eu-west-3.amazonaws.com",
88100
"me-south-1": "249704162688.dkr.ecr.me-south-1.amazonaws.com",
89101
"sa-east-1": "855470959533.dkr.ecr.sa-east-1.amazonaws.com",
102+
"eu-south-1": "257386234256.dkr.ecr.eu-south-1.amazonaws.com",
103+
"ap-northeast-3": "867004704886.dkr.ecr.ap-northeast-3.amazonaws.com",
104+
"eu-south-2": "104374241257.dkr.ecr.eu-south-2.amazonaws.com",
105+
"ap-southeast-3": "951798379941.dkr.ecr.ap-southeast-3.amazonaws.com",
106+
"me-central-1": "272398656194.dkr.ecr.me-central-1.amazonaws.com",
107+
"eu-central-2": "680994064768.dkr.ecr.eu-central-2.amazonaws.com",
108+
"af-south-1": "455444449433.dkr.ecr.af-south-1.amazonaws.com",
90109
}
91110

92111

@@ -113,6 +132,11 @@
113132
"sa-east-1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com",
114133
"cn-north-1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn",
115134
"cn-northwest-1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn",
135+
"ap-northeast-3": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com",
136+
"eu-south-2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com",
137+
"ap-southeast-3": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com",
138+
"me-central-1": "914824155844.dkr.ecr.me-central-1.amazonaws.com",
139+
"eu-central-2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com",
116140
}
117141

118142
# https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-pre-built-container.html
@@ -140,6 +164,8 @@
140164
"me-south-1": "607024016150.dkr.ecr.me-south-1.amazonaws.com",
141165
"sa-east-1": "539772159869.dkr.ecr.sa-east-1.amazonaws.com",
142166
"us-gov-west-1": "362178532790.dkr.ecr.us-gov-west-1.amazonaws.com",
167+
"ap-northeast-3": "990339680094.dkr.ecr.ap-northeast-3.amazonaws.com",
168+
"ap-southeast-3": "669540362728.dkr.ecr.ap-southeast-3.amazonaws.com",
143169
}
144170

145171
# https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-configure-processing-jobs.html#clarify-processing-job-configure-container
@@ -164,6 +190,8 @@
164190
"sa-east-1": "520018980103.dkr.ecr.sa-east-1.amazonaws.com",
165191
"af-south-1": "811711786498.dkr.ecr.af-south-1.amazonaws.com",
166192
"eu-south-1": "638885417683.dkr.ecr.eu-south-1.amazonaws.com",
193+
"ap-northeast-3": "912233562940.dkr.ecr.ap-northeast-3.amazonaws.com",
194+
"ap-southeast-3": "705930551576.dkr.ecr.ap-southeast-3.amazonaws.com",
167195
}
168196

169197
ENDPOINT_INSTANCE_TYPES = {
@@ -174,10 +202,28 @@
174202
TRAINING_JOB_INSTANCE_TYPES = {
175203
"eu-west-3": "ml.m5.xlarge",
176204
"eu-north-1": "ml.m5.xlarge",
205+
"ap-northeast-3": "ml.m5.xlarge",
206+
"ap-east-1": "ml.m5.xlarge",
207+
"me-south-1": "ml.m5.xlarge",
208+
"eu-south-1": "ml.m5.xlarge",
209+
"af-south-1": "ml.m5.xlarge",
210+
"eu-south-2": "ml.m5.xlarge",
211+
"ap-southeast-3": "ml.m5.xlarge",
212+
"me-central-1": "ml.m5.xlarge",
213+
"eu-central-2": "ml.m5.xlarge",
177214
}
178215

179216
NOTEBOOK_INSTANCE_INSTANCE_TYPES = {
180217
"eu-north-1": "ml.t3.medium",
218+
"ap-northeast-3": "ml.t3.medium",
219+
"ap-east-1": "ml.t3.medium",
220+
"me-south-1": "ml.t3.medium",
221+
"eu-south-1": "ml.t3.medium",
222+
"af-south-1": "ml.t3.medium",
223+
"eu-south-2": "ml.t3.medium",
224+
"ap-southeast-3": "ml.t3.medium",
225+
"me-central-1": "ml.t3.medium",
226+
"eu-central-2": "ml.t3.medium",
181227
}
182228

183229
REPLACEMENT_VALUES = {

‎test/e2e/resources/kmeans_processingjob.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ spec:
2121
processingInputs:
2222
- inputName: mnist_tar
2323
s3Input:
24-
s3URI: s3://sagemaker-sample-data-$AWS_REGION/algorithms/kmeans/mnist/mnist.pkl.gz
24+
s3URI: s3://$SAGEMAKER_DATA_BUCKET/sagemaker/processing/algorithms/kmeans/mnist/mnist.pkl.gz
2525
localPath: /opt/ml/processing/input
2626
s3DataType: S3Prefix
2727
s3InputMode: File

‎test/e2e/resources/pipeline_processing.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ spec:
77
pipelineName: $PIPELINE_NAME
88
pipelineDefinition: '{"Version": "2020-12-01", "Metadata": {}, "Parameters": [{"Name": "ProcessingInstanceType",
99
"Type": "String", "DefaultValue": "ml.m5.xlarge"}, {"Name": "ProcessingInstanceCount", "Type": "Integer", "DefaultValue": 1},
10-
{"Name": "InputData", "Type": "String", "DefaultValue": "s3://sagemaker-sample-data-$AWS_REGION/algorithms/kmeans/mnist/mnist.pkl.gz"}],
10+
{"Name": "InputData", "Type": "String", "DefaultValue": "s3://$SAGEMAKER_DATA_BUCKET/sagemaker/processing/algorithms/kmeans/mnist/mnist.pkl.gz"}],
1111
"PipelineExperimentConfig": {"ExperimentName": {"Get": "Execution.PipelineName"}, "TrialName": {"Get": "Execution.PipelineExecutionId"}},
1212
"Steps": [{"Name": "PreprocessData", "Type": "Processing", "Arguments": {"ProcessingResources":
1313
{"ClusterConfig": {"InstanceType": "ml.m5.large", "InstanceCount": {"Get": "Parameters.ProcessingInstanceCount"}, "VolumeSizeInGB": 20}},

‎test/e2e/service_bootstrap.py

+2
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ def sync_data_bucket(bucket) -> str:
5555
"sync",
5656
f"s3://{SAGEMAKER_SOURCE_DATA_BUCKET}",
5757
f"./{temp_dir}/",
58+
"--region",
59+
"us-west-2",
5860
"--quiet",
5961
]
6062
)

‎test/e2e/tests/test_endpoint.py

+1
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ def faulty_config(name_suffix, single_container_model):
218218

219219

220220
@service_marker
221+
@pytest.mark.shallow_canary
221222
@pytest.mark.canary
222223
class TestEndpoint:
223224
def create_endpoint_test(self, xgboost_endpoint):

‎test/e2e/tests/test_trainingjob.py

+1
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ def test_stopped(self, xgboost_training_job):
8585
training_job_desc = get_sagemaker_training_job(training_job_name)
8686
assert training_job_desc["TrainingJobStatus"] in cfg.LIST_JOB_STATUS_STOPPED
8787

88+
@pytest.mark.shallow_canary
8889
@pytest.mark.canary
8990
def test_completed(self, xgboost_training_job):
9091
(reference, resource) = xgboost_training_job

0 commit comments

Comments
 (0)