Skip to content

Commit 6636822

Browse files
wty-BryantTianyi Wanglucix-aws
authored
feat: flexible checksum updates (#2808)
* add config and change default behavior of checksum middleware * regenerate client * regenerate clients * merge protocol test code from main * merge from main * add more test cases * add changelog * modify s3 internal test * separate checksum config check and workflow * restore s3 test * remove unused md5 header * separate checksum config and workflow * change default checksum to const * add checksum unset enum and modify comment of cfg * change comment * Update aws/checksum.go * change checksum value check logic * remove old check * correct unseekable stream logic without tls and its test cases * revert extra codegen * change tmv1 upload test cases after introducing flex checksum * add error test case for crc64 * change test name * default tmv1 checksum and add flex checksum metrics tracking * regenerate client and add metrics mw test * add comment to exported type * update s3 snapshot * update tmv1 integ test * exclude default checksum from presign op * reorder feature id and simplify metric tracking test * update changelog --------- Co-authored-by: Tianyi Wang <[email protected]> Co-authored-by: Luc Talatinian <[email protected]>
1 parent 4ffbb7c commit 6636822

File tree

89 files changed

+1520
-338
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+1520
-338
lines changed

‎.changelog/9ebe24c4791541e0840da49eab6f9d97.json

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"id": "9ebe24c4-7915-41e0-840d-a49eab6f9d97",
3+
"type": "feature",
4+
"description": "S3 client behavior is updated to always calculate a checksum by default for operations that support it (such as PutObject or UploadPart), or require it (such as DeleteObjects). The checksum algorithm used by default now becomes CRC32. Checksum behavior can be configured using `when_supported` and `when_required` options - in code using RequestChecksumCalculation, in shared config using request_checksum_calculation, or as env variable using AWS_REQUEST_CHECKSUM_CALCULATION. The S3 client attempts to validate response checksums for all S3 API operations that support checksums. However, if the SDK has not implemented the specified checksum algorithm then this validation is skipped. Checksum validation behavior can be configured using `when_supported` and `when_required` options - in code using ResponseChecksumValidation, in shared config using response_checksum_validation, or as env variable using AWS_RESPONSE_CHECKSUM_VALIDATION.",
5+
"modules": [
6+
".",
7+
"config",
8+
"service/internal/checksum",
9+
"service/s3"
10+
]
11+
}

‎aws/checksum.go

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package aws
2+
3+
// RequestChecksumCalculation controls request checksum calculation workflow
4+
type RequestChecksumCalculation int
5+
6+
const (
7+
// RequestChecksumCalculationUnset is the unset value for RequestChecksumCalculation
8+
RequestChecksumCalculationUnset RequestChecksumCalculation = iota
9+
10+
// RequestChecksumCalculationWhenSupported indicates request checksum will be calculated
11+
// if the operation supports input checksums
12+
RequestChecksumCalculationWhenSupported
13+
14+
// RequestChecksumCalculationWhenRequired indicates request checksum will be calculated
15+
// if required by the operation or if user elects to set a checksum algorithm in request
16+
RequestChecksumCalculationWhenRequired
17+
)
18+
19+
// ResponseChecksumValidation controls response checksum validation workflow
20+
type ResponseChecksumValidation int
21+
22+
const (
23+
// ResponseChecksumValidationUnset is the unset value for ResponseChecksumValidation
24+
ResponseChecksumValidationUnset ResponseChecksumValidation = iota
25+
26+
// ResponseChecksumValidationWhenSupported indicates response checksum will be validated
27+
// if the operation supports output checksums
28+
ResponseChecksumValidationWhenSupported
29+
30+
// ResponseChecksumValidationWhenRequired indicates response checksum will only
31+
// be validated if the operation requires output checksum validation
32+
ResponseChecksumValidationWhenRequired
33+
)

‎aws/config.go

+27
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,33 @@ type Config struct {
165165

166166
// Controls how a resolved AWS account ID is handled for endpoint routing.
167167
AccountIDEndpointMode AccountIDEndpointMode
168+
169+
// RequestChecksumCalculation determines when request checksum calculation is performed.
170+
//
171+
// There are two possible values for this setting:
172+
//
173+
// 1. RequestChecksumCalculationWhenSupported (default): The checksum is always calculated
174+
// if the operation supports it, regardless of whether the user sets an algorithm in the request.
175+
//
176+
// 2. RequestChecksumCalculationWhenRequired: The checksum is only calculated if the user
177+
// explicitly sets a checksum algorithm in the request.
178+
//
179+
// This setting is sourced from the environment variable AWS_REQUEST_CHECKSUM_CALCULATION
180+
// or the shared config profile attribute "request_checksum_calculation".
181+
RequestChecksumCalculation RequestChecksumCalculation
182+
183+
// ResponseChecksumValidation determines when response checksum validation is performed
184+
//
185+
// There are two possible values for this setting:
186+
//
187+
// 1. ResponseChecksumValidationWhenSupported (default): The checksum is always validated
188+
// if the operation supports it, regardless of whether the user sets the validation mode to ENABLED in request.
189+
//
190+
// 2. ResponseChecksumValidationWhenRequired: The checksum is only validated if the user
191+
// explicitly sets the validation mode to ENABLED in the request
192+
// This variable is sourced from environment variable AWS_RESPONSE_CHECKSUM_VALIDATION or
193+
// the shared config profile attribute "response_checksum_validation".
194+
ResponseChecksumValidation ResponseChecksumValidation
168195
}
169196

170197
// NewConfig returns a new Config pointer that can be chained with builder

‎aws/middleware/user_agent.go

+22-13
Original file line numberDiff line numberDiff line change
@@ -76,19 +76,28 @@ type UserAgentFeature string
7676

7777
// Enumerates UserAgentFeature.
7878
const (
79-
UserAgentFeatureResourceModel UserAgentFeature = "A" // n/a (we don't generate separate resource types)
80-
UserAgentFeatureWaiter = "B"
81-
UserAgentFeaturePaginator = "C"
82-
UserAgentFeatureRetryModeLegacy = "D" // n/a (equivalent to standard)
83-
UserAgentFeatureRetryModeStandard = "E"
84-
UserAgentFeatureRetryModeAdaptive = "F"
85-
UserAgentFeatureS3Transfer = "G"
86-
UserAgentFeatureS3CryptoV1N = "H" // n/a (crypto client is external)
87-
UserAgentFeatureS3CryptoV2 = "I" // n/a
88-
UserAgentFeatureS3ExpressBucket = "J"
89-
UserAgentFeatureS3AccessGrants = "K" // not yet implemented
90-
UserAgentFeatureGZIPRequestCompression = "L"
91-
UserAgentFeatureProtocolRPCV2CBOR = "M"
79+
UserAgentFeatureResourceModel UserAgentFeature = "A" // n/a (we don't generate separate resource types)
80+
UserAgentFeatureWaiter = "B"
81+
UserAgentFeaturePaginator = "C"
82+
UserAgentFeatureRetryModeLegacy = "D" // n/a (equivalent to standard)
83+
UserAgentFeatureRetryModeStandard = "E"
84+
UserAgentFeatureRetryModeAdaptive = "F"
85+
UserAgentFeatureS3Transfer = "G"
86+
UserAgentFeatureS3CryptoV1N = "H" // n/a (crypto client is external)
87+
UserAgentFeatureS3CryptoV2 = "I" // n/a
88+
UserAgentFeatureS3ExpressBucket = "J"
89+
UserAgentFeatureS3AccessGrants = "K" // not yet implemented
90+
UserAgentFeatureGZIPRequestCompression = "L"
91+
UserAgentFeatureProtocolRPCV2CBOR = "M"
92+
UserAgentFeatureRequestChecksumCRC32 = "U"
93+
UserAgentFeatureRequestChecksumCRC32C = "V"
94+
UserAgentFeatureRequestChecksumCRC64 = "W"
95+
UserAgentFeatureRequestChecksumSHA1 = "X"
96+
UserAgentFeatureRequestChecksumSHA256 = "Y"
97+
UserAgentFeatureRequestChecksumWhenSupported = "Z"
98+
UserAgentFeatureRequestChecksumWhenRequired = "a"
99+
UserAgentFeatureResponseChecksumWhenSupported = "b"
100+
UserAgentFeatureResponseChecksumWhenRequired = "c"
92101
)
93102

94103
// RequestUserAgent is a build middleware that set the User-Agent for the request.

‎codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java

+16
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ public class AddAwsConfigFields implements GoIntegration {
8484

8585
private static final String SDK_ACCOUNTID_ENDPOINT_MODE = "AccountIDEndpointMode";
8686

87+
private static final String REQUEST_CHECKSUM_CALCULATION = "RequestChecksumCalculation";
88+
89+
private static final String RESPONSE_CHECKSUM_VALIDATION = "ResponseChecksumValidation";
90+
8791
private static final List<AwsConfigField> AWS_CONFIG_FIELDS = ListUtils.of(
8892
AwsConfigField.builder()
8993
.name(REGION_CONFIG_NAME)
@@ -244,6 +248,18 @@ public class AddAwsConfigFields implements GoIntegration {
244248
.type(SdkGoTypes.Aws.AccountIDEndpointMode)
245249
.documentation("Indicates how aws account ID is applied in endpoint2.0 routing")
246250
.servicePredicate(AccountIDEndpointRouting::hasAccountIdEndpoints)
251+
.build(),
252+
AwsConfigField.builder()
253+
.name(REQUEST_CHECKSUM_CALCULATION)
254+
.type(SdkGoTypes.Aws.RequestChecksumCalculation)
255+
.documentation("Indicates how user opt-in/out request checksum calculation")
256+
.servicePredicate(AwsHttpChecksumGenerator::hasInputChecksumTrait)
257+
.build(),
258+
AwsConfigField.builder()
259+
.name(RESPONSE_CHECKSUM_VALIDATION)
260+
.type(SdkGoTypes.Aws.ResponseChecksumValidation)
261+
.documentation("Indicates how user opt-in/out response checksum validation")
262+
.servicePredicate(AwsHttpChecksumGenerator::hasOutputChecksumTrait)
247263
.build()
248264
);
249265

‎codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java

+26-9
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar;
2323
import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
2424
import software.amazon.smithy.model.Model;
25+
import software.amazon.smithy.model.knowledge.TopDownIndex;
2526
import software.amazon.smithy.model.shapes.MemberShape;
2627
import software.amazon.smithy.model.shapes.OperationShape;
2728
import software.amazon.smithy.model.shapes.ServiceShape;
@@ -73,9 +74,7 @@ public byte getOrder() {
7374
@Override
7475
public void processFinalizedModel(GoSettings settings, Model model) {
7576
ServiceShape service = settings.getService(model);
76-
for (ShapeId operationId : service.getAllOperations()) {
77-
final OperationShape operation = model.expectShape(operationId, OperationShape.class);
78-
77+
for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
7978
// Create a symbol provider because one is not available in this call.
8079
SymbolProvider symbolProvider = GoCodegenPlugin.createSymbolProvider(model, settings);
8180

@@ -128,8 +127,7 @@ public void writeAdditionalFiles(
128127
boolean supportsComputeInputChecksumsWorkflow = false;
129128
boolean supportsChecksumValidationWorkflow = false;
130129

131-
for (ShapeId operationID : service.getAllOperations()) {
132-
OperationShape operation = model.expectShape(operationID, OperationShape.class);
130+
for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
133131
if (!hasChecksumTrait(model, service, operation)) {
134132
continue;
135133
}
@@ -178,26 +176,44 @@ public List<RuntimeClientPlugin> getClientPlugins() {
178176
}
179177

180178
// return true if operation shape is decorated with `httpChecksum` trait.
181-
private boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
179+
private static boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
182180
return operation.hasTrait(HttpChecksumTrait.class);
183181
}
184182

185-
private boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
183+
private static boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
186184
if (!hasChecksumTrait(model, service, operation)) {
187185
return false;
188186
}
189187
HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
190188
return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent();
191189
}
192190

193-
private boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
191+
public static boolean hasInputChecksumTrait(Model model, ServiceShape service) {
192+
for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
193+
if (hasInputChecksumTrait(model, service, operation)) {
194+
return true;
195+
}
196+
}
197+
return false;
198+
}
199+
200+
private static boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
194201
if (!hasChecksumTrait(model, service, operation)) {
195202
return false;
196203
}
197204
HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
198205
return trait.getRequestValidationModeMember().isPresent() && !trait.getResponseAlgorithms().isEmpty();
199206
}
200207

208+
public static boolean hasOutputChecksumTrait(Model model, ServiceShape service) {
209+
for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
210+
if (hasOutputChecksumTrait(model, service, operation)) {
211+
return true;
212+
}
213+
}
214+
return false;
215+
}
216+
201217
private boolean isS3ServiceShape(Model model, ServiceShape service) {
202218
String serviceId = service.expectTrait(ServiceTrait.class).getSdkId();
203219
return serviceId.equalsIgnoreCase("S3");
@@ -244,6 +260,7 @@ private void writeInputMiddlewareHelper(
244260
return $T(stack, $T{
245261
GetAlgorithm: $L,
246262
RequireChecksum: $L,
263+
RequestChecksumCalculation: options.RequestChecksumCalculation,
247264
EnableTrailingChecksum: $L,
248265
EnableComputeSHA256PayloadHash: true,
249266
EnableDecodedContentLengthHeader: $L,
@@ -284,6 +301,7 @@ private void writeOutputMiddlewareHelper(
284301
writer.write("""
285302
return $T(stack, $T{
286303
GetValidationMode: $L,
304+
ResponseChecksumValidation: options.ResponseChecksumValidation,
287305
ValidationAlgorithms: $L,
288306
IgnoreMultipartValidation: $L,
289307
LogValidationSkipped: true,
@@ -293,7 +311,6 @@ private void writeOutputMiddlewareHelper(
293311
AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(),
294312
SymbolUtils.createValueSymbolBuilder("OutputMiddlewareOptions",
295313
AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(),
296-
297314
getRequestValidationModeAccessorFuncName(operationName),
298315
convertToGoStringList(responseAlgorithms),
299316
ignoreMultipartChecksumValidationMap.getOrDefault(

‎codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java

+37-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.util.TreeSet;
2424
import software.amazon.smithy.aws.go.codegen.customization.AwsCustomGoDependency;
2525
import software.amazon.smithy.aws.go.codegen.customization.PresignURLAutoFill;
26+
import software.amazon.smithy.aws.traits.HttpChecksumTrait;
2627
import software.amazon.smithy.aws.traits.ServiceTrait;
2728
import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait;
2829
import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait;
@@ -67,7 +68,7 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration {
6768
private static final String CONVERT_TO_PRESIGN_MIDDLEWARE_NAME = "convertToPresignMiddleware";
6869
private static final String CONVERT_TO_PRESIGN_TYPE_NAME = "presignConverter";
6970
private static final String NOP_HTTP_CLIENT_OPTION_FUNC_NAME = "withNopHTTPClientAPIOption";
70-
71+
private static final String NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME = "withNoDefaultChecksumAPIOption";
7172
private static final String PRESIGN_CLIENT = "PresignClient";
7273
private static final Symbol presignClientSymbol = buildSymbol(PRESIGN_CLIENT, true);
7374

@@ -218,7 +219,11 @@ public void writeAdditionalFiles(
218219
writeConvertToPresignMiddleware(writer, model, symbolProvider, serviceShape);
219220
});
220221

222+
boolean supportsComputeInputChecksumsWorkflow = false;
221223
for (OperationShape operationShape : TopDownIndex.of(model).getContainedOperations(serviceShape)) {
224+
if (hasInputChecksumTrait(operationShape)) {
225+
supportsComputeInputChecksumsWorkflow = true;
226+
}
222227
if (!validOperations.contains(operationShape.getId())) {
223228
continue;
224229
}
@@ -231,6 +236,10 @@ public void writeAdditionalFiles(
231236
writeS3AddAsUnsignedPayloadHelper(writer, model, symbolProvider, serviceShape, operationShape);
232237
});
233238
}
239+
240+
if (supportsComputeInputChecksumsWorkflow) {
241+
writePresignRequestChecksumConfigHelpers(settings, goDelegator);
242+
}
234243
}
235244

236245
private void writePresignOperationFunction(
@@ -263,6 +272,10 @@ private void writePresignOperationFunction(
263272

264273
writer.write("clientOptFns := append(options.ClientOptions, $L)", NOP_HTTP_CLIENT_OPTION_FUNC_NAME);
265274
writer.write("");
275+
if (hasInputChecksumTrait(operationShape)) {
276+
writer.write("clientOptFns = append(options.ClientOptions, $L)", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME);
277+
writer.write("");
278+
}
266279

267280
writer.openBlock("result, _, err := c.client.invokeOperation(ctx, $S, params, clientOptFns,", ")",
268281
operationSymbol.getName(), () -> {
@@ -572,6 +585,29 @@ private void writePresignClientHelpers(
572585
writer.write("");
573586
}
574587

588+
private void writePresignRequestChecksumConfigHelpers(
589+
GoSettings settings,
590+
GoDelegator goDelegator
591+
) {
592+
goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate("""
593+
func $fn:L(options *Options) {
594+
options.RequestChecksumCalculation = $requestChecksumCalculationWhenRequired:T
595+
}""",
596+
Map.of(
597+
"fn", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME,
598+
"requestChecksumCalculationWhenRequired",
599+
AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculationWhenRequired")
600+
)));
601+
}
602+
603+
private static boolean hasInputChecksumTrait(OperationShape operation) {
604+
if (!operation.hasTrait(HttpChecksumTrait.class)) {
605+
return false;
606+
}
607+
HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
608+
return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent();
609+
}
610+
575611
/**
576612
* Writes the presigner interface used by the presign url client
577613
*/

‎codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java

+2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ public static final class Aws {
3939
public static final Symbol AccountIDEndpointModeRequired = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeRequired");
4040
public static final Symbol AccountIDEndpointModeDisabled = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeDisabled");
4141

42+
public static final Symbol RequestChecksumCalculation = AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculation");
43+
public static final Symbol ResponseChecksumValidation = AwsGoDependency.AWS_CORE.valueSymbol("ResponseChecksumValidation");
4244

4345
public static final class Middleware {
4446
public static final Symbol GetRequiresLegacyEndpoints = AwsGoDependency.AWS_MIDDLEWARE.valueSymbol("GetRequiresLegacyEndpoints");

0 commit comments

Comments
 (0)