Skip to content

Commit 98ab455

Browse files
trivikrtrivikr
authored
fix(lib-storage): set ChecksumAlgorithm when calling CreateMPU (#6802)
1 parent ee6abd0 commit 98ab455

File tree

3 files changed

+133
-147
lines changed

3 files changed

+133
-147
lines changed

‎lib/lib-storage/src/Upload.spec.ts

+4
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,15 @@ vi.mock("@aws-sdk/client-s3", async () => {
2727
send: sendMock,
2828
config: {
2929
endpoint: endpointMock,
30+
requestChecksumCalculation: () => Promise.resolve("WHEN_SUPPORTED"),
3031
},
3132
}),
3233
S3Client: vi.fn().mockReturnValue({
3334
send: sendMock,
3435
config: {
3536
endpoint: endpointMock,
3637
requestHandler: mockRequestHandler,
38+
requestChecksumCalculation: () => Promise.resolve("WHEN_SUPPORTED"),
3739
},
3840
}),
3941
CreateMultipartUploadCommand: vi.fn().mockReturnValue({
@@ -392,6 +394,7 @@ describe(Upload.name, () => {
392394
expect(CreateMultipartUploadCommand).toHaveBeenCalledWith({
393395
...actionParams,
394396
Body: undefined,
397+
ChecksumAlgorithm: "CRC32",
395398
});
396399
// upload parts is called correctly.
397400
expect(UploadPartCommand).toHaveBeenCalledTimes(2);
@@ -457,6 +460,7 @@ describe(Upload.name, () => {
457460
expect(CreateMultipartUploadCommand).toHaveBeenCalledWith({
458461
...actionParams,
459462
Body: undefined,
463+
ChecksumAlgorithm: "CRC32",
460464
});
461465

462466
// upload parts is called correctly.

‎lib/lib-storage/src/Upload.ts

+5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
AbortMultipartUploadCommand,
3+
ChecksumAlgorithm,
34
CompletedPart,
45
CompleteMultipartUploadCommand,
56
CompleteMultipartUploadCommandOutput,
@@ -199,8 +200,12 @@ export class Upload extends EventEmitter {
199200
}
200201

201202
private async __createMultipartUpload(): Promise<CreateMultipartUploadCommandOutput> {
203+
const requestChecksumCalculation = await this.client.config.requestChecksumCalculation();
202204
if (!this.createMultiPartPromise) {
203205
const createCommandParams = { ...this.params, Body: undefined };
206+
if (requestChecksumCalculation === "WHEN_SUPPORTED") {
207+
createCommandParams.ChecksumAlgorithm = this.params.ChecksumAlgorithm || ChecksumAlgorithm.CRC32;
208+
}
204209
this.createMultiPartPromise = this.client
205210
.send(new CreateMultipartUploadCommand(createCommandParams))
206211
.then((createMpuResponse) => {

‎lib/lib-storage/src/lib-storage.e2e.spec.ts

+124-147
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { S3 } from "@aws-sdk/client-s3";
1+
import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3";
22
import { Upload } from "@aws-sdk/lib-storage";
33
import { randomBytes } from "crypto";
44
import { Readable } from "stream";
@@ -7,153 +7,130 @@ import { afterAll, beforeAll, describe, expect, test as it } from "vitest";
77
import { getIntegTestResources } from "../../../tests/e2e/get-integ-test-resources";
88

99
describe("@aws-sdk/lib-storage", () => {
10-
let Key: string;
11-
let client: S3;
12-
let data: Uint8Array;
13-
let dataString: string;
14-
let Bucket: string;
15-
let region: string;
16-
17-
beforeAll(async () => {
18-
const integTestResourcesEnv = await getIntegTestResources();
19-
Object.assign(process.env, integTestResourcesEnv);
20-
21-
region = process?.env?.AWS_SMOKE_TEST_REGION as string;
22-
Bucket = process?.env?.AWS_SMOKE_TEST_BUCKET as string;
23-
24-
Key = ``;
25-
data = randomBytes(20_240_000);
26-
dataString = data.toString();
27-
28-
client = new S3({
29-
region,
30-
// ToDo(JS-5678): Remove this when default checksum is supported by Upload.
31-
requestChecksumCalculation: "WHEN_REQUIRED",
32-
});
33-
});
34-
35-
describe("Upload", () => {
36-
beforeAll(() => {
37-
Key = `multi-part-file-${Date.now()}`;
38-
});
39-
afterAll(async () => {
40-
await client.deleteObject({ Bucket, Key });
41-
});
42-
43-
it("should upload in parts for input type bytes", async () => {
44-
const s3Upload = new Upload({
45-
client,
46-
params: {
47-
Bucket,
48-
Key,
49-
Body: data,
50-
},
51-
});
52-
await s3Upload.done();
53-
54-
const object = await client.getObject({
55-
Bucket,
56-
Key,
57-
});
58-
59-
expect(await object.Body?.transformToString()).toEqual(dataString);
60-
});
61-
62-
it("should upload in parts for input type string", async () => {
63-
const s3Upload = new Upload({
64-
client,
65-
params: {
66-
Bucket,
67-
Key,
68-
Body: dataString,
69-
},
70-
});
71-
await s3Upload.done();
72-
73-
const object = await client.getObject({
74-
Bucket,
75-
Key,
76-
});
77-
78-
expect(await object.Body?.transformToString()).toEqual(dataString);
79-
});
80-
81-
it("should upload in parts for input type Readable", async () => {
82-
const s3Upload = new Upload({
83-
client,
84-
params: {
85-
Bucket,
86-
Key,
87-
Body: Readable.from(data),
88-
},
89-
});
90-
await s3Upload.done();
91-
92-
const object = await client.getObject({
93-
Bucket,
94-
Key,
95-
});
96-
97-
expect(await object.Body?.transformToString()).toEqual(dataString);
98-
});
99-
100-
it("should call AbortMultipartUpload if unable to complete a multipart upload.", async () => {
101-
class MockFailureS3 extends S3 {
102-
public counter = 0;
103-
async send(command: any, ...rest: any[]) {
104-
if (command?.constructor?.name === "UploadPartCommand" && this.counter++ % 3 === 0) {
105-
throw new Error("simulated upload part error");
10+
describe.each([undefined, "WHEN_REQUIRED", "WHEN_SUPPORTED"])(
11+
"requestChecksumCalculation: %s",
12+
(requestChecksumCalculation) => {
13+
describe.each([
14+
undefined,
15+
ChecksumAlgorithm.SHA1,
16+
ChecksumAlgorithm.SHA256,
17+
ChecksumAlgorithm.CRC32,
18+
ChecksumAlgorithm.CRC32C,
19+
])("ChecksumAlgorithm: %s", (ChecksumAlgorithm) => {
20+
let Key: string;
21+
let client: S3;
22+
let data: Uint8Array;
23+
let dataString: string;
24+
let Bucket: string;
25+
let region: string;
26+
27+
beforeAll(async () => {
28+
const integTestResourcesEnv = await getIntegTestResources();
29+
Object.assign(process.env, integTestResourcesEnv);
30+
31+
region = process?.env?.AWS_SMOKE_TEST_REGION as string;
32+
Bucket = process?.env?.AWS_SMOKE_TEST_BUCKET as string;
33+
34+
Key = ``;
35+
data = randomBytes(20_240_000);
36+
dataString = data.toString();
37+
38+
// @ts-expect-error: Types of property 'requestChecksumCalculation' are incompatible
39+
client = new S3({
40+
region,
41+
requestChecksumCalculation,
42+
});
43+
Key = `multi-part-file-${requestChecksumCalculation}-${ChecksumAlgorithm}-${Date.now()}`;
44+
});
45+
46+
afterAll(async () => {
47+
await client.deleteObject({ Bucket, Key });
48+
});
49+
50+
it("should upload in parts for input type bytes", async () => {
51+
const s3Upload = new Upload({
52+
client,
53+
params: { Bucket, Key, Body: data, ChecksumAlgorithm },
54+
});
55+
await s3Upload.done();
56+
57+
const object = await client.getObject({ Bucket, Key });
58+
expect(await object.Body?.transformToString()).toEqual(dataString);
59+
});
60+
61+
it("should upload in parts for input type string", async () => {
62+
const s3Upload = new Upload({
63+
client,
64+
params: { Bucket, Key, Body: dataString, ChecksumAlgorithm },
65+
});
66+
await s3Upload.done();
67+
68+
const object = await client.getObject({ Bucket, Key });
69+
expect(await object.Body?.transformToString()).toEqual(dataString);
70+
});
71+
72+
it("should upload in parts for input type Readable", async () => {
73+
const s3Upload = new Upload({
74+
client,
75+
params: { Bucket, Key, Body: Readable.from(data), ChecksumAlgorithm },
76+
});
77+
await s3Upload.done();
78+
79+
const object = await client.getObject({ Bucket, Key });
80+
expect(await object.Body?.transformToString()).toEqual(dataString);
81+
});
82+
83+
it("should call AbortMultipartUpload if unable to complete a multipart upload.", async () => {
84+
class MockFailureS3 extends S3 {
85+
public counter = 0;
86+
async send(command: any, ...rest: any[]) {
87+
if (command?.constructor?.name === "UploadPartCommand" && this.counter++ % 3 === 0) {
88+
throw new Error("simulated upload part error");
89+
}
90+
return super.send(command, ...rest);
91+
}
10692
}
107-
return super.send(command, ...rest);
108-
}
109-
}
110-
111-
const client = new MockFailureS3({
112-
region,
113-
});
11493

115-
const requestLog = [] as string[];
116-
117-
client.middlewareStack.add(
118-
(next, context) => async (args) => {
119-
const result = await next(args);
120-
requestLog.push([context.clientName, context.commandName, result.output.$metadata.httpStatusCode].join(" "));
121-
return result;
122-
},
123-
{
124-
name: "E2eRequestLog",
125-
step: "build",
126-
override: true,
127-
}
128-
);
129-
130-
const s3Upload = new Upload({
131-
client,
132-
params: {
133-
Bucket,
134-
Key,
135-
Body: data,
136-
},
94+
const client = new MockFailureS3({ region });
95+
96+
const requestLog = [] as string[];
97+
98+
client.middlewareStack.add(
99+
(next, context) => async (args) => {
100+
const result = await next(args);
101+
requestLog.push(
102+
[context.clientName, context.commandName, result.output.$metadata.httpStatusCode].join(" ")
103+
);
104+
return result;
105+
},
106+
{
107+
name: "E2eRequestLog",
108+
step: "build",
109+
override: true,
110+
}
111+
);
112+
113+
const s3Upload = new Upload({
114+
client,
115+
params: { Bucket, Key, Body: data, ChecksumAlgorithm },
116+
});
117+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
118+
await s3Upload.done().catch((ignored) => {});
119+
120+
const uploadStatus = await client
121+
.listParts({ Bucket, Key, UploadId: s3Upload.uploadId })
122+
.then((listParts) => listParts.$metadata.httpStatusCode)
123+
.catch((err) => err.toString());
124+
125+
expect(uploadStatus).toMatch(/NoSuchUpload:(.*?)aborted or completed\./);
126+
expect(requestLog).toEqual([
127+
"S3Client CreateMultipartUploadCommand 200",
128+
"S3Client UploadPartCommand 200",
129+
"S3Client UploadPartCommand 200",
130+
"S3Client AbortMultipartUploadCommand 204",
131+
]);
132+
});
137133
});
138-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
139-
await s3Upload.done().catch((ignored) => {});
140-
141-
const uploadStatus = await client
142-
.listParts({
143-
Bucket,
144-
Key,
145-
UploadId: s3Upload.uploadId,
146-
})
147-
.then((listParts) => listParts.$metadata.httpStatusCode)
148-
.catch((err) => err.toString());
149-
150-
expect(uploadStatus).toMatch(/NoSuchUpload:(.*?)aborted or completed\./);
151-
expect(requestLog).toEqual([
152-
"S3Client CreateMultipartUploadCommand 200",
153-
"S3Client UploadPartCommand 200",
154-
"S3Client UploadPartCommand 200",
155-
"S3Client AbortMultipartUploadCommand 204",
156-
]);
157-
});
158-
});
134+
}
135+
);
159136
}, 45_000);

0 commit comments

Comments
 (0)