Skip to content

Commit 781f6fc

Browse files
jeskewjeskew
authored
S3 checksum middleware (#102)
* Add body-checksum-applying middleware * Add a universal JS MD5 package * Add S3 checksum middleware bindings * Add SigV4 unsigned payload middleware * Adapt body checksum and unsigned payload middleware to reflect recent changes to mw stack * Add Glacier .gitignores * Incorporate changes made for Glacier middleware addition * Change browser stream type from ReadableStream to Blob * Regenerate browser SDK packages * Fix failing tests
1 parent d624c96 commit 781f6fc

File tree

461 files changed

+2595
-2718
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

461 files changed

+2595
-2718
lines changed

‎packages/add-glacier-checksum-headers-browser/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
"karma-jasmine": "^1.1.1",
3434
"karma-typescript": "3.0.8",
3535
"puppeteer": "^1.0.0",
36-
"typescript": "^2.3",
36+
"typescript": "^2.6",
3737
"jest": "^20.0.4"
3838
}
39-
}
39+
}

‎packages/add-glacier-checksum-headers-browser/src/index.spec.ts

Lines changed: 4 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,10 @@ describe("addChecksumHeaders", () => {
1616
Promise.resolve()
1717
);
1818

19-
const composedHandler: BuildHandler<
20-
any,
21-
any,
22-
ReadableStream | Blob
23-
> = addChecksumHeaders(Sha256, fromUtf8)(mockNextHandler);
19+
const composedHandler: BuildHandler<any, any, Blob> = addChecksumHeaders(
20+
Sha256,
21+
fromUtf8
22+
)(mockNextHandler);
2423

2524
beforeEach(() => {
2625
mockNextHandler.calls.reset();
@@ -105,93 +104,6 @@ describe("addChecksumHeaders", () => {
105104
);
106105
});
107106

108-
it("will calculate sha256 hashes when request body is a stream", async () => {
109-
const mockStream = new (ReadableStream as any)({
110-
start(controller: any) {
111-
const totalSize = 5767168; // 5.5 MiB
112-
let readSize = 0;
113-
function generateData(size: number) {
114-
setTimeout(() => {
115-
const data = new Uint8Array(size);
116-
controller.enqueue(data);
117-
118-
readSize += data.byteLength;
119-
120-
if (readSize < totalSize) {
121-
generateData(Math.min(1048576, totalSize - readSize));
122-
} else {
123-
controller.close();
124-
}
125-
}, 1);
126-
}
127-
generateData(1048576);
128-
}
129-
});
130-
131-
await composedHandler({
132-
input: {},
133-
request: {
134-
...minimalRequest,
135-
headers: {},
136-
body: mockStream
137-
}
138-
});
139-
140-
expect(mockNextHandler.calls.count()).toBe(1);
141-
const { request } = mockNextHandler.calls.allArgs()[0][0];
142-
expect(request.headers["x-amz-content-sha256"]).toBe(
143-
"733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60"
144-
);
145-
expect(request.headers["x-amz-sha256-tree-hash"]).toBe(
146-
"a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455"
147-
);
148-
});
149-
150-
it("will set a ReadableStream request body to a collected stream", async () => {
151-
const expectedRequestBody = new Uint8Array(5767168);
152-
153-
const mockStream = new (ReadableStream as any)({
154-
start(controller: any) {
155-
const totalSize = 5767168; // 5.5 MiB
156-
let readSize = 0;
157-
function generateData(size: number) {
158-
setTimeout(() => {
159-
const data = new Uint8Array(size);
160-
controller.enqueue(data);
161-
162-
readSize += data.byteLength;
163-
164-
if (readSize < totalSize) {
165-
generateData(Math.min(1048576, totalSize - readSize));
166-
} else {
167-
controller.close();
168-
}
169-
}, 1);
170-
}
171-
generateData(1048576);
172-
}
173-
});
174-
175-
await composedHandler({
176-
input: {},
177-
request: {
178-
...minimalRequest,
179-
headers: {},
180-
body: mockStream
181-
}
182-
});
183-
184-
expect(mockNextHandler.calls.count()).toBe(1);
185-
const { request } = mockNextHandler.calls.allArgs()[0][0];
186-
expect(request.body).toEqual(expectedRequestBody);
187-
expect(request.headers["x-amz-content-sha256"]).toBe(
188-
"733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60"
189-
);
190-
expect(request.headers["x-amz-sha256-tree-hash"]).toBe(
191-
"a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455"
192-
);
193-
});
194-
195107
it("will calculate sha256 hashes when request body is a blob", async () => {
196108
const data = new Uint8Array(5767168);
197109
const blob = new Blob([data]);

‎packages/add-glacier-checksum-headers-browser/src/index.ts

Lines changed: 54 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -3,97 +3,74 @@ import {
33
BuildHandlerArguments,
44
Decoder,
55
HandlerExecutionContext,
6-
HashConstructor
6+
HashConstructor,
7+
Hash
78
} from "@aws/types";
89
import { blobReader } from "@aws/chunked-blob-reader";
910
import { isArrayBuffer } from "@aws/is-array-buffer";
1011
import { toHex } from "@aws/util-hex-encoding";
1112
import { TreeHash } from "@aws/sha256-tree-hash";
1213
import { streamCollector } from "@aws/stream-collector-browser";
1314

14-
export function addChecksumHeaders(Sha256: HashConstructor, fromUtf8: Decoder) {
15-
return (next: BuildHandler<any, any, any>) => {
16-
return async (args: BuildHandlerArguments<any, any>) => {
17-
const request = args.request;
18-
19-
const hasTreeHash = !!request.headers["x-amz-sha256-tree-hash"];
20-
const hasContentHash = !!request.headers["x-amz-content-sha256"];
21-
22-
let body = request.body;
23-
if (body) {
24-
const treeHash = !hasTreeHash ? new TreeHash(Sha256, fromUtf8) : null;
25-
const contentHash = !hasContentHash ? new Sha256() : null;
26-
const MiB = 1048576;
27-
28-
let buffer: Uint8Array | undefined;
15+
const MiB = 1024 * 1024;
2916

30-
if (
31-
typeof ReadableStream !== "undefined" &&
32-
body instanceof ReadableStream
33-
) {
34-
// since the body was consumed, reset the body
35-
body = buffer = await streamCollector(body);
36-
} else {
37-
buffer = await convertToUint8Array(body, fromUtf8);
38-
}
17+
export function addChecksumHeaders(Sha256: HashConstructor, fromUtf8: Decoder) {
18+
return (next: BuildHandler<any, any, Blob>) => async ({
19+
request: { body, headers, ...requestRest },
20+
...rest
21+
}: BuildHandlerArguments<any, Blob>) => {
22+
if (body) {
23+
const treeHash = !("x-amz-sha256-tree-hash" in headers)
24+
? new TreeHash(Sha256, fromUtf8)
25+
: null;
26+
const contentHash = !("x-amz-content-sha256" in headers)
27+
? new Sha256()
28+
: null;
3929

40-
// working with a Uint8Array
41-
if (buffer) {
42-
contentHash && contentHash.update(buffer);
43-
if (treeHash) {
44-
for (let i = 0; i < buffer.length; i += MiB) {
45-
treeHash.update(
46-
buffer.subarray(i, Math.min(i + MiB, buffer.byteLength))
47-
);
48-
}
49-
}
50-
} else if (typeof body.size === "number") {
51-
await blobReader(
52-
body,
53-
chunk => {
54-
treeHash && treeHash.update(chunk);
55-
contentHash && contentHash.update(chunk);
56-
},
57-
MiB
58-
);
59-
}
30+
if (
31+
typeof body === "string" ||
32+
ArrayBuffer.isView(body) ||
33+
isArrayBuffer(body)
34+
) {
35+
contentHash && contentHash.update(body);
36+
treeHash && treeHash.update(body);
37+
} else if (isBlob(body)) {
38+
await blobReader(
39+
body,
40+
chunk => {
41+
treeHash && treeHash.update(chunk);
42+
contentHash && contentHash.update(chunk);
43+
},
44+
MiB
45+
);
46+
}
6047

61-
if (contentHash) {
62-
request.headers["x-amz-content-sha256"] = toHex(
63-
await contentHash.digest()
64-
);
65-
}
66-
if (treeHash) {
67-
request.headers["x-amz-sha256-tree-hash"] = toHex(
68-
await treeHash.digest()
69-
);
48+
for (const [headerName, hash] of <Array<[string, Hash]>>[
49+
["x-amz-content-sha256", contentHash],
50+
["x-amz-sha256-tree-hash", treeHash]
51+
]) {
52+
if (hash) {
53+
headers = {
54+
...headers,
55+
[headerName]: toHex(await hash.digest())
56+
};
7057
}
7158
}
59+
}
7260

73-
return next({
74-
...args,
75-
request: {
76-
...request,
77-
body
78-
}
79-
});
80-
};
61+
return next({
62+
...rest,
63+
request: {
64+
...requestRest,
65+
headers,
66+
body
67+
}
68+
});
8169
};
8270
}
8371

84-
function convertToUint8Array(
85-
data: string | ArrayBuffer | ArrayBufferView,
86-
fromUtf8: Decoder
87-
): Uint8Array | undefined {
88-
if (typeof data === "string") {
89-
return fromUtf8(data);
90-
}
91-
92-
if (ArrayBuffer.isView(data)) {
93-
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
94-
}
95-
96-
if (isArrayBuffer(data)) {
97-
return new Uint8Array(data, 0, data.byteLength);
98-
}
72+
function isBlob(arg: any): arg is Blob {
73+
return (
74+
Boolean(arg) && Object.prototype.toString.call(arg) === "[object Blob]"
75+
);
9976
}

‎packages/add-glacier-checksum-headers-node/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
"@aws/util-utf8-node": "^0.0.1",
2828
"@types/node": "*",
2929
"@types/jest": "^20.0.2",
30-
"typescript": "^2.3",
30+
"typescript": "^2.6",
3131
"jest": "^20.0.4"
3232
}
33-
}
33+
}

0 commit comments

Comments
 (0)