Skip to content

Commit 3c642b4

Browse files
jeskewjeskew
authored
S3 checksum middleware (#102)
* Add body-checksum-applying middleware * Add a universal JS MD5 package * Add S3 checksum middleware bindings * Add SigV4 unsigned payload middleware * Adapt body checksum and unsigned payload middleware to reflect recent changes to mw stack * Add Glacier .gitignores * Incorporate changes made for Glacier middleware addition * Change browser stream type from ReadableStream to Blob * Regenerate browser SDK packages * Fix failing tests
1 parent 3681131 commit 3c642b4

File tree

462 files changed

+2789
-2287
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

462 files changed

+2789
-2287
lines changed

‎packages/add-glacier-checksum-headers-browser/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
"karma-jasmine": "^1.1.1",
3434
"karma-typescript": "3.0.8",
3535
"puppeteer": "^1.0.0",
36-
"typescript": "^2.3",
36+
"typescript": "^2.6",
3737
"jest": "^20.0.4"
3838
}
39-
}
39+
}

‎packages/add-glacier-checksum-headers-browser/src/index.spec.ts

Lines changed: 1 addition & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ describe('addChecksumHeaders', () => {
1818

1919
const mockNextHandler = jasmine.createSpy('nextHandler', () => Promise.resolve());
2020

21-
const composedHandler: BuildHandler<any, any, ReadableStream|Blob> = addChecksumHeaders(
21+
const composedHandler: BuildHandler<any, any, Blob> = addChecksumHeaders(
2222
Sha256,
2323
fromUtf8,
2424
)(mockNextHandler);
@@ -96,89 +96,6 @@ describe('addChecksumHeaders', () => {
9696
expect(request.headers['x-amz-sha256-tree-hash']).toBe('fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9');
9797
});
9898

99-
it('will calculate sha256 hashes when request body is a stream', async () => {
100-
const mockStream = new (ReadableStream as any)({
101-
start(controller: any) {
102-
const totalSize = 5767168; // 5.5 MiB
103-
let readSize = 0;
104-
function generateData(size: number) {
105-
setTimeout(() => {
106-
const data = new Uint8Array(size);
107-
controller.enqueue(data);
108-
109-
readSize += data.byteLength;
110-
111-
if (readSize < totalSize) {
112-
generateData(
113-
Math.min(1048576, totalSize - readSize)
114-
);
115-
} else {
116-
controller.close();
117-
}
118-
}, 1);
119-
}
120-
generateData(1048576)
121-
}
122-
});
123-
124-
await composedHandler({
125-
input: {},
126-
request: {
127-
...minimalRequest,
128-
headers: {},
129-
body: mockStream
130-
}
131-
});
132-
133-
expect(mockNextHandler.calls.count()).toBe(1);
134-
const {request} = mockNextHandler.calls.allArgs()[0][0];
135-
expect(request.headers['x-amz-content-sha256']).toBe('733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60');
136-
expect(request.headers['x-amz-sha256-tree-hash']).toBe('a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455');
137-
});
138-
139-
it('will set a ReadableStream request body to a collected stream', async () => {
140-
const expectedRequestBody = new Uint8Array(5767168);
141-
142-
const mockStream = new (ReadableStream as any)({
143-
start(controller: any) {
144-
const totalSize = 5767168; // 5.5 MiB
145-
let readSize = 0;
146-
function generateData(size: number) {
147-
setTimeout(() => {
148-
const data = new Uint8Array(size);
149-
controller.enqueue(data);
150-
151-
readSize += data.byteLength;
152-
153-
if (readSize < totalSize) {
154-
generateData(
155-
Math.min(1048576, totalSize - readSize)
156-
);
157-
} else {
158-
controller.close();
159-
}
160-
}, 1);
161-
}
162-
generateData(1048576)
163-
}
164-
});
165-
166-
await composedHandler({
167-
input: {},
168-
request: {
169-
...minimalRequest,
170-
headers: {},
171-
body: mockStream
172-
}
173-
});
174-
175-
expect(mockNextHandler.calls.count()).toBe(1);
176-
const {request} = mockNextHandler.calls.allArgs()[0][0];
177-
expect(request.body).toEqual(expectedRequestBody);
178-
expect(request.headers['x-amz-content-sha256']).toBe('733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60');
179-
expect(request.headers['x-amz-sha256-tree-hash']).toBe('a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455');
180-
});
181-
18299
it('will calculate sha256 hashes when request body is a blob', async () => {
183100
const data = new Uint8Array(5767168);
184101
const blob = new Blob([

‎packages/add-glacier-checksum-headers-browser/src/index.ts

Lines changed: 53 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -3,104 +3,76 @@ import {
33
BuildHandlerArguments,
44
Decoder,
55
HandlerExecutionContext,
6-
HashConstructor
6+
HashConstructor,
7+
Hash
78
} from '@aws/types';
89
import {blobReader} from '@aws/chunked-blob-reader';
910
import {isArrayBuffer} from '@aws/is-array-buffer';
1011
import {toHex} from '@aws/util-hex-encoding';
1112
import {TreeHash} from '@aws/sha256-tree-hash';
1213
import {streamCollector} from '@aws/stream-collector-browser';
1314

15+
const MiB = 1024 * 1024;
16+
1417
export function addChecksumHeaders(
1518
Sha256: HashConstructor,
1619
fromUtf8: Decoder,
1720
) {
18-
return (next: BuildHandler<any, any, any>) => {
19-
return async(args: BuildHandlerArguments<any, any>) => {
20-
const request = args.request;
21-
22-
const hasTreeHash = !!request.headers['x-amz-sha256-tree-hash'];
23-
const hasContentHash = !!request.headers['x-amz-content-sha256'];
24-
25-
let body = request.body;
26-
if (body) {
27-
const treeHash = !hasTreeHash ? new TreeHash(Sha256, fromUtf8) : null;
28-
const contentHash = !hasContentHash ? new Sha256() : null;
29-
const MiB = 1048576;
21+
return (next: BuildHandler<any, any, Blob>) => async ({
22+
request: { body, headers, ...requestRest },
23+
...rest,
24+
}: BuildHandlerArguments<any, Blob>) => {
25+
if (body) {
26+
const treeHash = !('x-amz-sha256-tree-hash' in headers)
27+
? new TreeHash(Sha256, fromUtf8)
28+
: null;
29+
const contentHash = !('x-amz-content-sha256' in headers)
30+
? new Sha256()
31+
: null;
3032

31-
let buffer: Uint8Array|undefined;
32-
33-
if (typeof ReadableStream !== 'undefined' && body instanceof ReadableStream) {
34-
// since the body was consumed, reset the body
35-
body = buffer = await streamCollector(body);
36-
} else {
37-
buffer = await convertToUint8Array(body, fromUtf8);
38-
}
33+
if (
34+
typeof body === 'string' ||
35+
ArrayBuffer.isView(body) ||
36+
isArrayBuffer(body)
37+
) {
38+
contentHash && contentHash.update(body);
39+
treeHash && treeHash.update(body);
40+
} else if (isBlob(body)) {
41+
await blobReader(
42+
body,
43+
(chunk) => {
44+
treeHash && treeHash.update(chunk);
45+
contentHash && contentHash.update(chunk);
46+
},
47+
MiB
48+
);
49+
}
3950

40-
// working with a Uint8Array
41-
if (buffer) {
42-
contentHash && contentHash.update(buffer);
43-
if (treeHash) {
44-
for (let i = 0; i < buffer.length; i += MiB) {
45-
treeHash.update(
46-
buffer.subarray(
47-
i,
48-
Math.min(i + MiB, buffer.byteLength)
49-
)
50-
);
51-
}
51+
for (const [headerName, hash] of <Array<[string, Hash]>>[
52+
['x-amz-content-sha256', contentHash],
53+
['x-amz-sha256-tree-hash', treeHash],
54+
]) {
55+
if (hash) {
56+
headers = {
57+
...headers,
58+
[headerName]: toHex(await hash.digest()),
5259
}
53-
} else if (typeof body.size === 'number') {
54-
await blobReader(
55-
body,
56-
(chunk) => {
57-
treeHash && treeHash.update(chunk);
58-
contentHash && contentHash.update(chunk);
59-
},
60-
MiB
61-
);
62-
}
63-
64-
if (contentHash) {
65-
request.headers['x-amz-content-sha256'] = toHex(await contentHash.digest());
66-
}
67-
if (treeHash) {
68-
request.headers['x-amz-sha256-tree-hash'] = toHex(await treeHash.digest());
6960
}
7061
}
71-
72-
return next({
73-
...args,
74-
request: {
75-
...request,
76-
body
77-
}
78-
});
7962
}
80-
}
81-
}
82-
83-
function convertToUint8Array(
84-
data: string|ArrayBuffer|ArrayBufferView,
85-
fromUtf8: Decoder
86-
): Uint8Array|undefined {
87-
if (typeof data === 'string') {
88-
return fromUtf8(data);
89-
}
9063

91-
if (ArrayBuffer.isView(data)) {
92-
return new Uint8Array(
93-
data.buffer,
94-
data.byteOffset,
95-
data.byteLength
96-
);
64+
return next({
65+
...rest,
66+
request: {
67+
...requestRest,
68+
headers,
69+
body,
70+
}
71+
});
9772
}
73+
}
9874

99-
if (isArrayBuffer(data)) {
100-
return new Uint8Array(
101-
data,
102-
0,
103-
data.byteLength
104-
);
105-
}
106-
}
75+
function isBlob(arg: any): arg is Blob {
76+
return Boolean(arg)
77+
&& Object.prototype.toString.call(arg) === '[object Blob]';
78+
}

‎packages/add-glacier-checksum-headers-node/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
"@aws/util-utf8-node": "^0.0.1",
2828
"@types/node": "*",
2929
"@types/jest": "^20.0.2",
30-
"typescript": "^2.3",
30+
"typescript": "^2.6",
3131
"jest": "^20.0.4"
3232
}
33-
}
33+
}

0 commit comments

Comments
 (0)