Skip to content

S3 checksum middleware #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Feb 15, 2018
Merged

S3 checksum middleware #102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
f6aad5d
Add body-checksum-applying middleware
jeskew Jan 18, 2018
3c18aaa
Add a universal JS MD5 package
jeskew Jan 18, 2018
5298809
Add SigV4 unsigned payload middleware
jeskew Jan 23, 2018
fdf42b8
Add S3 checksum middleware bindings
jeskew Jan 19, 2018
03674b4
Adapt body checksum and unsigned payload middleware to reflect recent…
jeskew Jan 30, 2018
7c3501f
Add Glacier .gitignores
jeskew Feb 9, 2018
a51d5c7
Incorporate changes made for Glacier middleware addition
jeskew Feb 10, 2018
6ccacf0
Change browser stream type from ReadableStream to Blob
jeskew Feb 10, 2018
47a3b03
Regenerate browser SDK packages
Feb 10, 2018
9a5873d
Fix failing tests
jeskew Feb 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
4 changes: 2 additions & 2 deletions packages/add-glacier-checksum-headers-browser/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
"karma-jasmine": "^1.1.1",
"karma-typescript": "3.0.8",
"puppeteer": "^1.0.0",
"typescript": "^2.3",
"typescript": "^2.6",
"jest": "^20.0.4"
}
}
}
85 changes: 1 addition & 84 deletions packages/add-glacier-checksum-headers-browser/src/index.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ describe('addChecksumHeaders', () => {

const mockNextHandler = jasmine.createSpy('nextHandler', () => Promise.resolve());

const composedHandler: BuildHandler<any, any, ReadableStream|Blob> = addChecksumHeaders(
const composedHandler: BuildHandler<any, any, Blob> = addChecksumHeaders(
Sha256,
fromUtf8,
)(mockNextHandler);
Expand Down Expand Up @@ -96,89 +96,6 @@ describe('addChecksumHeaders', () => {
expect(request.headers['x-amz-sha256-tree-hash']).toBe('fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9');
});

it('will calculate sha256 hashes when request body is a stream', async () => {
const mockStream = new (ReadableStream as any)({
start(controller: any) {
const totalSize = 5767168; // 5.5 MiB
let readSize = 0;
function generateData(size: number) {
setTimeout(() => {
const data = new Uint8Array(size);
controller.enqueue(data);

readSize += data.byteLength;

if (readSize < totalSize) {
generateData(
Math.min(1048576, totalSize - readSize)
);
} else {
controller.close();
}
}, 1);
}
generateData(1048576)
}
});

await composedHandler({
input: {},
request: {
...minimalRequest,
headers: {},
body: mockStream
}
});

expect(mockNextHandler.calls.count()).toBe(1);
const {request} = mockNextHandler.calls.allArgs()[0][0];
expect(request.headers['x-amz-content-sha256']).toBe('733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60');
expect(request.headers['x-amz-sha256-tree-hash']).toBe('a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455');
});

it('will set a ReadableStream request body to a collected stream', async () => {
const expectedRequestBody = new Uint8Array(5767168);

const mockStream = new (ReadableStream as any)({
start(controller: any) {
const totalSize = 5767168; // 5.5 MiB
let readSize = 0;
function generateData(size: number) {
setTimeout(() => {
const data = new Uint8Array(size);
controller.enqueue(data);

readSize += data.byteLength;

if (readSize < totalSize) {
generateData(
Math.min(1048576, totalSize - readSize)
);
} else {
controller.close();
}
}, 1);
}
generateData(1048576)
}
});

await composedHandler({
input: {},
request: {
...minimalRequest,
headers: {},
body: mockStream
}
});

expect(mockNextHandler.calls.count()).toBe(1);
const {request} = mockNextHandler.calls.allArgs()[0][0];
expect(request.body).toEqual(expectedRequestBody);
expect(request.headers['x-amz-content-sha256']).toBe('733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60');
expect(request.headers['x-amz-sha256-tree-hash']).toBe('a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455');
});

it('will calculate sha256 hashes when request body is a blob', async () => {
const data = new Uint8Array(5767168);
const blob = new Blob([
Expand Down
134 changes: 53 additions & 81 deletions packages/add-glacier-checksum-headers-browser/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,104 +3,76 @@ import {
BuildHandlerArguments,
Decoder,
HandlerExecutionContext,
HashConstructor
HashConstructor,
Hash
} from '@aws/types';
import {blobReader} from '@aws/chunked-blob-reader';
import {isArrayBuffer} from '@aws/is-array-buffer';
import {toHex} from '@aws/util-hex-encoding';
import {TreeHash} from '@aws/sha256-tree-hash';
import {streamCollector} from '@aws/stream-collector-browser';

const MiB = 1024 * 1024;

export function addChecksumHeaders(
Sha256: HashConstructor,
fromUtf8: Decoder,
) {
return (next: BuildHandler<any, any, any>) => {
return async(args: BuildHandlerArguments<any, any>) => {
const request = args.request;

const hasTreeHash = !!request.headers['x-amz-sha256-tree-hash'];
const hasContentHash = !!request.headers['x-amz-content-sha256'];

let body = request.body;
if (body) {
const treeHash = !hasTreeHash ? new TreeHash(Sha256, fromUtf8) : null;
const contentHash = !hasContentHash ? new Sha256() : null;
const MiB = 1048576;
return (next: BuildHandler<any, any, Blob>) => async ({
request: { body, headers, ...requestRest },
...rest,
}: BuildHandlerArguments<any, Blob>) => {
if (body) {
const treeHash = !('x-amz-sha256-tree-hash' in headers)
? new TreeHash(Sha256, fromUtf8)
: null;
const contentHash = !('x-amz-content-sha256' in headers)
? new Sha256()
: null;

let buffer: Uint8Array|undefined;

if (typeof ReadableStream !== 'undefined' && body instanceof ReadableStream) {
// since the body was consumed, reset the body
body = buffer = await streamCollector(body);
} else {
buffer = await convertToUint8Array(body, fromUtf8);
}
if (
typeof body === 'string' ||
ArrayBuffer.isView(body) ||
isArrayBuffer(body)
) {
contentHash && contentHash.update(body);
treeHash && treeHash.update(body);
} else if (isBlob(body)) {
await blobReader(
body,
(chunk) => {
treeHash && treeHash.update(chunk);
contentHash && contentHash.update(chunk);
},
MiB
);
}

// working with a Uint8Array
if (buffer) {
contentHash && contentHash.update(buffer);
if (treeHash) {
for (let i = 0; i < buffer.length; i += MiB) {
treeHash.update(
buffer.subarray(
i,
Math.min(i + MiB, buffer.byteLength)
)
);
}
for (const [headerName, hash] of <Array<[string, Hash]>>[
['x-amz-content-sha256', contentHash],
['x-amz-sha256-tree-hash', treeHash],
]) {
if (hash) {
headers = {
...headers,
[headerName]: toHex(await hash.digest()),
}
} else if (typeof body.size === 'number') {
await blobReader(
body,
(chunk) => {
treeHash && treeHash.update(chunk);
contentHash && contentHash.update(chunk);
},
MiB
);
}

if (contentHash) {
request.headers['x-amz-content-sha256'] = toHex(await contentHash.digest());
}
if (treeHash) {
request.headers['x-amz-sha256-tree-hash'] = toHex(await treeHash.digest());
}
}

return next({
...args,
request: {
...request,
body
}
});
}
}
}

function convertToUint8Array(
data: string|ArrayBuffer|ArrayBufferView,
fromUtf8: Decoder
): Uint8Array|undefined {
if (typeof data === 'string') {
return fromUtf8(data);
}

if (ArrayBuffer.isView(data)) {
return new Uint8Array(
data.buffer,
data.byteOffset,
data.byteLength
);
return next({
...rest,
request: {
...requestRest,
headers,
body,
}
});
}
}

if (isArrayBuffer(data)) {
return new Uint8Array(
data,
0,
data.byteLength
);
}
}
function isBlob(arg: any): arg is Blob {
return Boolean(arg)
&& Object.prototype.toString.call(arg) === '[object Blob]';
}
4 changes: 2 additions & 2 deletions packages/add-glacier-checksum-headers-node/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"@aws/util-utf8-node": "^0.0.1",
"@types/node": "*",
"@types/jest": "^20.0.2",
"typescript": "^2.3",
"typescript": "^2.6",
"jest": "^20.0.4"
}
}
}
Loading