Skip to content

S3 checksum middleware #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Feb 15, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
4 changes: 2 additions & 2 deletions packages/add-glacier-checksum-headers-browser/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
"karma-jasmine": "^1.1.1",
"karma-typescript": "3.0.8",
"puppeteer": "^1.0.0",
"typescript": "^2.3",
"typescript": "^2.6",
"jest": "^20.0.4"
}
}
}
85 changes: 1 addition & 84 deletions packages/add-glacier-checksum-headers-browser/src/index.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ describe('addChecksumHeaders', () => {

const mockNextHandler = jasmine.createSpy('nextHandler', () => Promise.resolve());

const composedHandler: BuildHandler<any, any, ReadableStream|Blob> = addChecksumHeaders(
const composedHandler: BuildHandler<any, any, Blob> = addChecksumHeaders(
Sha256,
fromUtf8,
)(mockNextHandler);
Expand Down Expand Up @@ -96,89 +96,6 @@ describe('addChecksumHeaders', () => {
expect(request.headers['x-amz-sha256-tree-hash']).toBe('fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9');
});

it('will calculate sha256 hashes when request body is a stream', async () => {
const mockStream = new (ReadableStream as any)({
start(controller: any) {
const totalSize = 5767168; // 5.5 MiB
let readSize = 0;
function generateData(size: number) {
setTimeout(() => {
const data = new Uint8Array(size);
controller.enqueue(data);

readSize += data.byteLength;

if (readSize < totalSize) {
generateData(
Math.min(1048576, totalSize - readSize)
);
} else {
controller.close();
}
}, 1);
}
generateData(1048576)
}
});

await composedHandler({
input: {},
request: {
...minimalRequest,
headers: {},
body: mockStream
}
});

expect(mockNextHandler.calls.count()).toBe(1);
const {request} = mockNextHandler.calls.allArgs()[0][0];
expect(request.headers['x-amz-content-sha256']).toBe('733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60');
expect(request.headers['x-amz-sha256-tree-hash']).toBe('a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455');
});

it('will set a ReadableStream request body to a collected stream', async () => {
const expectedRequestBody = new Uint8Array(5767168);

const mockStream = new (ReadableStream as any)({
start(controller: any) {
const totalSize = 5767168; // 5.5 MiB
let readSize = 0;
function generateData(size: number) {
setTimeout(() => {
const data = new Uint8Array(size);
controller.enqueue(data);

readSize += data.byteLength;

if (readSize < totalSize) {
generateData(
Math.min(1048576, totalSize - readSize)
);
} else {
controller.close();
}
}, 1);
}
generateData(1048576)
}
});

await composedHandler({
input: {},
request: {
...minimalRequest,
headers: {},
body: mockStream
}
});

expect(mockNextHandler.calls.count()).toBe(1);
const {request} = mockNextHandler.calls.allArgs()[0][0];
expect(request.body).toEqual(expectedRequestBody);
expect(request.headers['x-amz-content-sha256']).toBe('733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60');
expect(request.headers['x-amz-sha256-tree-hash']).toBe('a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455');
});

it('will calculate sha256 hashes when request body is a blob', async () => {
const data = new Uint8Array(5767168);
const blob = new Blob([
Expand Down
134 changes: 53 additions & 81 deletions packages/add-glacier-checksum-headers-browser/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,104 +3,76 @@ import {
BuildHandlerArguments,
Decoder,
HandlerExecutionContext,
HashConstructor
HashConstructor,
Hash
} from '@aws/types';
import {blobReader} from '@aws/chunked-blob-reader';
import {isArrayBuffer} from '@aws/is-array-buffer';
import {toHex} from '@aws/util-hex-encoding';
import {TreeHash} from '@aws/sha256-tree-hash';
import {streamCollector} from '@aws/stream-collector-browser';

const MiB = 1024 * 1024;

export function addChecksumHeaders(
Sha256: HashConstructor,
fromUtf8: Decoder,
) {
return (next: BuildHandler<any, any, any>) => {
return async(args: BuildHandlerArguments<any, any>) => {
const request = args.request;

const hasTreeHash = !!request.headers['x-amz-sha256-tree-hash'];
const hasContentHash = !!request.headers['x-amz-content-sha256'];

let body = request.body;
if (body) {
const treeHash = !hasTreeHash ? new TreeHash(Sha256, fromUtf8) : null;
const contentHash = !hasContentHash ? new Sha256() : null;
const MiB = 1048576;
return (next: BuildHandler<any, any, Blob>) => async ({
request: { body, headers, ...requestRest },
...rest,
}: BuildHandlerArguments<any, Blob>) => {
if (body) {
const treeHash = !('x-amz-sha256-tree-hash' in headers)
? new TreeHash(Sha256, fromUtf8)
: null;
const contentHash = !('x-amz-content-sha256' in headers)
? new Sha256()
: null;

let buffer: Uint8Array|undefined;

if (typeof ReadableStream !== 'undefined' && body instanceof ReadableStream) {
// since the body was consumed, reset the body
body = buffer = await streamCollector(body);
} else {
buffer = await convertToUint8Array(body, fromUtf8);
}
if (
typeof body === 'string' ||
ArrayBuffer.isView(body) ||
isArrayBuffer(body)
) {
contentHash && contentHash.update(body);
treeHash && treeHash.update(body);
} else if (isBlob(body)) {
await blobReader(
body,
(chunk) => {
treeHash && treeHash.update(chunk);
contentHash && contentHash.update(chunk);
},
MiB
);
}

// working with a Uint8Array
if (buffer) {
contentHash && contentHash.update(buffer);
if (treeHash) {
for (let i = 0; i < buffer.length; i += MiB) {
treeHash.update(
buffer.subarray(
i,
Math.min(i + MiB, buffer.byteLength)
)
);
}
for (const [headerName, hash] of <Array<[string, Hash]>>[
['x-amz-content-sha256', contentHash],
['x-amz-sha256-tree-hash', treeHash],
]) {
if (hash) {
headers = {
...headers,
[headerName]: toHex(await hash.digest()),
}
} else if (typeof body.size === 'number') {
await blobReader(
body,
(chunk) => {
treeHash && treeHash.update(chunk);
contentHash && contentHash.update(chunk);
},
MiB
);
}

if (contentHash) {
request.headers['x-amz-content-sha256'] = toHex(await contentHash.digest());
}
if (treeHash) {
request.headers['x-amz-sha256-tree-hash'] = toHex(await treeHash.digest());
}
}

return next({
...args,
request: {
...request,
body
}
});
}
}
}

function convertToUint8Array(
data: string|ArrayBuffer|ArrayBufferView,
fromUtf8: Decoder
): Uint8Array|undefined {
if (typeof data === 'string') {
return fromUtf8(data);
}

if (ArrayBuffer.isView(data)) {
return new Uint8Array(
data.buffer,
data.byteOffset,
data.byteLength
);
return next({
...rest,
request: {
...requestRest,
headers,
body,
}
});
}
}

if (isArrayBuffer(data)) {
return new Uint8Array(
data,
0,
data.byteLength
);
}
}
function isBlob(arg: any): arg is Blob {
return Boolean(arg)
&& Object.prototype.toString.call(arg) === '[object Blob]';
}
4 changes: 2 additions & 2 deletions packages/add-glacier-checksum-headers-node/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"@aws/util-utf8-node": "^0.0.1",
"@types/node": "*",
"@types/jest": "^20.0.2",
"typescript": "^2.3",
"typescript": "^2.6",
"jest": "^20.0.4"
}
}
}
Loading