From 2542e533be351697b6864ecafe27c79704c657e8 Mon Sep 17 00:00:00 2001 From: Yamagishi Kazutoshi Date: Fri, 24 Jun 2022 14:25:21 +0000 Subject: [PATCH 1/2] feat(fetch-requester): add `@algolia/requester-fetch` --- .eslintrc.js | 1 + jest.config.js | 1 + package.json | 1 + packages/requester-fetch/api-extractor.json | 7 + packages/requester-fetch/index.js | 2 + packages/requester-fetch/package.json | 22 ++ .../__tests__/unit/fetch-requester.test.ts | 270 ++++++++++++++++++ .../src/createFetchRequester.ts | 86 ++++++ packages/requester-fetch/src/index.ts | 5 + rollup.config.js | 2 +- yarn.lock | 32 +++ 11 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 packages/requester-fetch/api-extractor.json create mode 100644 packages/requester-fetch/index.js create mode 100644 packages/requester-fetch/package.json create mode 100644 packages/requester-fetch/src/__tests__/unit/fetch-requester.test.ts create mode 100644 packages/requester-fetch/src/createFetchRequester.ts create mode 100644 packages/requester-fetch/src/index.ts diff --git a/.eslintrc.js b/.eslintrc.js index 7fe06168c..e8708839c 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -70,6 +70,7 @@ module.exports = { ['@algolia/recommend', './packages/recommend/src'], ['@algolia/requester-browser-xhr', './packages/requester-browser-xhr/src'], ['@algolia/requester-common', './packages/requester-common/src'], + ['@algolia/requester-fetch', './packages/requester-fetch/src'], ['@algolia/requester-node-http', './packages/requester-node-http/src'], ['@algolia/transporter', './packages/transporter/src'], ], diff --git a/jest.config.js b/jest.config.js index f0135e341..c5765208e 100644 --- a/jest.config.js +++ b/jest.config.js @@ -83,6 +83,7 @@ module.exports = { testEnvironment: 'node', testPathIgnorePatterns: [ 'packages/requester-browser-xhr/*', + 'packages/requester-fetch/*', 'packages/cache-browser-local-storage/*', 'packages/algoliasearch/src/__tests__/lite.test.ts', ], diff --git a/package.json b/package.json index 4ed87189d..1794fd215 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "@wdio/static-server-service": "5.16.10", "barrelsby": "2.2.0", "bundlesize": "0.18.0", + "cross-fetch": "^3.1.5", "dotenv": "8.2.0", "eslint": "6.8.0", "eslint-config-algolia": "15.0.0", diff --git a/packages/requester-fetch/api-extractor.json b/packages/requester-fetch/api-extractor.json new file mode 100644 index 000000000..d182b70fb --- /dev/null +++ b/packages/requester-fetch/api-extractor.json @@ -0,0 +1,7 @@ +{ + "extends": "../../api-extractor.json", + "mainEntryPointFilePath": "./dist/packages//src/index.d.ts", + "dtsRollup": { + "untrimmedFilePath": "./dist/.d.ts" + } +} diff --git a/packages/requester-fetch/index.js b/packages/requester-fetch/index.js new file mode 100644 index 000000000..3ea51a8c5 --- /dev/null +++ b/packages/requester-fetch/index.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line functional/immutable-data, import/no-commonjs +module.exports = require('./dist/requester-fetch.cjs.js'); diff --git a/packages/requester-fetch/package.json b/packages/requester-fetch/package.json new file mode 100644 index 000000000..2470ea31b --- /dev/null +++ b/packages/requester-fetch/package.json @@ -0,0 +1,22 @@ +{ + "name": "@algolia/requester-fetch", + "version": "4.13.1", + "private": false, + "description": "Promise-based request library for Fetch.", + "repository": { + "type": "git", + "url": "git://github.com/algolia/algoliasearch-client-javascript.git" + }, + "license": "MIT", + "sideEffects": false, + "main": "index.js", + "module": "dist/requester-fetch.esm.js", + "types": "dist/requester-fetch.d.ts", + "files": [ + "index.js", + "dist" + ], + "dependencies": { + "@algolia/requester-common": "4.13.1" + } +} diff --git a/packages/requester-fetch/src/__tests__/unit/fetch-requester.test.ts b/packages/requester-fetch/src/__tests__/unit/fetch-requester.test.ts new file mode 100644 index 000000000..b4b64f3f1 --- /dev/null +++ b/packages/requester-fetch/src/__tests__/unit/fetch-requester.test.ts @@ -0,0 +1,270 @@ +import { MethodEnum, Request } from '@algolia/requester-common'; +import crossFetch from 'cross-fetch'; +import nock from 'nock'; +// @ts-ignore +import { Readable } from 'readable-stream'; + +import { createFetchRequester } from '../..'; + +const originalFetch = window.fetch; + +beforeEach(() => { + window.fetch = crossFetch; +}); + +afterEach(() => { + window.fetch = originalFetch; +}); + +const requester = createFetchRequester(); + +const headers = { + 'content-type': 'application/x-www-form-urlencoded', +}; + +const timeoutRequest: Request = { + url: 'missing-url-here', + data: '', + headers: {}, + method: 'GET', + responseTimeout: 5, + connectTimeout: 2, +}; + +const requestStub: Request = { + url: 'https://algolia-dns.net/foo?x-algolia-header=foo', + method: MethodEnum.Post, + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + data: JSON.stringify({ foo: 'bar' }), + responseTimeout: 2, + connectTimeout: 1, +}; + +describe('status code handling', () => { + it('sends requests', async () => { + const body = JSON.stringify({ foo: 'bar' }); + + nock('https://algolia-dns.net', { reqheaders: headers }) + .post('/foo') + .query({ 'x-algolia-header': 'foo' }) + .reply(200, body); + + const response = await requester.send(requestStub); + + expect(response.content).toEqual(JSON.stringify({ foo: 'bar' })); + }); + + it('resolves status 200', async () => { + const body = JSON.stringify({ foo: 'bar' }); + + nock('https://algolia-dns.net', { reqheaders: headers }) + .post('/foo') + .query({ 'x-algolia-header': 'foo' }) + .reply(200, body); + + const response = await requester.send(requestStub); + + expect(response.status).toBe(200); + expect(response.content).toBe(body); + expect(response.isTimedOut).toBe(false); + }); + + it('resolves status 300', async () => { + const reason = 'Multiple Choices'; + + nock('https://algolia-dns.net', { reqheaders: headers }) + .post('/foo') + .query({ 'x-algolia-header': 'foo' }) + .reply(300, reason); + + const response = await requester.send(requestStub); + + expect(response.status).toBe(300); + expect(response.content).toBe(reason); + expect(response.isTimedOut).toBe(false); + }); + + it('resolves status 400', async () => { + const body = { message: 'Invalid Application-Id or API-Key' }; + + nock('https://algolia-dns.net', { reqheaders: headers }) + .post('/foo') + .query({ 'x-algolia-header': 'foo' }) + .reply(400, JSON.stringify(body)); + + const response = await requester.send(requestStub); + + expect(response.status).toBe(400); + expect(response.content).toBe(JSON.stringify(body)); + expect(response.isTimedOut).toBe(false); + }); + + it('handles chunked responses inside unicode character boundaries', async () => { + const testdata = Buffer.from('äöü'); + + // create a test response stream that is chunked inside a unicode character + function* generate() { + yield testdata.slice(0, 3); + yield testdata.slice(3); + } + + const testStream = Readable.from(generate()); + + nock('https://algolia-dns.net', { reqheaders: headers }) + .post('/foo') + .query({ 'x-algolia-header': 'foo' }) + .reply(200, testStream); + + const response = await requester.send(requestStub); + + expect(response.content).toEqual(testdata.toString()); + }); +}); + +describe('timeout handling', () => { + it('timeouts with the given 1 seconds connection timeout', async () => { + const before = Date.now(); + const response = await requester.send({ + ...timeoutRequest, + ...{ connectTimeout: 1, url: 'http://www.google.com:81' }, + }); + + const now = Date.now(); + + expect(response.content).toBe('Connection timeout'); + expect(now - before).toBeGreaterThan(999); + expect(now - before).toBeLessThan(1200); + }); + + it('connection timeouts with the given 2 seconds connection timeout', async () => { + const before = Date.now(); + const response = await requester.send({ + ...timeoutRequest, + ...{ connectTimeout: 2, url: 'http://www.google.com:81' }, + }); + + const now = Date.now(); + + expect(response.content).toBe('Connection timeout'); + expect(now - before).toBeGreaterThan(1999); + expect(now - before).toBeLessThan(2200); + }); + + it('socket timeouts if response dont appears before the timeout with 2 seconds timeout', async () => { + const before = Date.now(); + + const response = await requester.send({ + ...timeoutRequest, + ...{ responseTimeout: 2, url: 'http://localhost:1111/' }, + }); + + const now = Date.now(); + + expect(now - before).toBeGreaterThan(1999); + expect(now - before).toBeLessThan(2200); + expect(response.content).toBe('Socket timeout'); + }); + + it('socket timeouts if response dont appears before the timeout with 3 seconds timeout', async () => { + const before = Date.now(); + const response = await requester.send({ + ...timeoutRequest, + ...{ + responseTimeout: 3, + url: 'http://localhost:1111', + }, + }); + + const now = Date.now(); + + expect(response.content).toBe('Socket timeout'); + expect(now - before).toBeGreaterThan(2999); + expect(now - before).toBeLessThan(3200); + }); + + it('do not timeouts if response appears before the timeout', async () => { + const request = Object.assign({}, requestStub); + const before = Date.now(); + const response = await requester.send({ + ...request, + url: 'http://localhost:1111', + responseTimeout: 6, // the fake server sleeps for 5 seconds... + }); + + const now = Date.now(); + + expect(response.isTimedOut).toBe(false); + expect(response.status).toBe(200); + expect(response.content).toBe('{"foo": "bar"}'); + expect(now - before).toBeGreaterThan(4999); + expect(now - before).toBeLessThan(5200); + }); +}); + +describe('error handling', (): void => { + it('resolves dns not found', async () => { + const request = { + url: 'https://this-dont-exist.algolia.com', + method: MethodEnum.Post, + headers: { + 'X-Algolia-Application-Id': 'ABCDE', + 'X-Algolia-API-Key': '12345', + 'Content-Type': 'application/x-www-form-urlencoded', + }, + data: JSON.stringify({ foo: 'bar' }), + responseTimeout: 2, + connectTimeout: 1, + }; + + const response = await requester.send(request); + + expect(response.status).toBe(0); + expect(response.content).toContain(''); + expect(response.isTimedOut).toBe(false); + }); + + it('resolves general network errors', async () => { + nock('https://algolia-dns.net', { reqheaders: headers }) + .post('/foo') + .query({ 'x-algolia-header': 'foo' }) + .replyWithError('This is a general error'); + + const response = await requester.send(requestStub); + + expect(response.status).toBe(0); + expect(response.content).toBe( + 'request to https://algolia-dns.net/foo?x-algolia-header=foo failed, reason: This is a general error' + ); + expect(response.isTimedOut).toBe(false); + }); +}); + +describe('requesterOptions', () => { + it('allows to pass requesterOptions', async () => { + const body = JSON.stringify({ foo: 'bar' }); + const requesterTmp = createFetchRequester({ + requesterOptions: { + headers: { + 'x-algolia-foo': 'bar', + }, + }, + }); + + nock('https://algolia-dns.net', { + reqheaders: { + ...headers, + 'x-algolia-foo': 'bar', + }, + }) + .post('/foo') + .query({ 'x-algolia-header': 'foo' }) + .reply(200, body); + + const response = await requesterTmp.send(requestStub); + + expect(response.status).toBe(200); + expect(response.content).toBe(body); + }); +}); diff --git a/packages/requester-fetch/src/createFetchRequester.ts b/packages/requester-fetch/src/createFetchRequester.ts new file mode 100644 index 000000000..7bc6b476c --- /dev/null +++ b/packages/requester-fetch/src/createFetchRequester.ts @@ -0,0 +1,86 @@ +import { Request, Requester, Response as AlgoliaResponse } from '@algolia/requester-common'; + +function isAbortError(error: unknown): boolean { + return ( + // browser fetch + (error instanceof DOMException && error.name === 'AbortError') || + // node-fetch or undici + (error instanceof Error && error.name === 'AbortError') + ); +} + +function getErrorMessage(error: unknown, abortContent: string): string { + if (isAbortError(error)) { + return abortContent; + } else { + return error instanceof Error ? error.message : 'Network request failed'; + } +} + +export type FetchRequesterOptions = { + readonly requesterOptions?: RequestInit; +}; + +export function createFetchRequester({ + requesterOptions = {}, +}: FetchRequesterOptions = {}): Requester { + return { + async send(request: Request): Promise> { + const abortController = new AbortController(); + const signal = abortController.signal; + + const createTimeout = (timeout: number): NodeJS.Timeout => { + return setTimeout(() => { + abortController.abort(); + }, timeout * 1000); + }; + + const connectTimeout = createTimeout(request.connectTimeout); + + // eslint-disable-next-line functional/no-let + let fetchRes: Response; + // eslint-disable-next-line functional/no-try-statement + try { + fetchRes = await fetch(request.url, { + ...requesterOptions, + method: request.method, + headers: { + ...(requesterOptions.headers || {}), + ...request.headers, + }, + body: request.data || null, + mode: 'cors', + redirect: 'manual', + signal, + }); + } catch (error) { + return { + status: 0, + content: getErrorMessage(error, 'Connection timeout'), + isTimedOut: isAbortError(error), + }; + } + + clearTimeout(connectTimeout); + + createTimeout(request.responseTimeout); + + // eslint-disable-next-line functional/no-try-statement + try { + const content = await fetchRes.text(); + + return { + content, + isTimedOut: false, + status: fetchRes.status, + }; + } catch (error) { + return { + status: 0, + content: getErrorMessage(error, 'Socket timeout'), + isTimedOut: isAbortError(error), + }; + } + }, + }; +} diff --git a/packages/requester-fetch/src/index.ts b/packages/requester-fetch/src/index.ts new file mode 100644 index 000000000..6eaed90a6 --- /dev/null +++ b/packages/requester-fetch/src/index.ts @@ -0,0 +1,5 @@ +/** + * @file Automatically generated by barrelsby. + */ + +export * from './createFetchRequester'; diff --git a/rollup.config.js b/rollup.config.js index 93c540558..41f41e96a 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -52,7 +52,7 @@ packagesConfig.push({ external: ['crypto'], }); -['cache-browser-local-storage', 'requester-browser-xhr'].forEach(packageId => { +['cache-browser-local-storage', 'requester-browser-xhr', 'requester-fetch'].forEach(packageId => { packagesConfig.push({ output: packageId, package: packageId, diff --git a/yarn.lock b/yarn.lock index b6ba14f84..c255e828d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4181,6 +4181,13 @@ crc@^3.4.4: dependencies: buffer "^5.1.0" +cross-fetch@^3.1.5: + version "3.1.5" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" + integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw== + dependencies: + node-fetch "2.6.7" + cross-spawn@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-4.0.2.tgz#7b9247621c23adfdd3856004a823cbe397424d41" @@ -8074,6 +8081,13 @@ node-fetch-npm@^2.0.2: json-parse-better-errors "^1.0.0" safe-buffer "^5.1.1" +node-fetch@2.6.7: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" + node-fetch@^2.3.0, node-fetch@^2.5.0: version "2.6.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" @@ -10877,6 +10891,11 @@ tr46@^1.0.1: dependencies: punycode "^2.1.0" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + trim-newlines@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613" @@ -11351,6 +11370,11 @@ webdriverio@5.18.7: serialize-error "^5.0.0" webdriver "5.18.7" +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + webidl-conversions@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" @@ -11368,6 +11392,14 @@ whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + whatwg-url@^7.0.0: version "7.1.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-7.1.0.tgz#c2c492f1eca612988efd3d2266be1b9fc6170d06" From cbebf59deca2f407b20aa6ad77185d85d0d89963 Mon Sep 17 00:00:00 2001 From: Haroen Viaene Date: Wed, 6 Jul 2022 16:02:24 +0200 Subject: [PATCH 2/2] chore: unpin cross-fetch --- package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 1794fd215..284907611 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "@wdio/static-server-service": "5.16.10", "barrelsby": "2.2.0", "bundlesize": "0.18.0", - "cross-fetch": "^3.1.5", + "cross-fetch": "3.1.5", "dotenv": "8.2.0", "eslint": "6.8.0", "eslint-config-algolia": "15.0.0", diff --git a/yarn.lock b/yarn.lock index c255e828d..71e1680ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4181,7 +4181,7 @@ crc@^3.4.4: dependencies: buffer "^5.1.0" -cross-fetch@^3.1.5: +cross-fetch@3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==