From 5ef1fed7fe7442288f112c77682c8897ac8aab90 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Thu, 10 Feb 2022 12:01:39 +0100 Subject: [PATCH 1/3] fix(sources): update spec --- .../client-sources/model/postURLJob.ts | 6 ++ .../client-sources/model/postURLJobAuth.ts | 19 ++++++ .../client-sources/model/postURLJobInput.ts | 9 +++ .../client-sources/model/postURLJobTarget.ts | 21 +++++++ .../client-sources/src/sourcesApi.ts | 5 ++ specs/sources/paths/ingest/postUrl.yml | 62 ++++++++++++++++++- .../requests/sources/postIngestUrl.json | 54 ++++++++++++++++ .../src/methods/requests/sources.test.ts | 29 +++++++++ 8 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 clients/algoliasearch-client-javascript/client-sources/model/postURLJobAuth.ts create mode 100644 clients/algoliasearch-client-javascript/client-sources/model/postURLJobTarget.ts diff --git a/clients/algoliasearch-client-javascript/client-sources/model/postURLJob.ts b/clients/algoliasearch-client-javascript/client-sources/model/postURLJob.ts index dd7c6d98d3..cb99c27493 100644 --- a/clients/algoliasearch-client-javascript/client-sources/model/postURLJob.ts +++ b/clients/algoliasearch-client-javascript/client-sources/model/postURLJob.ts @@ -1,4 +1,5 @@ import type { PostURLJobInput } from './postURLJobInput'; +import type { PostURLJobTarget } from './postURLJobTarget'; /** * Object containing a URL job. @@ -8,7 +9,12 @@ export type PostURLJob = { * The type of the file to ingest. */ type: PostURLJobType; + /** + * The name of the column that hold the unique identifier. + */ + uniqueIDColumn?: string; input: PostURLJobInput; + target: PostURLJobTarget; }; export type PostURLJobType = 'csv'; diff --git a/clients/algoliasearch-client-javascript/client-sources/model/postURLJobAuth.ts b/clients/algoliasearch-client-javascript/client-sources/model/postURLJobAuth.ts new file mode 100644 index 0000000000..b28e166959 --- /dev/null +++ b/clients/algoliasearch-client-javascript/client-sources/model/postURLJobAuth.ts @@ -0,0 +1,19 @@ +/** + * The authentication scheme for the URL that will be fetched. + */ +export type PostURLJobAuth = { + /** + * The type of authentication to use. + */ + type: PostURLJobAuthType; + /** + * The login to use for Basic Auth. + */ + login: string; + /** + * The password to use for Basic Auth. + */ + password: string; +}; + +export type PostURLJobAuthType = 'basic'; diff --git a/clients/algoliasearch-client-javascript/client-sources/model/postURLJobInput.ts b/clients/algoliasearch-client-javascript/client-sources/model/postURLJobInput.ts index 8a145acf4b..7a51e0dfc5 100644 --- a/clients/algoliasearch-client-javascript/client-sources/model/postURLJobInput.ts +++ b/clients/algoliasearch-client-javascript/client-sources/model/postURLJobInput.ts @@ -1,3 +1,5 @@ +import type { PostURLJobAuth } from './postURLJobAuth'; + /** * The input of the job. */ @@ -6,4 +8,11 @@ export type PostURLJobInput = { * The URL of the file to ingest. */ url: string; + /** + * The HTTP method that will be used to fetch the URL. + */ + method?: PostURLJobInputMethod; + auth?: PostURLJobAuth; }; + +export type PostURLJobInputMethod = 'GET' | 'POST'; diff --git a/clients/algoliasearch-client-javascript/client-sources/model/postURLJobTarget.ts b/clients/algoliasearch-client-javascript/client-sources/model/postURLJobTarget.ts new file mode 100644 index 0000000000..c9a3aae9c9 --- /dev/null +++ b/clients/algoliasearch-client-javascript/client-sources/model/postURLJobTarget.ts @@ -0,0 +1,21 @@ +/** + * The target of the job. + */ +export type PostURLJobTarget = { + /** + * The product to target. + */ + type: PostURLJobTargetType; + /** + * The index name of the product. + */ + indexName: string; + /** + * The type of operation to execute. + */ + operation: PostURLJobTargetOperation; +}; + +export type PostURLJobTargetType = 'search'; + +export type PostURLJobTargetOperation = 'replace'; diff --git a/clients/algoliasearch-client-javascript/client-sources/src/sourcesApi.ts b/clients/algoliasearch-client-javascript/client-sources/src/sourcesApi.ts index 1bb52600a7..d97c284097 100644 --- a/clients/algoliasearch-client-javascript/client-sources/src/sourcesApi.ts +++ b/clients/algoliasearch-client-javascript/client-sources/src/sourcesApi.ts @@ -73,6 +73,11 @@ export const createSourcesApi = ( 'Parameter `postURLJob.input` is required when calling `postIngestUrl`.' ); } + if (!postURLJob.target) { + throw new Error( + 'Parameter `postURLJob.target` is required when calling `postIngestUrl`.' + ); + } const request: Request = { method: 'POST', diff --git a/specs/sources/paths/ingest/postUrl.yml b/specs/sources/paths/ingest/postUrl.yml index 57c4dd537e..4959e3950a 100644 --- a/specs/sources/paths/ingest/postUrl.yml +++ b/specs/sources/paths/ingest/postUrl.yml @@ -16,22 +16,82 @@ post: required: - type - input + - target properties: type: type: string description: The type of the file to ingest. enum: - csv + uniqueIDColumn: + type: string + description: The name of the column that hold the unique identifier + example: objectID input: type: object title: postURLJobInput description: The input of the job. + additionalProperties: false + required: + - url properties: url: type: string description: The URL of the file to ingest. + method: + type: string + description: The HTTP method that will be used to fetch the URL. + enum: + - GET + - POST + auth: + type: object + title: postURLJobAuth + description: The authentication scheme for the URL that will be fetched. + additionalProperties: false + required: + - type + - login + - password + properties: + type: + type: string + description: The type of authentication to use. + enum: + - basic + login: + type: string + description: The login to use for Basic Auth. + example: johndoe + password: + type: string + description: The password to use for Basic Auth. + example: mypassword + + target: + type: object + title: postURLJobTarget + description: The target of the job. + additionalProperties: false required: - - url + - type + - indexName + - operation + properties: + type: + type: string + description: The product to target + enum: + - search + indexName: + type: string + description: The index name of the product + example: my_index_name + operation: + type: string + description: The type of operation to execute + enum: + - replace responses: '200': description: OK diff --git a/tests/CTS/methods/requests/sources/postIngestUrl.json b/tests/CTS/methods/requests/sources/postIngestUrl.json index e7ebba4ed0..7bb6c1fed1 100644 --- a/tests/CTS/methods/requests/sources/postIngestUrl.json +++ b/tests/CTS/methods/requests/sources/postIngestUrl.json @@ -6,6 +6,11 @@ "type": "csv", "input": { "url": "https://example.com/file.csv" + }, + "target": { + "type": "search", + "indexName": "pageviews", + "operation": "replace" } }, "request": { @@ -15,6 +20,55 @@ "type": "csv", "input": { "url": "https://example.com/file.csv" + }, + "target": { + "type": "search", + "indexName": "pageviews", + "operation": "replace" + } + } + } + }, + { + "method": "postIngestUrl", + "testName": "post postIngestUrl with all parameters", + "parameters": { + "type": "csv", + "uniqueIDColumn": "foobar", + "input": { + "url": "https://example.com/file.csv", + "method": "POST", + "auth": { + "type": "basic", + "login": "johndoe", + "password": "password" + } + }, + "target": { + "type": "search", + "indexName": "pageviews", + "operation": "replace" + } + }, + "request": { + "path": "/1/ingest/url", + "method": "POST", + "data": { + "type": "csv", + "uniqueIDColumn": "foobar", + "input": { + "url": "https://example.com/file.csv", + "method": "POST", + "auth": { + "type": "basic", + "login": "johndoe", + "password": "password" + } + }, + "target": { + "type": "search", + "indexName": "pageviews", + "operation": "replace" } } } diff --git a/tests/output/javascript/src/methods/requests/sources.test.ts b/tests/output/javascript/src/methods/requests/sources.test.ts index 6a80113218..9a3c8a377d 100644 --- a/tests/output/javascript/src/methods/requests/sources.test.ts +++ b/tests/output/javascript/src/methods/requests/sources.test.ts @@ -14,6 +14,7 @@ describe('postIngestUrl', () => { const req = (await client.postIngestUrl({ type: 'csv', input: { url: 'https://example.com/file.csv' }, + target: { type: 'search', indexName: 'pageviews', operation: 'replace' }, })) as unknown as EchoResponse; expect(req.path).toEqual('/1/ingest/url'); @@ -21,6 +22,34 @@ describe('postIngestUrl', () => { expect(req.data).toEqual({ type: 'csv', input: { url: 'https://example.com/file.csv' }, + target: { type: 'search', indexName: 'pageviews', operation: 'replace' }, + }); + expect(req.searchParams).toEqual(undefined); + }); + + test('post postIngestUrl with all parameters', async () => { + const req = (await client.postIngestUrl({ + type: 'csv', + uniqueIDColumn: 'foobar', + input: { + url: 'https://example.com/file.csv', + method: 'POST', + auth: { type: 'basic', login: 'johndoe', password: 'password' }, + }, + target: { type: 'search', indexName: 'pageviews', operation: 'replace' }, + })) as unknown as EchoResponse; + + expect(req.path).toEqual('/1/ingest/url'); + expect(req.method).toEqual('POST'); + expect(req.data).toEqual({ + type: 'csv', + uniqueIDColumn: 'foobar', + input: { + url: 'https://example.com/file.csv', + method: 'POST', + auth: { type: 'basic', login: 'johndoe', password: 'password' }, + }, + target: { type: 'search', indexName: 'pageviews', operation: 'replace' }, }); expect(req.searchParams).toEqual(undefined); }); From 72f76448224d81cfcea1afdf4d59ca741be279d7 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Thu, 10 Feb 2022 12:32:11 +0100 Subject: [PATCH 2/3] fix test --- tests/CTS/client/sources/api.json | 42 +++++++++++++++++-- .../javascript/src/client/sources.test.ts | 18 ++++++-- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/tests/CTS/client/sources/api.json b/tests/CTS/client/sources/api.json index ac87c64ead..d0e62ab113 100644 --- a/tests/CTS/client/sources/api.json +++ b/tests/CTS/client/sources/api.json @@ -6,7 +6,19 @@ "type": "method", "object": "$client", "path": "postIngestUrl", - "parameters": [{ "type": "csv", "input": { "url": "..." } }], + "parameters": [ + { + "type": "csv", + "input": { + "url": "https://example.com/file.csv" + }, + "target": { + "type": "search", + "indexName": "pageviews", + "operation": "replace" + } + } + ], "expected": { "match": { "objectContaining": { @@ -24,7 +36,19 @@ "type": "method", "object": "$client", "path": "postIngestUrl", - "parameters": [{ "type": "csv", "input": { "url": "..." } }], + "parameters": [ + { + "type": "csv", + "input": { + "url": "https://example.com/file.csv" + }, + "target": { + "type": "search", + "indexName": "pageviews", + "operation": "replace" + } + } + ], "expected": { "testSubject": "actual.userAgent", "match": { @@ -41,7 +65,19 @@ "type": "method", "object": "$client", "path": "postIngestUrl", - "parameters": [{ "type": "csv", "input": { "url": "..." } }], + "parameters": [ + { + "type": "csv", + "input": { + "url": "https://example.com/file.csv" + }, + "target": { + "type": "search", + "indexName": "pageviews", + "operation": "replace" + } + } + ], "expected": { "match": { "objectContaining": { diff --git a/tests/output/javascript/src/client/sources.test.ts b/tests/output/javascript/src/client/sources.test.ts index 846f1e1cd2..559393895e 100644 --- a/tests/output/javascript/src/client/sources.test.ts +++ b/tests/output/javascript/src/client/sources.test.ts @@ -18,7 +18,11 @@ describe('api', () => { let actual; - actual = $client.postIngestUrl({ type: 'csv', input: { url: '...' } }); + actual = $client.postIngestUrl({ + type: 'csv', + input: { url: 'https://example.com/file.csv' }, + target: { type: 'search', indexName: 'pageviews', operation: 'replace' }, + }); if (actual instanceof Promise) { actual = await actual; @@ -35,7 +39,11 @@ describe('api', () => { let actual; - actual = $client.postIngestUrl({ type: 'csv', input: { url: '...' } }); + actual = $client.postIngestUrl({ + type: 'csv', + input: { url: 'https://example.com/file.csv' }, + target: { type: 'search', indexName: 'pageviews', operation: 'replace' }, + }); if (actual instanceof Promise) { actual = await actual; @@ -52,7 +60,11 @@ describe('api', () => { let actual; - actual = $client.postIngestUrl({ type: 'csv', input: { url: '...' } }); + actual = $client.postIngestUrl({ + type: 'csv', + input: { url: 'https://example.com/file.csv' }, + target: { type: 'search', indexName: 'pageviews', operation: 'replace' }, + }); if (actual instanceof Promise) { actual = await actual; From 139a95600821cba9646d7b47c94e75a714165620 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Thu, 10 Feb 2022 13:11:21 +0100 Subject: [PATCH 3/3] review --- specs/sources/paths/ingest/postUrl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/sources/paths/ingest/postUrl.yml b/specs/sources/paths/ingest/postUrl.yml index 4959e3950a..3b2ca70adf 100644 --- a/specs/sources/paths/ingest/postUrl.yml +++ b/specs/sources/paths/ingest/postUrl.yml @@ -41,6 +41,7 @@ post: method: type: string description: The HTTP method that will be used to fetch the URL. + default: GET enum: - GET - POST @@ -67,7 +68,6 @@ post: type: string description: The password to use for Basic Auth. example: mypassword - target: type: object title: postURLJobTarget