feat: common test suite generation APIC-186 (#22)

Author: millotp

.eslintrc.js

@@ -30,7 +30,7 @@ module.exports = {
     'no-bitwise': 0,
     '@typescript-eslint/no-namespace': 0,
     'max-classes-per-file': 0,
-    'no-unused-vars': 0,
+    'no-continue': 0,
     '@typescript-eslint/prefer-enum-initializers': 0,
     // there's a conflict when declaring `type` and `namespaces`, even with `ignoreDeclarationMerge`
     'no-redeclare': 0,

.gitignore

@@ -15,3 +15,5 @@ yarn-error.log
 **/dist
 **/.openapi-generator-ignore
 **/git_push.sh
+
+.vscode

clients/algoliasearch-client-javascript/client-search/package.json

@@ -18,7 +18,7 @@
     "yarn": "^3.0.0"
   },
   "devDependencies": {
-    "@types/node": "^16.11.6",
+    "@types/node": "16.11.11",
     "typescript": "4.5.2"
   }
 }

clients/algoliasearch-client-javascript/recommend/package.json

@@ -18,7 +18,7 @@
     "yarn": "^3.0.0"
   },
   "devDependencies": {
-    "@types/node": "^16.11.6",
+    "@types/node": "16.11.11",
     "typescript": "4.5.2"
   }
 }

doc/CTS.md

@@ -0,0 +1,49 @@
+# Common Test Suite
+
+The CTS aims at ensuring minimal working operation for the API clients, by comparing the request formed by sample parameters.
+It is automaticaly generated for all languages, from a JSON entry point.
+
+## How to run it
+
+```bash
+yarn cts:generate
+yarn cts:test
+```
+
+If you only want to generate the tests for a set of languages, you can run:
+
+```bash
+yarn cts:generate "javascript ruby"
+```
+
+## How to add test
+
+The test generation script requires a JSON file name from the `operationId` (e.g. `search.json`), located in the `CTS/<client>/` folder (e.g. `CTS/search/`).
+
+```json
+[
+  {
+    "testName": "the name of the test (e.g. test('search endpoint'))",
+    "method": "the method to call (e.g. search)",
+    "parameters": [
+      "indexName",
+      {
+        "$objectName": "the name of the object for strongly type language",
+        "query": "the string to search"
+      }
+    ],
+    "request": {
+      "path": "/1/indexes/indexName/query",
+      "method": "POST",
+      "data": { "query": "the string to search" }
+    }
+  }
+]
+```
+
+And that's it! If the name of the file matches a real `operationId` in the spec, then a test will be generated.
+
+## How to add a new language
+
+- Create a template in `test/CTS/templates/<your language>.mustache` that parse a array of test into your test framework of choice
+- Add the language in the array `languages` in `tests/generateCTS.ts`.

doc/contribution_addNewClient.md

@@ -28,6 +28,10 @@ You will need to implement:
 
 - An `init` method
 - The `retry strategy` with your custom transporter
+- At least 2 requester:
+  - http requester, using the standard library
+  - echo requester that send the request back, used by the CTS
+- A logger that the user can swap
 - More to come...
 
 ### Init method

openapitools.json

@@ -3,7 +3,7 @@
   "generator-cli": {
     "version": "5.3.0",
     "generators": {
-      "javascript-client-search": {
+      "javascript-search": {
         "generatorName": "typescript-node",
         "templateDir": "#{cwd}/templates/javascript/",
         "config": "#{cwd}/openapitools.json",
@@ -17,6 +17,7 @@
           "modelPropertyNaming": "original",
           "supportsES6": true,
           "npmName": "@algolia/client-search",
+          "packageName": "@algolia/client-search",
           "npmVersion": "5.0.0"
         }
       },
@@ -35,6 +36,7 @@
           "modelPropertyNaming": "original",
           "supportsES6": true,
           "npmName": "@algolia/recommend",
+          "packageName": "@algolia/recommend",
           "npmVersion": "5.0.0"
         }
       }

package.json

@@ -3,7 +3,8 @@
   "version": "0.0.0",
   "workspaces": [
     "clients/algoliasearch-client-javascript/*",
-    "playground/javascript/"
+    "playground/javascript/",
+    "tests/"
   ],
   "scripts": {
     "build:spec:recommend:json": "yarn swagger-cli bundle specs/recommend/spec.yml --outfile specs/dist/recommend.json --type json",
@@ -17,11 +18,13 @@
     "client:build-js:recommend": "yarn workspace @algolia/recommend build",
     "client:build-js": "yarn client:build-js:search && yarn client:build-js:recommend",
     "client:build": "yarn client:build-js",
+    "cts:generate": "yarn workspace tests cts:generate",
+    "cts:test": "yarn workspace tests test",
     "lint:client:fix": "eslint --ext=ts ${CLIENT} --fix",
     "lint": "eslint --ext=ts .",
     "format:specs": "yarn prettier --write specs",
     "generate:js:recommend": "yarn openapi-generator-cli generate --generator-key javascript-recommend && CLIENT=recommend yarn utils:import-js && CLIENT=clients/algoliasearch-client-javascript/recommend/ yarn lint:client:fix",
-    "generate:js:search": "yarn openapi-generator-cli generate --generator-key javascript-client-search && CLIENT=client-search yarn utils:import-js && CLIENT=clients/algoliasearch-client-javascript/client-search/ yarn lint:client:fix",
+    "generate:js:search": "yarn openapi-generator-cli generate --generator-key javascript-search && CLIENT=client-search yarn utils:import-js && CLIENT=clients/algoliasearch-client-javascript/client-search/ yarn lint:client:fix",
     "generate:js": "yarn generate:js:search && yarn generate:js:recommend",
     "generate:recommend": "yarn generate:js:recommend",
     "generate:search": "yarn generate:js:search",

templates/javascript/package.mustache

@@ -1,7 +1,7 @@
 {
-  "name": "{{npmName}}",
+  "name": "{{packageName}}",
   "version": "{{npmVersion}}",
-  "description": "JavaScript client for {{npmName}}",
+  "description": "JavaScript client for {{packageName}}",
   "repository": "{{gitUserId}}/{{gitRepoId}}",
   "author": "Algolia",
   "private": true,
@@ -18,7 +18,7 @@
     "yarn": "^3.0.0"
   },
   "devDependencies": {
-    "@types/node": "^16.11.6",
+    "@types/node": "16.11.11",
     "typescript": "4.5.2"
   }{{#npmRepository}},
   "publishConfig": {

tests/CTS/clients/search/search.json

@@ -0,0 +1,20 @@
+[
+  {
+    "testName": "search",
+    "method": "search",
+    "parameters": [
+      "indexName",
+      {
+        "$objectName": "Query",
+        "query": "queryString"
+      }
+    ],
+    "request": {
+      "path": "/1/indexes/indexName/query",
+      "method": "POST",
+      "data": {
+        "query": "queryString"
+      }
+    }
+  }
+]

tests/CTS/templates/javascript.mustache

@@ -0,0 +1,17 @@
+import { {{client}}, EchoRequester } from '{{{import}}}';
+
+describe('Common Test Suite', () => {
+  const client = new {{client}}(process.env.ALGOLIA_APPLICATION_ID, process.env.ALGOLIA_SEARCH_KEY, { requester: new EchoRequester() });
+
+  {{#tests}}
+  test('{{testName}}', async () => {
+    const req = await client.{{method}}({{#parameters}}{{{value}}}{{^-last}}, {{/-last}}{{/parameters}});
+    expect(req).toMatchObject({
+      path: '{{{request.path}}}',
+      method: '{{{request.method}}}',
+      data: {{{request.data}}},
+    })
+  });
+
+  {{/tests}}
+});

tests/generateCTS.ts

@@ -0,0 +1,184 @@
+/* eslint-disable no-console */
+import fsp from 'fs/promises';
+import path from 'path';
+
+import Mustache from 'mustache';
+import type { OpenAPIV3 } from 'openapi-types';
+import SwaggerParser from 'swagger-parser';
+
+import openapitools from '../openapitools.json';
+
+const availableLanguages = ['javascript'] as const;
+type Language = typeof availableLanguages[number];
+
+type CTSBlock = {
+  name: string;
+  method: string;
+  parameters: any[];
+  request: {
+    path: string;
+    method: string;
+    data: string;
+  };
+};
+
+// Array of test per client
+type CTS = Record<string, CTSBlock[]>;
+
+const extensionForLanguage: Record<Language, string> = {
+  javascript: '.test.ts',
+};
+
+const cts: CTS = {};
+
+// For each generator, we map the packageName with the language and client
+const packageNames: Record<string, Record<Language, string>> = Object.entries(
+  openapitools['generator-cli'].generators
+).reduce((prev, [clientName, clientConfig]) => {
+  const obj = prev;
+  const [lang, client] = clientName.split('-') as [Language, string];
+
+  if (!(lang in prev)) {
+    obj[lang] = {};
+  }
+
+  obj[lang][client] = clientConfig.additionalProperties.packageName;
+
+  return obj;
+}, {} as Record<string, Record<string, string>>);
+
+async function createOutputDir(language: Language): Promise<void> {
+  await fsp.mkdir(`output/${language}`, { recursive: true });
+}
+
+async function* walk(
+  dir: string
+): AsyncGenerator<{ path: string; name: string }> {
+  for await (const d of await fsp.opendir(dir)) {
+    const entry = path.join(dir, d.name);
+    if (d.isDirectory()) yield* walk(entry);
+    else if (d.isFile()) yield { path: entry, name: d.name };
+  }
+}
+
+function capitalize(str: string): string {
+  return str.charAt(0).toUpperCase() + str.slice(1);
+}
+
+async function loadCTSForClient(client: string): Promise<CTSBlock[]> {
+  // load the list of operations from the spec
+  const spec = await SwaggerParser.validate(`../specs/${client}/spec.yml`);
+  const operations = Object.values(spec.paths)
+    .flatMap<OpenAPIV3.OperationObject>((p) => Object.values(p))
+    .map((obj) => obj.operationId);
+
+  const ctsClient: CTSBlock[] = [];
+
+  for await (const file of walk(`./CTS/clients/${client}`)) {
+    if (!file.name.endsWith('json')) {
+      continue;
+    }
+    const operationId = file.name.replace('.json', '');
+    const tests: CTSBlock[] = JSON.parse(
+      (await fsp.readFile(file.path)).toString()
+    );
+
+    // check test validity against spec
+    if (!operations.includes(operationId)) {
+      throw new Error(
+        `cannot find operationId ${operationId} for the ${client} client`
+      );
+    }
+
+    // for now we stringify all params for mustache to render them properly
+    for (const test of tests) {
+      for (let i = 0; i < test.parameters.length; i++) {
+        // delete the object name for now, but it could be use for `new $objectName(params)`
+        delete test.parameters[i].$objectName;
+
+        // include the `-last` param to join with comma in mustache
+        test.parameters[i] = {
+          value: JSON.stringify(test.parameters[i]),
+          '-last': i === test.parameters.length - 1,
+        };
+      }
+
+      // stringify request.data too
+      test.request.data = JSON.stringify(test.request.data);
+    }
+    ctsClient.push(...tests);
+  }
+  return ctsClient;
+}
+
+async function loadCTS(): Promise<void> {
+  for await (const { name: client } of await fsp.opendir('./CTS/clients/')) {
+    cts[client] = await loadCTSForClient(client);
+  }
+}
+
+async function loadTemplate(language: Language): Promise<string> {
+  return (await fsp.readFile(`CTS/templates/${language}.mustache`)).toString();
+}
+
+async function generateCode(language: Language): Promise<void> {
+  const template = await loadTemplate(language);
+  await createOutputDir(language);
+  for (const client in cts) {
+    if (cts[client].length === 0) {
+      continue;
+    }
+
+    const code = Mustache.render(template, {
+      import: packageNames[language][client],
+      client: `${capitalize(client)}Api`,
+      tests: cts[client],
+    });
+    await fsp.writeFile(
+      `output/${language}/${client}${extensionForLanguage[language]}`,
+      code
+    );
+  }
+}
+
+function printUsage(): void {
+  console.log(`usage: generateCTS all | language1 language2...`);
+  console.log(`\tavailable languages: ${availableLanguages.join(',')}`);
+  // eslint-disable-next-line no-process-exit
+  process.exit(1);
+}
+
+async function parseCLI(args: string[]): Promise<void> {
+  if (args.length < 3) {
+    console.log('not enough arguments');
+    printUsage();
+  }
+
+  let toGenerate: Language[];
+  if (args.length === 3 && args[2] === 'all') {
+    toGenerate = [...availableLanguages];
+  } else {
+    const languages = args[2].split(' ') as Language[];
+    const unknownLanguages = languages.filter(
+      (lang) => !availableLanguages.includes(lang)
+    );
+    if (unknownLanguages.length > 0) {
+      console.log('unkown language(s): ', unknownLanguages.join(', '));
+      printUsage();
+    }
+    toGenerate = languages;
+  }
+
+  try {
+    await loadCTS();
+    for (const lang of toGenerate) {
+      generateCode(lang);
+    }
+  } catch (e) {
+    if (e instanceof Error) {
+      console.error(e);
+    }
+  }
+}
+
+parseCLI(process.argv);

tests/jest.config.js

@@ -0,0 +1,7 @@
+require('dotenv').config();
+
+/** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+};