From 15fbc64f71d3eb99ea6b3c2995632188d272dab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Thu, 7 Apr 2022 13:41:35 +0200 Subject: [PATCH 1/5] chore: use js scripts to set output variables --- .github/actions/setup/action.yml | 27 +---- scripts/ci/codegen/__tests__/codegen.test.ts | 4 +- scripts/ci/codegen/pushGeneratedCode.ts | 9 +- scripts/ci/codegen/spreadGeneration.ts | 14 +-- scripts/ci/codegen/text.ts | 7 +- scripts/ci/createMatrix.ts | 34 +++--- scripts/ci/setRunVariables.ts | 119 +++++++++++++++++++ scripts/ci/utils.ts | 24 ++++ scripts/package.json | 1 + 9 files changed, 178 insertions(+), 61 deletions(-) create mode 100644 scripts/ci/setRunVariables.ts create mode 100644 scripts/ci/utils.ts diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index e176501322..e351c33743 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -47,32 +47,7 @@ runs: baseRef=${{ github.base_ref }} origin=$( [[ -z $baseRef ]] && echo $previousCommit || echo "origin/$baseRef" ) - echo "Checking diff with branch: $origin" - - echo "::set-output name=GITHUB_ACTIONS_CHANGED::$(git diff --shortstat $origin..HEAD -- .github/actions .github/workflows | wc -l)" - - echo "::set-output name=SPECS_CHANGED::$(git diff --shortstat $origin..HEAD -- specs | wc -l)" - echo "::set-output name=COMMON_SPECS_CHANGED::$(git diff --shortstat $origin..HEAD -- specs/common | wc -l)" - - echo "::set-output name=TESTS_CHANGED::$(git diff --shortstat $origin..HEAD -- tests | wc -l)" - - echo "::set-output name=SCRIPTS_CHANGED::$(git diff --shortstat $origin..HEAD -- scripts | wc -l)" - - echo "::set-output name=GENERATORS_CHANGED::$(git diff --shortstat $origin..HEAD -- generators | wc -l)" - - echo "::set-output name=JS_CLIENT_CHANGED::$(git diff --shortstat $origin..HEAD -- clients/algoliasearch-client-javascript | wc -l)" - echo "::set-output name=JS_ALGOLIASEARCH_CHANGED::$(git diff --shortstat $origin..HEAD -- clients/algoliasearch-client-javascript/packages/algoliasearch clients/algoliasearch-client-javascript/packages/client-search clients/algoliasearch-client-javascript/packages/client-analytics clients/algoliasearch-client-javascript/packages/client-personalization | wc -l)" - echo "::set-output name=JS_COMMON_CHANGED::$(git diff --shortstat $origin..HEAD -- clients/algoliasearch-client-javascript/packages/client-common clients/algoliasearch-client-javascript/packages/requester-browser-xhr clients/algoliasearch-client-javascript/packages/requester-node-http | wc -l)" - echo "::set-output name=JS_COMMON_TESTS_CHANGED::$(git diff --shortstat $origin..HEAD -- clients/algoliasearch-client-javascript/packages/client-common/src/__tests__ | wc -l)" - echo "::set-output name=JS_TEMPLATE_CHANGED::$(git diff --shortstat $origin..HEAD -- templates/javascript | wc -l)" - - echo "::set-output name=JAVA_CLIENT_CHANGED::$(git diff --shortstat $origin..HEAD -- clients/algoliasearch-client-java-2 | wc -l)" - echo "::set-output name=JAVA_TEMPLATE_CHANGED::$(git diff --shortstat $origin..HEAD -- templates/java | wc -l)" - - echo "::set-output name=PHP_CLIENT_CHANGED::$(git diff --shortstat $origin..HEAD -- clients/algoliasearch-client-php | wc -l)" - echo "::set-output name=PHP_TEMPLATE_CHANGED::$(git diff --shortstat $origin..HEAD -- templates/php | wc -l)" - - echo "::set-output name=ORIGIN_BRANCH::$origin" + yarn workspace scripts setRunVariables $origin - name: Compute specs matrix if: inputs.type != 'minimal' diff --git a/scripts/ci/codegen/__tests__/codegen.test.ts b/scripts/ci/codegen/__tests__/codegen.test.ts index 4bcf4680c4..32df0daf0e 100644 --- a/scripts/ci/codegen/__tests__/codegen.test.ts +++ b/scripts/ci/codegen/__tests__/codegen.test.ts @@ -1,6 +1,6 @@ import { cleanGeneratedBranch } from '../cleanGeneratedBranch'; import { pushGeneratedCode } from '../pushGeneratedCode'; -import commentText from '../text'; +import commentText, { GENERATED_MAIN_BRANCH } from '../text'; import { getCommentBody, upsertGenerationComment, @@ -66,7 +66,7 @@ describe('codegen', () => { expect(await getCommentBody('cleanup')).toMatchInlineSnapshot(` "### ✗ The generated branch has been deleted. - If the PR has been merged, you can check the generated code on the [\`generated/main\` branch](https://github.com/algolia/api-clients-automation/tree/generated/main)." + If the PR has been merged, you can check the generated code on the [\`${GENERATED_MAIN_BRANCH}\` branch](https://github.com/algolia/api-clients-automation/tree/${GENERATED_MAIN_BRANCH})." `); }); diff --git a/scripts/ci/codegen/pushGeneratedCode.ts b/scripts/ci/codegen/pushGeneratedCode.ts index cf71b8a07a..bd4ed677a9 100644 --- a/scripts/ci/codegen/pushGeneratedCode.ts +++ b/scripts/ci/codegen/pushGeneratedCode.ts @@ -1,6 +1,7 @@ /* eslint-disable no-console */ import { run } from '../../common'; import { configureGitHubAuthor } from '../../release/common'; +import { getNbGitDiff } from '../utils'; const PR_NUMBER = parseInt(process.env.PR_NUMBER || '0', 10); const FOLDERS_TO_CHECK = 'yarn.lock openapitools.json clients specs/bundled'; @@ -22,9 +23,11 @@ export async function pushGeneratedCode(): Promise { const generatedCodeBranch = `generated/${baseBranch}`; if ( - (await run( - `git status --porcelain ${FOLDERS_TO_CHECK} | wc -l | tr -d ' '` - )) === '0' + (await getNbGitDiff({ + branch: baseBranch, + head: 'noHead', + path: FOLDERS_TO_CHECK, + })) === 0 ) { console.log(`No generated code changes found for '${baseBranch}'.`); diff --git a/scripts/ci/codegen/spreadGeneration.ts b/scripts/ci/codegen/spreadGeneration.ts index 25711540fe..28177e0aed 100644 --- a/scripts/ci/codegen/spreadGeneration.ts +++ b/scripts/ci/codegen/spreadGeneration.ts @@ -10,14 +10,9 @@ import { toAbsolutePath, } from '../../common'; import { getLanguageFolder } from '../../config'; -import { - cloneRepository, - configureGitHubAuthor, - OWNER, - REPO, -} from '../../release/common'; +import { cloneRepository, configureGitHubAuthor } from '../../release/common'; -const GENERATED_MAIN_BRANCH = `generated/main`; +import { GENERATED_MAIN_BRANCH, REPO_URL } from './text'; export function decideWhereToSpread(commitMessage: string): string[] { if (commitMessage.startsWith('chore: release')) { @@ -40,10 +35,7 @@ export function cleanUpCommitMessage(commitMessage: string): string { return commitMessage; } - return [ - result[1], - `https://github.com/${OWNER}/${REPO}/pull/${result[2]}`, - ].join('\n\n'); + return [result[1], `${REPO_URL}/pull/${result[2]}`].join('\n\n'); } async function spreadGeneration(): Promise { diff --git a/scripts/ci/codegen/text.ts b/scripts/ci/codegen/text.ts index 14e942c004..318f4bde5e 100644 --- a/scripts/ci/codegen/text.ts +++ b/scripts/ci/codegen/text.ts @@ -1,6 +1,7 @@ -import { OWNER, REPO } from '../../release/common'; +import { MAIN_BRANCH, OWNER, REPO } from '../../release/common'; -const REPO_URL = `https://github.com/${OWNER}/${REPO}`; +export const REPO_URL = `https://github.com/${OWNER}/${REPO}`; +export const GENERATED_MAIN_BRANCH = `generated/${MAIN_BRANCH}`; export default { notification: { @@ -13,7 +14,7 @@ export default { }, cleanup: { header: '### ✗ The generated branch has been deleted.', - body: `If the PR has been merged, you can check the generated code on the [\`generated/main\` branch](${REPO_URL}/tree/generated/main).`, + body: `If the PR has been merged, you can check the generated code on the [\`${GENERATED_MAIN_BRANCH}\` branch](${REPO_URL}/tree/${GENERATED_MAIN_BRANCH}).`, }, codegen: { header: '### ✔️ Code generated!', diff --git a/scripts/ci/createMatrix.ts b/scripts/ci/createMatrix.ts index 6ffe07b9d8..459febda35 100644 --- a/scripts/ci/createMatrix.ts +++ b/scripts/ci/createMatrix.ts @@ -1,6 +1,8 @@ -import { CLIENTS, GENERATORS, run } from '../common'; +import { CLIENTS, GENERATORS } from '../common'; import type { Language } from '../types'; +import { getNbGitDiff } from './utils'; + type CreateMatrix = { baseChanged: boolean; baseBranch: string; @@ -21,18 +23,6 @@ type Matrix = { // This empty matrix is required by the CI, otherwise it throws const EMPTY_MATRIX = JSON.stringify({ client: ['no-run'] }); -/** - * Returns the number of diff between a `branch` and the current HEAD for the given `path`. - */ -async function getNbGitDiff(branch: string, path: string): Promise { - return parseInt( - ( - await run(`git diff --shortstat ${branch}..HEAD -- ${path} | wc -l`) - ).trim(), - 10 - ); -} - async function getClientMatrix({ language, baseBranch, @@ -54,8 +44,14 @@ async function getClientMatrix({ continue; } - const specChanges = await getNbGitDiff(baseBranch, `specs/${client}`); - const clientChanges = await getNbGitDiff(baseBranch, output); + const specChanges = await getNbGitDiff({ + branch: baseBranch, + path: `specs/${client}`, + }); + const clientChanges = await getNbGitDiff({ + branch: baseBranch, + path: output, + }); if (clientChanges === 0 && specChanges === 0 && !baseChanged) { continue; @@ -89,7 +85,10 @@ async function getSpecMatrix({ const matrix: Matrix = { client: [] }; for (const client of CLIENTS) { - const specChanges = await getNbGitDiff(baseBranch, `specs/${client}`); + const specChanges = await getNbGitDiff({ + branch: baseBranch, + path: `specs/${client}`, + }); if (specChanges === 0 && !baseChanged) { continue; @@ -101,6 +100,9 @@ async function getSpecMatrix({ return matrix; } +/** + * Creates a matrix for the CI jobs based on the files that changed. + */ async function createMatrix(opts: CreateMatrix): Promise { const matrix = opts.language ? await getClientMatrix(opts) diff --git a/scripts/ci/setRunVariables.ts b/scripts/ci/setRunVariables.ts new file mode 100644 index 0000000000..abdad6a05a --- /dev/null +++ b/scripts/ci/setRunVariables.ts @@ -0,0 +1,119 @@ +/* eslint-disable no-console */ +import { getNbGitDiff } from './utils'; + +const JS_CLIENT_FOLDER = 'clients/algoliasearch-client-javascript'; + +/** + * Exhaustive list of output variables to use in the CI. + * + * Those variables are used to determine if jobs should run, based on the changes + * made in their respective `path`s. + * + * Negative paths should start with `:!`. + * + * The variable will be accessible in the CI via `steps.diff.outputs.`. + */ +const VARIABLES_TO_CHECK = [ + { + name: 'GITHUB_ACTIONS_CHANGED', + path: ['.github/actions', '.github/workflows'], + }, + { + name: 'SPECS_CHANGED', + path: ['specs', ':!specs/bundled'], + }, + { + name: 'COMMON_SPECS_CHANGED', + path: ['specs/common'], + }, + { + name: 'TESTS_CHANGED', + path: ['tests'], + }, + { + name: 'SCRIPTS_CHANGED', + path: ['scripts'], + }, + { + name: 'GENERATORS_CHANGED', + path: ['generators'], + }, + { + name: 'JS_CLIENT_CHANGED', + path: [JS_CLIENT_FOLDER, `:!${JS_CLIENT_FOLDER}/.github`], + }, + { + name: 'JS_ALGOLIASEARCH_CHANGED', + path: [ + `${JS_CLIENT_FOLDER}/packages/algoliasearch`, + `${JS_CLIENT_FOLDER}/packages/client-search`, + `${JS_CLIENT_FOLDER}/packages/client-analytics`, + `${JS_CLIENT_FOLDER}/packages/client-personalization`, + ], + }, + { + name: 'JS_COMMON_CHANGED', + path: [ + `${JS_CLIENT_FOLDER}/packages/client-common`, + `${JS_CLIENT_FOLDER}/packages/requester-browser-xhr`, + `${JS_CLIENT_FOLDER}/packages/requester-node-http`, + ], + }, + { + name: 'JS_COMMON_TESTS_CHANGED', + path: [`${JS_CLIENT_FOLDER}/packages/client-common/src/__tests__`], + }, + { + name: 'JS_TEMPLATE_CHANGED', + path: ['templates/javascript'], + }, + { + name: 'JAVA_CLIENT_CHANGED', + path: ['clients/algoliasearch-client-java-2'], + }, + { + name: 'JAVA_TEMPLATE_CHANGED', + path: ['templates/java'], + }, + { + name: 'PHP_CLIENT_CHANGED', + path: ['clients/algoliasearch-client-php'], + }, + { + name: 'PHP_TEMPLATE_CHANGED', + path: ['templates/php'], + }, +]; + +/** + * Outputs variables used in the CI to determine if a job should run. + */ +async function setRunVariables({ + originBranch, +}: { + originBranch: string; +}): Promise { + console.log(`Checking diff between ${originBranch} and HEAD`); + + for (const check of VARIABLES_TO_CHECK) { + const diff = await getNbGitDiff({ + branch: originBranch, + path: check.path.join(' '), + }); + + console.log(`Found ${diff} changes for '${check.name}'`); + console.log(`::set-output name=${check.name}::${diff}`); + } + + console.log(`::set-output name=ORIGIN_BRANCH::${originBranch}`); +} + +if (require.main === module) { + const [origin] = process.argv.slice(2); + + if (!origin) { + throw new Error(`Unable to retrieve the origin: ${origin}`); + } + + setRunVariables({ originBranch: origin }); +} diff --git a/scripts/ci/utils.ts b/scripts/ci/utils.ts new file mode 100644 index 0000000000..a625fa83f1 --- /dev/null +++ b/scripts/ci/utils.ts @@ -0,0 +1,24 @@ +import { run } from '../common'; + +/** + * Returns the number of diff between a `branch` and its `HEAD` for the given `path`. + * Head defaults to `HEAD`, providing `noHead` will check unstaged changes. + */ +export async function getNbGitDiff({ + branch, + head = 'HEAD', + path, +}: { + branch: string; + head?: string; + path: string; +}): Promise { + const checkHead = head === 'noHead' ? '' : `...${head}`; + + return parseInt( + ( + await run(`git diff --shortstat ${branch}${checkHead} -- ${path} | wc -l`) + ).trim(), + 10 + ); +} diff --git a/scripts/package.json b/scripts/package.json index 5c9f98f3ba..ce8036c503 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -9,6 +9,7 @@ "cleanGeneratedBranch": "ts-node ci/codegen/cleanGeneratedBranch.ts", "spreadGeneration": "ts-node ci/codegen/spreadGeneration.ts", "upsertGenerationComment": "ts-node ci/codegen/upsertGenerationComment.ts", + "setRunVariables": "ts-node ci/setRunVariables.ts", "test": "jest" }, "devDependencies": { From 83739792157338b5f6fada315cd370d25f63f4f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Thu, 7 Apr 2022 14:57:23 +0200 Subject: [PATCH 2/5] replace `noHead` with `null` --- scripts/ci/codegen/pushGeneratedCode.ts | 2 +- scripts/ci/utils.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ci/codegen/pushGeneratedCode.ts b/scripts/ci/codegen/pushGeneratedCode.ts index bd4ed677a9..0d58ddc68b 100644 --- a/scripts/ci/codegen/pushGeneratedCode.ts +++ b/scripts/ci/codegen/pushGeneratedCode.ts @@ -25,7 +25,7 @@ export async function pushGeneratedCode(): Promise { if ( (await getNbGitDiff({ branch: baseBranch, - head: 'noHead', + head: null, path: FOLDERS_TO_CHECK, })) === 0 ) { diff --git a/scripts/ci/utils.ts b/scripts/ci/utils.ts index a625fa83f1..d00c1a0576 100644 --- a/scripts/ci/utils.ts +++ b/scripts/ci/utils.ts @@ -2,7 +2,7 @@ import { run } from '../common'; /** * Returns the number of diff between a `branch` and its `HEAD` for the given `path`. - * Head defaults to `HEAD`, providing `noHead` will check unstaged changes. + * Head defaults to `HEAD`, providing `null` will check unstaged changes. */ export async function getNbGitDiff({ branch, @@ -10,10 +10,10 @@ export async function getNbGitDiff({ path, }: { branch: string; - head?: string; + head?: string | null; path: string; }): Promise { - const checkHead = head === 'noHead' ? '' : `...${head}`; + const checkHead = head === null ? '' : `...${head}`; return parseInt( ( From 8e3014d8c4c22111c778d7e642d90fe06b01d187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Thu, 7 Apr 2022 15:26:17 +0200 Subject: [PATCH 3/5] quote origin --- .github/.cache_version | 2 +- .github/actions/setup/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/.cache_version b/.github/.cache_version index a8907c025d..a3fcc7121b 100644 --- a/.github/.cache_version +++ b/.github/.cache_version @@ -1 +1 @@ -7.0.2 +7.1.0 diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index e351c33743..e089d22e53 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -47,7 +47,7 @@ runs: baseRef=${{ github.base_ref }} origin=$( [[ -z $baseRef ]] && echo $previousCommit || echo "origin/$baseRef" ) - yarn workspace scripts setRunVariables $origin + yarn workspace scripts setRunVariables "$origin" - name: Compute specs matrix if: inputs.type != 'minimal' From 2d0b5227e7afa1e3c454eb679bd6b4bddf57d244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Thu, 7 Apr 2022 16:02:54 +0200 Subject: [PATCH 4/5] same eslint version --- eslint/package.json | 2 +- yarn.lock | 47 +-------------------------------------------- 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/eslint/package.json b/eslint/package.json index b71a51b61d..884822d1d8 100644 --- a/eslint/package.json +++ b/eslint/package.json @@ -13,7 +13,7 @@ }, "devDependencies": { "@types/jest": "27.4.1", - "eslint": "8.11.0", + "eslint": "8.12.0", "jest": "27.5.1", "ts-jest": "27.1.3", "ts-node": "10.7.0", diff --git a/yarn.lock b/yarn.lock index 17a539fea7..e567d1efb6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10083,7 +10083,7 @@ __metadata: resolution: "eslint-plugin-automation-custom@workspace:eslint" dependencies: "@types/jest": 27.4.1 - eslint: 8.11.0 + eslint: 8.12.0 jest: 27.5.1 ts-jest: 27.1.3 ts-node: 10.7.0 @@ -10247,51 +10247,6 @@ __metadata: languageName: node linkType: hard -"eslint@npm:8.11.0": - version: 8.11.0 - resolution: "eslint@npm:8.11.0" - dependencies: - "@eslint/eslintrc": ^1.2.1 - "@humanwhocodes/config-array": ^0.9.2 - ajv: ^6.10.0 - chalk: ^4.0.0 - cross-spawn: ^7.0.2 - debug: ^4.3.2 - doctrine: ^3.0.0 - escape-string-regexp: ^4.0.0 - eslint-scope: ^7.1.1 - eslint-utils: ^3.0.0 - eslint-visitor-keys: ^3.3.0 - espree: ^9.3.1 - esquery: ^1.4.0 - esutils: ^2.0.2 - fast-deep-equal: ^3.1.3 - file-entry-cache: ^6.0.1 - functional-red-black-tree: ^1.0.1 - glob-parent: ^6.0.1 - globals: ^13.6.0 - ignore: ^5.2.0 - import-fresh: ^3.0.0 - imurmurhash: ^0.1.4 - is-glob: ^4.0.0 - js-yaml: ^4.1.0 - json-stable-stringify-without-jsonify: ^1.0.1 - levn: ^0.4.1 - lodash.merge: ^4.6.2 - minimatch: ^3.0.4 - natural-compare: ^1.4.0 - optionator: ^0.9.1 - regexpp: ^3.2.0 - strip-ansi: ^6.0.1 - strip-json-comments: ^3.1.0 - text-table: ^0.2.0 - v8-compile-cache: ^2.0.3 - bin: - eslint: bin/eslint.js - checksum: a06a2ea37002d6c0a4f462fe31b4411185dc3da7857fafb896eb392ba95a1289cc3538056474b2f44f08012f265bede01a39d46df4ac39ebc6d7be90e2c8f9fa - languageName: node - linkType: hard - "eslint@npm:8.12.0": version: 8.12.0 resolution: "eslint@npm:8.12.0" From 6d5b9615d6848e79caf54cba887a0a2f484e73c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Thu, 7 Apr 2022 16:21:57 +0200 Subject: [PATCH 5/5] sort tests --- scripts/cts/client/generate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cts/client/generate.ts b/scripts/cts/client/generate.ts index 20a74cbd95..e18f93bb40 100644 --- a/scripts/cts/client/generate.ts +++ b/scripts/cts/client/generate.ts @@ -54,7 +54,7 @@ async function loadTests(client: string): Promise { }); } - return testsBlocks; + return testsBlocks.sort((a, b) => a.operationId.localeCompare(b.operationId)); } export async function generateClientTests(