From 82a3b5a0b23c8bef743c859b754a87a0f7348791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Tue, 10 May 2022 14:42:20 +0200 Subject: [PATCH 1/3] chore: fix specs run condition --- .github/actions/setup/action.yml | 8 +-- scripts/ci/githubActions/createMatrix.ts | 77 ++++++--------------- scripts/ci/githubActions/setRunVariables.ts | 49 +++++++------ 3 files changed, 54 insertions(+), 80 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index e35182de48..36b10b7e64 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -98,7 +98,7 @@ runs: echo "::set-output name=RUN_JS_ALGOLIASEARCH::true" fi - if [[ ${{ steps.gen-matrix.outputs.RUN_GEN_JAVASCRIPT || steps.diff.outputs.JS_UTILS_CHANGED > 0 }} ]]; then + if [[ ${{ steps.gen-matrix.outputs.RUN_GEN_JAVASCRIPT || steps.diff.outputs.JAVASCRIPT_UTILS_CHANGED > 0 }} ]]; then echo "Running JavaScript utils: true" echo "::set-output name=RUN_JS_UTILS::true" fi @@ -135,12 +135,12 @@ outputs: RUN_CTS: description: Determine if the `cts` jobs should run value: ${{ + steps.spec-matrix.outputs.RUN_SPECS || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || - steps.diff.outputs.SHOULD_RUN_SPECS > 0 || steps.diff.outputs.TESTS_CHANGED > 0 || - steps.diff.outputs.JS_CLIENT_CHANGED > 0 || - steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 || + steps.diff.outputs.JAVASCRIPT_CLIENT_CHANGED > 0 || + steps.diff.outputs.JAVASCRIPT_TEMPLATE_CHANGED > 0 || steps.diff.outputs.JAVA_CLIENT_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} diff --git a/scripts/ci/githubActions/createMatrix.ts b/scripts/ci/githubActions/createMatrix.ts index 292e75fafd..4a071e2201 100644 --- a/scripts/ci/githubActions/createMatrix.ts +++ b/scripts/ci/githubActions/createMatrix.ts @@ -4,39 +4,13 @@ import { getLanguageFolder, getTestOutputFolder } from '../../config'; import type { Language } from '../../types'; import { getNbGitDiff } from '../utils'; -import { DEPENDENCIES } from './setRunVariables'; +import { DEPENDENCIES, COMMON_DEPENDENCIES } from './setRunVariables'; import type { ClientMatrix, CreateMatrix, Matrix, SpecMatrix } from './types'; import { computeCacheKey, isBaseChanged } from './utils'; // This empty matrix is required by the CI, otherwise it throws const EMPTY_MATRIX = { client: ['no-run'] }; -/** - * List of dependencies based on the language, inherited from `./setRunVariables.ts` in a more dynamic form. - */ -const MATRIX_DEPENDENCIES = { - common: { - GITHUB_ACTIONS_CHANGED: DEPENDENCIES.GITHUB_ACTIONS_CHANGED, - SCRIPTS_CHANGED: DEPENDENCIES.SCRIPTS_CHANGED, - COMMON_SPECS_CHANGED: DEPENDENCIES.COMMON_SPECS_CHANGED, - }, - clients: { - common: { - GENERATORS_CHANGED: DEPENDENCIES.GENERATORS_CHANGED, - }, - javascript: { - JS_UTILS_CHANGED: DEPENDENCIES.JS_UTILS_CHANGED, - JS_TEMPLATE_CHANGED: DEPENDENCIES.JS_TEMPLATE_CHANGED, - }, - php: { - PHP_TEMPLATE_CHANGED: DEPENDENCIES.PHP_TEMPLATE_CHANGED, - }, - java: { - JAVA_TEMPLATE_CHANGED: DEPENDENCIES.JAVA_TEMPLATE_CHANGED, - }, - }, -}; - type ToRunMatrix = { path: string; toRun: string[]; @@ -62,6 +36,20 @@ async function getClientMatrix(baseBranch: string): Promise { continue; } + const languageDependencies = Object.entries(DEPENDENCIES).reduce( + (finalDeps, [key, deps]) => { + if (key.startsWith(`${language.toUpperCase()}_`)) { + return { + ...finalDeps, + [key]: deps, + }; + } + + return finalDeps; + }, + {} as Record + ); + const bundledSpec = client === 'algoliasearch-lite' ? 'search' : client; const specChanges = await getNbGitDiff({ branch: baseBranch, @@ -72,9 +60,8 @@ async function getClientMatrix(baseBranch: string): Promise { path: output, }); const baseChanged = await isBaseChanged(baseBranch, { - ...MATRIX_DEPENDENCIES.common, - ...MATRIX_DEPENDENCIES.clients.common, - ...MATRIX_DEPENDENCIES.clients[language], + ...COMMON_DEPENDENCIES, + ...languageDependencies, }); // No changes found, we don't put this job in the matrix @@ -82,7 +69,6 @@ async function getClientMatrix(baseBranch: string): Promise { continue; } - console.log(`::set-output name=RUN_GEN_${language.toUpperCase()}::true`); matrix[language].toRun.push(client); matrix[language].cacheToCompute.push(`specs/${bundledSpec}`); } @@ -111,6 +97,7 @@ async function getClientMatrix(baseBranch: string): Promise { language )}`, }); + console.log(`::set-output name=RUN_GEN_${language.toUpperCase()}::true`); } const shouldRun = clientMatrix.client.length > 0; @@ -123,7 +110,7 @@ async function getClientMatrix(baseBranch: string): Promise { ); } -async function getSpecMatrix(baseBranch: string): Promise { +async function getSpecMatrix(): Promise { const matrix: ToRunMatrix = { path: 'specs/bundled', toRun: [], @@ -133,31 +120,9 @@ async function getSpecMatrix(baseBranch: string): Promise { for (const client of CLIENTS) { // The `algoliasearch-lite` spec is created by the `search` spec const bundledSpecName = client === 'algoliasearch-lite' ? 'search' : client; - const path = `specs/${bundledSpecName}`; - const specChanges = await getNbGitDiff({ - branch: baseBranch, - path, - }); - const baseChanged = await isBaseChanged(baseBranch, { - ...MATRIX_DEPENDENCIES.common, - ...MATRIX_DEPENDENCIES.clients.common, - }); - - // No changes found, we don't put this job in the matrix - if (specChanges === 0 && !baseChanged) { - continue; - } matrix.toRun.push(client); - matrix.cacheToCompute.push(path); - } - - // We have nothing to run - if (matrix.toRun.length === 0) { - console.log('::set-output name=RUN_SPECS::false'); - console.log(`::set-output name=MATRIX::${JSON.stringify(EMPTY_MATRIX)}`); - - return; + matrix.cacheToCompute.push(`specs/${bundledSpecName}`); } const ciMatrix: Matrix = { @@ -186,7 +151,7 @@ async function createMatrix(opts: CreateMatrix): Promise { return await getClientMatrix(opts.baseBranch); } - return await getSpecMatrix(opts.baseBranch); + return await getSpecMatrix(); } if (require.main === module) { diff --git a/scripts/ci/githubActions/setRunVariables.ts b/scripts/ci/githubActions/setRunVariables.ts index 01071f5049..334996c05f 100644 --- a/scripts/ci/githubActions/setRunVariables.ts +++ b/scripts/ci/githubActions/setRunVariables.ts @@ -11,8 +11,22 @@ const PHP_CLIENT_FOLDER = getLanguageFolder('php'); const CLIENTS_COMMON_FILES = [ 'config/openapitools.json', 'config/clients.config.json', + 'generators/src/main/java/com/algolia/codegen/Utils.java', ]; +/** + * Dependencies that are common to every specs, clients or CTS jobs. + */ +export const COMMON_DEPENDENCIES = { + GITHUB_ACTIONS_CHANGED: [ + '.github/actions', + '.github/workflows', + '.github/.cache_version', + ], + SCRIPTS_CHANGED: ['scripts'], + COMMON_SPECS_CHANGED: ['specs/common'], +}; + /** * Exhaustive list of output variables to use in the CI. * @@ -24,23 +38,9 @@ const CLIENTS_COMMON_FILES = [ * The variable will be accessible in the CI via `steps.diff.outputs.`. */ export const DEPENDENCIES = { - GITHUB_ACTIONS_CHANGED: [ - '.github/actions', - '.github/workflows', - '.github/.cache_version', - ], - SHOULD_RUN_SPECS: [ - ...CLIENTS_COMMON_FILES, - 'specs', - 'templates', - 'generators', - ':!specs/bundled', - ], - COMMON_SPECS_CHANGED: ['specs/common'], + ...COMMON_DEPENDENCIES, TESTS_CHANGED: ['tests'], - SCRIPTS_CHANGED: ['scripts'], - GENERATORS_CHANGED: ['generators'], - JS_CLIENT_CHANGED: [ + JAVASCRIPT_CLIENT_CHANGED: [ ...CLIENTS_COMMON_FILES, JS_CLIENT_FOLDER, `:!${JS_CLIENT_FOLDER}/.github`, @@ -52,7 +52,7 @@ export const DEPENDENCIES = { `${JS_CLIENT_FOLDER}/packages/client-analytics`, `${JS_CLIENT_FOLDER}/packages/client-personalization`, ], - JS_UTILS_CHANGED: [ + JAVASCRIPT_UTILS_CHANGED: [ `${JS_CLIENT_FOLDER}/packages/client-common`, `${JS_CLIENT_FOLDER}/packages/requester-browser-xhr`, `${JS_CLIENT_FOLDER}/packages/requester-node-http`, @@ -60,11 +60,20 @@ export const DEPENDENCIES = { JS_COMMON_TESTS_CHANGED: [ `${JS_CLIENT_FOLDER}/packages/client-common/src/__tests__`, ], - JS_TEMPLATE_CHANGED: ['templates/javascript'], + JAVASCRIPT_TEMPLATE_CHANGED: [ + 'templates/javascript', + 'generators/src/main/java/com/algolia/codegen/AlgoliaJavascriptGenerator.java', + ], JAVA_CLIENT_CHANGED: [...CLIENTS_COMMON_FILES, JAVA_CLIENT_FOLDER], - JAVA_TEMPLATE_CHANGED: ['templates/java'], + JAVA_TEMPLATE_CHANGED: [ + 'templates/java', + 'generators/src/main/java/com/algolia/codegen/AlgoliaJavaGenerator.java', + ], PHP_CLIENT_CHANGED: [...CLIENTS_COMMON_FILES, PHP_CLIENT_FOLDER], - PHP_TEMPLATE_CHANGED: ['templates/php'], + PHP_TEMPLATE_CHANGED: [ + 'templates/php', + 'generators/src/main/java/com/algolia/codegen/AlgoliaPhpGenerator.java', + ], }; /** From 4a529f5280017ac032a238563377daf3b292489f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Tue, 10 May 2022 15:51:11 +0200 Subject: [PATCH 2/3] less variables --- .github/actions/setup/action.yml | 12 -------- .github/workflows/check.yml | 6 ---- scripts/ci/githubActions/setRunVariables.ts | 33 +++++++++++---------- 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 36b10b7e64..bf175c0964 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -132,18 +132,6 @@ outputs: description: Determine if the `client_javascript_tests` job should run value: ${{ steps.diff.outputs.JS_COMMON_TESTS_CHANGED > 0 }} - RUN_CTS: - description: Determine if the `cts` jobs should run - value: ${{ - steps.spec-matrix.outputs.RUN_SPECS || - steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || - steps.diff.outputs.SCRIPTS_CHANGED > 0 || - steps.diff.outputs.TESTS_CHANGED > 0 || - steps.diff.outputs.JAVASCRIPT_CLIENT_CHANGED > 0 || - steps.diff.outputs.JAVASCRIPT_TEMPLATE_CHANGED > 0 || - steps.diff.outputs.JAVA_CLIENT_CHANGED > 0 || - steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} - RUN_CODEGEN: description: Determine if the `codegen` job should run value: ${{ diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 7b9aba1fe9..b1fb6bcebd 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -60,8 +60,6 @@ jobs: RUN_JS_UTILS: ${{ steps.setup.outputs.RUN_JS_UTILS }} RUN_JS_TESTS: ${{ steps.setup.outputs.RUN_JS_TESTS }} - RUN_CTS: ${{ steps.setup.outputs.RUN_CTS }} - RUN_CODEGEN: ${{ steps.setup.outputs.RUN_CODEGEN }} scripts: @@ -224,21 +222,17 @@ jobs: run: yarn cli build clients ${{ matrix.client.language }} ${{ matrix.client.toRun }} - name: Clean CTS output before generate - if: ${{ needs.setup.outputs.RUN_CTS == 'true' }} run: rm -rf ${{ matrix.client.testsOutputPath }} - name: Generate CTS - if: ${{ needs.setup.outputs.RUN_CTS == 'true' }} run: yarn cli cts generate ${{ matrix.client.language }} ${{ matrix.client.toRun }} - name: Check diff with pushed CTS - if: ${{ needs.setup.outputs.RUN_CTS == 'true' }} run: | git --no-pager diff exit $(git diff --name-only --diff-filter=d ${{ matrix.client.testsOutputPath }} | wc -l) - name: Run CTS - if: ${{ needs.setup.outputs.RUN_CTS == 'true' }} run: yarn cli cts run ${{ matrix.client.language }} - name: Zip artifact before storing diff --git a/scripts/ci/githubActions/setRunVariables.ts b/scripts/ci/githubActions/setRunVariables.ts index 334996c05f..8b7efd2eb1 100644 --- a/scripts/ci/githubActions/setRunVariables.ts +++ b/scripts/ci/githubActions/setRunVariables.ts @@ -31,46 +31,49 @@ export const COMMON_DEPENDENCIES = { * Exhaustive list of output variables to use in the CI. * * Those variables are used to determine if jobs should run, based on the changes - * made in their respective `path`s. + * made in their respective dependencies. * * Negative paths should start with `:!`. * * The variable will be accessible in the CI via `steps.diff.outputs.`. + * + * Variables starting by `LANGUAGENAME_` will be used in the `createMatrix` to determine + * if a job should be added. */ export const DEPENDENCIES = { ...COMMON_DEPENDENCIES, TESTS_CHANGED: ['tests'], - JAVASCRIPT_CLIENT_CHANGED: [ - ...CLIENTS_COMMON_FILES, - JS_CLIENT_FOLDER, - `:!${JS_CLIENT_FOLDER}/.github`, - `:!${JS_CLIENT_FOLDER}/README.md`, - ], JS_ALGOLIASEARCH_CHANGED: [ `${JS_CLIENT_FOLDER}/packages/algoliasearch`, `${JS_CLIENT_FOLDER}/packages/client-search`, `${JS_CLIENT_FOLDER}/packages/client-analytics`, `${JS_CLIENT_FOLDER}/packages/client-personalization`, ], + JS_COMMON_TESTS_CHANGED: [ + `${JS_CLIENT_FOLDER}/packages/client-common/src/__tests__`, + ], JAVASCRIPT_UTILS_CHANGED: [ `${JS_CLIENT_FOLDER}/packages/client-common`, `${JS_CLIENT_FOLDER}/packages/requester-browser-xhr`, `${JS_CLIENT_FOLDER}/packages/requester-node-http`, ], - JS_COMMON_TESTS_CHANGED: [ - `${JS_CLIENT_FOLDER}/packages/client-common/src/__tests__`, - ], - JAVASCRIPT_TEMPLATE_CHANGED: [ + JAVASCRIPT_CLIENT_CHANGED: [ + ...CLIENTS_COMMON_FILES, + JS_CLIENT_FOLDER, 'templates/javascript', 'generators/src/main/java/com/algolia/codegen/AlgoliaJavascriptGenerator.java', + `:!${JS_CLIENT_FOLDER}/.github`, + `:!${JS_CLIENT_FOLDER}/README.md`, ], - JAVA_CLIENT_CHANGED: [...CLIENTS_COMMON_FILES, JAVA_CLIENT_FOLDER], - JAVA_TEMPLATE_CHANGED: [ + JAVA_CLIENT_CHANGED: [ + ...CLIENTS_COMMON_FILES, + JAVA_CLIENT_FOLDER, 'templates/java', 'generators/src/main/java/com/algolia/codegen/AlgoliaJavaGenerator.java', ], - PHP_CLIENT_CHANGED: [...CLIENTS_COMMON_FILES, PHP_CLIENT_FOLDER], - PHP_TEMPLATE_CHANGED: [ + PHP_CLIENT_CHANGED: [ + ...CLIENTS_COMMON_FILES, + PHP_CLIENT_FOLDER, 'templates/php', 'generators/src/main/java/com/algolia/codegen/AlgoliaPhpGenerator.java', ], From 3b7b9cdfe2029969401e45d23a72a2dd2342271f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Tue, 10 May 2022 16:06:49 +0200 Subject: [PATCH 3/3] remove run_specs cond --- .github/actions/setup/action.yml | 7 +------ .github/workflows/check.yml | 2 -- scripts/ci/githubActions/createMatrix.ts | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index bf175c0964..318fe52dd1 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -108,9 +108,6 @@ outputs: description: Determine if the `scripts` job should run value: ${{ steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 }} - RUN_SPECS: - description: Determine if the `specs` job should run - value: ${{ steps.spec-matrix.outputs.RUN_SPECS }} SPECS_MATRIX: description: The generated `specs` matrix value: ${{ steps.spec-matrix.outputs.MATRIX }} @@ -134,6 +131,4 @@ outputs: RUN_CODEGEN: description: Determine if the `codegen` job should run - value: ${{ - steps.spec-matrix.outputs.RUN_SPECS == 'true' || - steps.gen-matrix.outputs.RUN_GEN == 'true' }} + value: ${{ steps.gen-matrix.outputs.RUN_GEN == 'true' }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b1fb6bcebd..754248dc98 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -50,7 +50,6 @@ jobs: outputs: RUN_SCRIPTS: ${{ steps.setup.outputs.RUN_SCRIPTS }} - RUN_SPECS: ${{ steps.setup.outputs.RUN_SPECS }} SPECS_MATRIX: ${{ steps.setup.outputs.SPECS_MATRIX }} RUN_GEN: ${{ steps.setup.outputs.RUN_GEN }} @@ -88,7 +87,6 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 10 needs: setup - if: ${{ needs.setup.outputs.RUN_SPECS == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX) }} steps: diff --git a/scripts/ci/githubActions/createMatrix.ts b/scripts/ci/githubActions/createMatrix.ts index 4a071e2201..1c3a2b13ae 100644 --- a/scripts/ci/githubActions/createMatrix.ts +++ b/scripts/ci/githubActions/createMatrix.ts @@ -139,7 +139,6 @@ async function getSpecMatrix(): Promise { ], }; - console.log('::set-output name=RUN_SPECS::true'); console.log(`::set-output name=MATRIX::${JSON.stringify(ciMatrix)}`); }