diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index 8d96b26a4a..cd444f6df5 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -190,8 +190,7 @@ runs: 'clients/algoliasearch-client-javascript/packages/algoliasearch/**', 'clients/algoliasearch-client-javascript/packages/client-search/**', 'clients/algoliasearch-client-javascript/packages/client-analytics/**', - 'clients/algoliasearch-client-javascript/packages/client-personalization/**', - 'templates/javascript/**' + 'clients/algoliasearch-client-javascript/packages/client-personalization/**' )}} - name: Restore built JavaScript search client @@ -207,7 +206,8 @@ runs: 'clients/algoliasearch-client-javascript/packages/client-search/builds/**', 'clients/algoliasearch-client-javascript/packages/client-search/package.json', 'specs/bundled/search.yml', - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} - name: Restore built JavaScript recommend client @@ -223,7 +223,8 @@ runs: 'clients/algoliasearch-client-javascript/packages/recommend/builds/**', 'clients/algoliasearch-client-javascript/packages/recommend/package.json', 'specs/bundled/recommend.yml', - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} - name: Restore built JavaScript query-suggestions client @@ -239,7 +240,8 @@ runs: 'clients/algoliasearch-client-javascript/packages/client-query-suggestions/builds/**', 'clients/algoliasearch-client-javascript/packages/client-query-suggestions/package.json', 'specs/bundled/query-suggestions.yml', - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} - name: Restore built JavaScript personalization client @@ -255,7 +257,8 @@ runs: 'clients/algoliasearch-client-javascript/packages/client-personalization/builds/**', 'clients/algoliasearch-client-javascript/packages/client-personalization/package.json', 'specs/bundled/personalization.yml', - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} - name: Restore built JavaScript analytics client @@ -271,7 +274,8 @@ runs: 'clients/algoliasearch-client-javascript/packages/client-analytics/builds/**', 'clients/algoliasearch-client-javascript/packages/client-analytics/package.json', 'specs/bundled/analytics.yml', - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} - name: Restore built JavaScript abtesting client @@ -287,7 +291,8 @@ runs: 'clients/algoliasearch-client-javascript/packages/client-abtesting/builds/**', 'clients/algoliasearch-client-javascript/packages/client-abtesting/package.json', 'specs/bundled/abtesting.yml', - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} - name: Restore built JavaScript insights client @@ -303,7 +308,8 @@ runs: 'clients/algoliasearch-client-javascript/packages/client-insights/builds/**', 'clients/algoliasearch-client-javascript/packages/client-insights/package.json', 'specs/bundled/insights.yml', - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} - name: Restore built JavaScript sources client @@ -319,7 +325,8 @@ runs: 'clients/algoliasearch-client-javascript/packages/client-sources/builds/**', 'clients/algoliasearch-client-javascript/packages/client-sources/package.json', 'specs/bundled/sources.yml', - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} - name: Restore built JavaScript predict client @@ -335,11 +342,12 @@ runs: 'clients/algoliasearch-client-javascript/packages/client-predict/builds/**', 'clients/algoliasearch-client-javascript/packages/client-predict/package.json', 'specs/bundled/predict.yml', - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} # Restore Java clients: used during 'cts' or 'codegen' - - name: Restore built Java client + - name: Restore built Java search client if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: @@ -347,13 +355,60 @@ runs: key: | ${{ env.CACHE_VERSION }}-${{ hashFiles( - 'clients/algoliasearch-client-java-2/**', + 'clients/algoliasearch-client-java-2/search/**', + 'clients/algoliasearch-client-java-2/model/search/**', + 'specs/bundled/search.yml', 'templates/java/**', - 'specs/bundled/search.yml' + 'generators/src/**' )}} # Restore PHP clients: used during 'cts' or 'codegen' - - name: Restore built PHP client + - name: Restore built PHP search client + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: clients/algoliasearch-client-php + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'clients/algoliasearch-client-php/lib/Api/SearchApi.php', + 'clients/algoliasearch-client-php/lib/Configuration/SearchConfig.php', + 'specs/bundled/search.yml', + 'templates/php/**', + 'generators/src/**' + )}} + + - name: Restore built PHP recommend client + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: clients/algoliasearch-client-php + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'clients/algoliasearch-client-php/lib/Api/RecommendApi.php', + 'clients/algoliasearch-client-php/lib/Configuration/RecommendConfig.php', + 'specs/bundled/recommend.yml', + 'templates/php/**', + 'generators/src/**' + )}} + + - name: Restore built PHP personalization client + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: clients/algoliasearch-client-php + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'clients/algoliasearch-client-php/lib/Api/PersonalizationApi.php', + 'clients/algoliasearch-client-php/lib/Configuration/PersonalizationConfig.php', + 'specs/bundled/personalization.yml', + 'templates/php/**', + 'generators/src/**' + )}} + + - name: Restore built PHP analytics client if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: @@ -361,8 +416,56 @@ runs: key: | ${{ env.CACHE_VERSION }}-${{ hashFiles( - 'clients/algoliasearch-client-php/*', - 'specs/bundled/search.yml' + 'clients/algoliasearch-client-php/lib/Api/AnalyticsApi.php', + 'clients/algoliasearch-client-php/lib/Configuration/AnalyticsConfig.php', + 'specs/bundled/analytics.yml', + 'templates/php/**', + 'generators/src/**' + )}} + + - name: Restore built PHP insights client + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: clients/algoliasearch-client-php + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'clients/algoliasearch-client-php/lib/Api/InsightsApi.php', + 'clients/algoliasearch-client-php/lib/Configuration/InsightsConfig.php', + 'specs/bundled/insights.yml', + 'templates/php/**', + 'generators/src/**' + )}} + + - name: Restore built PHP abtesting client + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: clients/algoliasearch-client-php + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'clients/algoliasearch-client-php/lib/Api/AbTestingApi.php', + 'clients/algoliasearch-client-php/lib/Configuration/AbTestingConfig.php', + 'specs/bundled/abtesting.yml', + 'templates/php/**', + 'generators/src/**' + )}} + + - name: Restore built PHP query-suggestions client + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: clients/algoliasearch-client-php + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'clients/algoliasearch-client-php/lib/Api/QuerySuggestionsApi.php', + 'clients/algoliasearch-client-php/lib/Configuration/QuerySuggestionsConfig.php', + 'specs/bundled/query-suggestions.yml', + 'templates/php/**', + 'generators/src/**' )}} - name: Install JavaScript dependencies diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index d1a31fa0ef..151af1480a 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -78,7 +78,7 @@ runs: run: | base_changed=${{ steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 }} - matrix=$(./scripts/ci/create-spec-matrix.sh $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }}) + matrix=$(yarn workspace scripts createMatrix $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }}) if [[ $matrix == '{"client":["no-run"]}' ]]; then run="false" @@ -101,7 +101,7 @@ runs: algoliasearch_changed=${{ steps.diff.outputs.JS_ALGOLIASEARCH_CHANGED > 0 }} common_changed=${{ steps.diff.outputs.JS_COMMON_CHANGED > 0 }} - matrix=$(./scripts/ci/create-client-matrix.sh javascript $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }}) + matrix=$(yarn workspace scripts createMatrix $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }} javascript) if [[ $algoliasearch_changed == 'true' || $base_changed == 'true' ]]; then echo "Running algoliasearch: true" @@ -132,7 +132,7 @@ runs: run: | base_changed=${{ steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.GENERATORS_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} - matrix=$(./scripts/ci/create-client-matrix.sh java $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }}) + matrix=$(yarn workspace scripts createMatrix $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }} java) if [[ $matrix == '{"client":["no-run"]}' ]]; then run="false" @@ -153,7 +153,7 @@ runs: run: | base_changed=${{ steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.GENERATORS_CHANGED > 0 || steps.diff.outputs.PHP_TEMPLATE_CHANGED > 0 }} - matrix=$(./scripts/ci/create-client-matrix.sh php $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }}) + matrix=$(yarn workspace scripts createMatrix $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }} php) if [[ $matrix == '{"client":["no-run"]}' ]]; then run="false" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3755cc055d..e1e445bef8 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -166,7 +166,8 @@ jobs: format('{0}/builds/**', matrix.client.folder), format('{0}/package.json', matrix.client.folder), format('specs/bundled/{0}.yml', matrix.client.name), - 'templates/javascript/**' + 'templates/javascript/**', + 'generators/src/**' )}} - name: Generate '${{ matrix.client.name }}' client @@ -210,8 +211,7 @@ jobs: 'clients/algoliasearch-client-javascript/packages/algoliasearch/**', 'clients/algoliasearch-client-javascript/packages/client-search/**', 'clients/algoliasearch-client-javascript/packages/client-analytics/**', - 'clients/algoliasearch-client-javascript/packages/client-personalization/**', - 'templates/javascript/**' + 'clients/algoliasearch-client-javascript/packages/client-personalization/**' )}} - name: Build 'algoliasearch' client @@ -248,9 +248,11 @@ jobs: key: | ${{ env.CACHE_VERSION }}-${{ hashFiles( - format('{0}/**', matrix.client.folder), + format('{0}/{1}/**', matrix.client.folder, matrix.client.name), + format('{0}/model/{1}/**', matrix.client.folder, matrix.client.name), + format('specs/bundled/{0}.yml', matrix.client.name), 'templates/java/**', - format('specs/bundled/{0}.yml', matrix.client.name) + 'generators/src/**' )}} - name: Generate '${{ matrix.client.name }}' client @@ -291,8 +293,11 @@ jobs: key: | ${{ env.CACHE_VERSION }}-${{ hashFiles( - format('{0}/**', matrix.client.folder), - format('specs/bundled/{0}.yml', matrix.client.name) + format('{0}/lib/Api/{1}.php', matrix.client.folder, matrix.client.api), + format('{0}/lib/Configuration/{1}.php', matrix.client.folder, matrix.client.config), + format('specs/bundled/{0}.yml', matrix.client.name), + 'templates/php/**', + 'generators/src/**' )}} - name: Generate '${{ matrix.client.name }}' client diff --git a/scripts/ci/create-client-matrix.sh b/scripts/ci/create-client-matrix.sh deleted file mode 100755 index 67a9319d17..0000000000 --- a/scripts/ci/create-client-matrix.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -LANGUAGE=$1 -BASE_CHANGED=$2 -BASE_BRANCH=$3 - -CLIENTS=$(cat openapitools.json | jq --arg lang $LANGUAGE -c '."generator-cli".generators - | with_entries( - if (.key | test($lang + "-.*")) then - ({key:.key,value:.value}) - else - empty - end - ) - | to_entries - | map({ - name:.key | sub($lang + "-";""), - folder:.value.output | sub("#{cwd}/";"") - }) - | .[]') - -to_test='{"client": []}' -for pair in $CLIENTS; do - name=$(echo $pair | jq -r '.name') - folder=$(echo $pair | jq -r '.folder') - spec_changed=$(git diff --shortstat $BASE_BRANCH..HEAD -- specs/$name | wc -l | tr -d ' ') - client_changed=$(git diff --stat $BASE_BRANCH..HEAD -- $folder | wc -l | tr -d ' ') - if [[ $BASE_CHANGED == "true" || $spec_changed != "0" || $client_changed != "0" ]]; then - to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') - fi -done - -# Convert the array to json for the matrix -if [[ $(echo $to_test | jq '.client | length') == 0 ]]; then - # client cannot be empty or the matrix will fail - matrix='{"client":["no-run"]}' -else - matrix=$(echo $to_test | jq -c) -fi - -echo $matrix diff --git a/scripts/ci/create-spec-matrix.sh b/scripts/ci/create-spec-matrix.sh deleted file mode 100755 index 96e46f1e1e..0000000000 --- a/scripts/ci/create-spec-matrix.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -BASE_CHANGED=$1 -BASE_BRANCH=$2 - -GENERATORS=( $(cat openapitools.json | jq '."generator-cli".generators' | jq -r 'keys[]') ) -SPECS=() - -to_check='{"client":[]}' -for generator in "${GENERATORS[@]}"; do - client=${generator#*-} - if [[ ! ${SPECS[*]} =~ $client ]]; then - changed=$(git diff --shortstat $BASE_BRANCH..HEAD -- specs/$client | wc -l) - SPECS+=($client) - if [[ $BASE_CHANGED == "true" || $changed > 0 ]]; then - to_check=$(echo $to_check | jq --arg client $client '.client |= .+ [$client]') - fi - fi -done - -# Convert the array to json for the matrix -if [[ $(echo $to_check | jq '.client | length') == 0 ]]; then - # client cannot be empty or the matrix will fail - matrix='{"client":["no-run"]}' -else - matrix=$(echo $to_check | jq -c) -fi - -echo $matrix diff --git a/scripts/ci/createMatrix.ts b/scripts/ci/createMatrix.ts new file mode 100644 index 0000000000..6ffe07b9d8 --- /dev/null +++ b/scripts/ci/createMatrix.ts @@ -0,0 +1,123 @@ +import { CLIENTS, GENERATORS, run } from '../common'; +import type { Language } from '../types'; + +type CreateMatrix = { + baseChanged: boolean; + baseBranch: string; + language?: Language; +}; + +type ClientMatrix = { + name: string; + folder: string; + config?: string; + api?: string; +}; + +type Matrix = { + client: TMatrix[]; +}; + +// This empty matrix is required by the CI, otherwise it throws +const EMPTY_MATRIX = JSON.stringify({ client: ['no-run'] }); + +/** + * Returns the number of diff between a `branch` and the current HEAD for the given `path`. + */ +async function getNbGitDiff(branch: string, path: string): Promise { + return parseInt( + ( + await run(`git diff --shortstat ${branch}..HEAD -- ${path} | wc -l`) + ).trim(), + 10 + ); +} + +async function getClientMatrix({ + language, + baseBranch, + baseChanged, +}: CreateMatrix): Promise> { + const matrix: Matrix = { client: [] }; + + for (const { + client, + output, + additionalProperties, + ...options + } of Object.values(GENERATORS)) { + if ( + options.language !== language || + // `algoliasearch` is an aggregation of clients + client === 'algoliasearch' + ) { + continue; + } + + const specChanges = await getNbGitDiff(baseBranch, `specs/${client}`); + const clientChanges = await getNbGitDiff(baseBranch, output); + + if (clientChanges === 0 && specChanges === 0 && !baseChanged) { + continue; + } + + const matchedGenerator: ClientMatrix = { + name: client, + folder: output, + }; + + // Extra informations for the PHP matrix in order to properly scope the + // GitHub action cache + if (language === 'php') { + matchedGenerator.config = additionalProperties.configClassname; + matchedGenerator.api = additionalProperties.configClassname.replace( + 'Config', + 'Api' + ); + } + + matrix.client.push(matchedGenerator); + } + + return matrix; +} + +async function getSpecMatrix({ + baseBranch, + baseChanged, +}: CreateMatrix): Promise> { + const matrix: Matrix = { client: [] }; + + for (const client of CLIENTS) { + const specChanges = await getNbGitDiff(baseBranch, `specs/${client}`); + + if (specChanges === 0 && !baseChanged) { + continue; + } + + matrix.client.push(client); + } + + return matrix; +} + +async function createMatrix(opts: CreateMatrix): Promise { + const matrix = opts.language + ? await getClientMatrix(opts) + : await getSpecMatrix(opts); + + // eslint-disable-next-line no-console + console.log( + matrix.client.length === 0 ? EMPTY_MATRIX : JSON.stringify(matrix) + ); +} + +if (require.main === module) { + const args = process.argv.slice(2); + + createMatrix({ + baseChanged: args[0] === 'true', + baseBranch: args[1], + language: args[2] as Language, + }); +} diff --git a/scripts/package.json b/scripts/package.json index 37a56637e4..5c9f98f3ba 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -2,6 +2,7 @@ "name": "scripts", "version": "1.0.0", "scripts": { + "createMatrix": "ts-node ci/createMatrix.ts", "createReleaseIssue": "ts-node release/create-release-issue.ts", "processRelease": "ts-node release/process-release.ts", "pushGeneratedCode": "ts-node ci/codegen/pushGeneratedCode.ts",