diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index 067dd01431..33c3574b29 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -40,6 +40,115 @@ runs: shell: bash run: curl -L "https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/5.4.0/openapi-generator-cli-5.4.0.jar" > /tmp/openapi-generator-cli.jar + # Restore bundled specs: used during 'client' generation or pushing the 'codegen' + - name: Restore built abtesting spec + if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: specs/bundled/abtesting.yml + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'specs/abtesting/**', + 'specs/common/**' + )}} + + - name: Restore built analytics spec + if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: specs/bundled/analytics.yml + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'specs/analytics/**', + 'specs/common/**' + )}} + + - name: Restore built insights spec + if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: specs/bundled/insights.yml + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'specs/insights/**', + 'specs/common/**' + )}} + + - name: Restore built personalization spec + if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: specs/bundled/personalization.yml + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'specs/personalization/**', + 'specs/common/**' + )}} + + - name: Restore built predict spec + if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: specs/bundled/predict.yml + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'specs/predict/**', + 'specs/common/**' + )}} + + - name: Restore built query-suggestions spec + if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: specs/bundled/query-suggestions.yml + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'specs/query-suggestions/**', + 'specs/common/**' + )}} + + - name: Restore built recommend spec + if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: specs/bundled/recommend.yml + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'specs/recommend/**', + 'specs/common/**' + )}} + + - name: Restore built search spec + if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: specs/bundled/search.yml + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'specs/search/**', + 'specs/common/**' + )}} + + - name: Restore built sources spec + if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: specs/bundled/sources.yml + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'specs/sources/**', + 'specs/common/**' + )}} + # Restore JavaScript clients utils: used during 'javascript' generation or 'cts' - name: Restore built JavaScript common client if: ${{ inputs.language == 'javascript' || inputs.job == 'cts' }} @@ -77,9 +186,9 @@ runs: '!clients/algoliasearch-client-javascript/packages/requester-browser-xhr/dist' )}} - # Restore JavaScript clients: used during 'cts' + # Restore JavaScript clients: used during 'cts' or 'codegen' - name: Restore built JavaScript algoliasearch client - if: ${{ inputs.job == 'cts'}} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/algoliasearch @@ -91,7 +200,7 @@ runs: )}} - name: Restore built JavaScript search client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/client-search @@ -104,7 +213,7 @@ runs: )}} - name: Restore built JavaScript recommend client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/recommend @@ -117,7 +226,7 @@ runs: )}} - name: Restore built JavaScript query-suggestions client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/client-query-suggestions @@ -130,7 +239,7 @@ runs: )}} - name: Restore built JavaScript personalization client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/client-personalization @@ -143,7 +252,7 @@ runs: )}} - name: Restore built JavaScript analytics client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/client-analytics @@ -156,7 +265,7 @@ runs: )}} - name: Restore built JavaScript abtesting client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/client-abtesting @@ -169,7 +278,7 @@ runs: )}} - name: Restore built JavaScript insights client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/client-insights @@ -182,7 +291,7 @@ runs: )}} - name: Restore built JavaScript sources client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/client-sources @@ -195,7 +304,7 @@ runs: )}} - name: Restore built JavaScript predict client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-javascript/packages/client-predict @@ -207,16 +316,29 @@ runs: 'specs/bundled/predict.yml' )}} - # Restore Java clients: used during 'cts' + # Restore Java clients: used during 'cts' or 'codegen' - name: Restore built Java client - if: ${{ inputs.job == 'cts' }} + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} uses: actions/cache@v2 with: path: clients/algoliasearch-client-java-2 key: | ${{ env.CACHE_VERSION }}-${{ hashFiles( - 'clients/algoliasearch-client-java-2/client-predict/**', - '!clients/algoliasearch-client-java-2/client-predict/target', + 'clients/algoliasearch-client-java-2/**', + '!clients/algoliasearch-client-java-2/target', + 'specs/bundled/search.yml' + )}} + + # Restore PHP clients: used during 'cts' or 'codegen' + - name: Restore built PHP client + if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} + uses: actions/cache@v2 + with: + path: clients/algoliasearch-client-php + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + 'clients/algoliasearch-client-php/*', 'specs/bundled/search.yml' )}} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3705790260..b7ae1804b7 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -8,7 +8,7 @@ on: - main env: - CACHE_VERSION: '8' + CACHE_VERSION: '9' concurrency: group: ${{ github.ref }} @@ -66,7 +66,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 10 needs: setup - if: ${{ needs.setup.outputs.RUN_SPECS == 'true' }} + if: ${{ always() && needs.setup.outputs.RUN_SPECS == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX) }} steps: @@ -80,22 +80,22 @@ jobs: uses: actions/cache@v2 with: path: specs/bundled/${{ matrix.client }}.yml - key: ${{ env.CACHE_VERSION }}-${{ hashFiles(format('specs/{0}/**', matrix.client), 'specs/common/**') }} + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + format('specs/{0}/**', matrix.client), + 'specs/common/**' + )}} - name: Building '${{ matrix.client }}' specs if: steps.cache.outputs.cache-hit != 'true' run: yarn cli build specs ${{ matrix.client }} - - name: Check diff with pushed spec - run: | - git status - exit $(git status --porcelain specs/bundled/${{ matrix.client }}.yml | wc -l) - client_javascript_common: timeout-minutes: 10 runs-on: ubuntu-20.04 needs: setup - if: ${{ needs.setup.outputs.RUN_JS_COMMON == 'true' }} + if: ${{ always() && needs.setup.outputs.RUN_JS_COMMON == 'true' }} strategy: matrix: client: @@ -131,7 +131,7 @@ jobs: - setup - specs - client_javascript_common - if: ${{ needs.setup.outputs.RUN_JS == 'true' }} + if: ${{ always() && needs.setup.outputs.RUN_JS == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.JS_MATRIX) }} steps: @@ -160,12 +160,6 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' && matrix.client.name != 'algoliasearch' run: yarn cli generate javascript ${{ matrix.client.name }} - - name: Check diff with pushed client - if: steps.cache.outputs.cache-hit != 'true' - run: | - git status - exit $(git status --porcelain ${{ matrix.client.folder }} | wc -l) - - name: Build '${{ matrix.client.name }}' client if: steps.cache.outputs.cache-hit != 'true' run: yarn cli build clients javascript ${{ matrix.client.name }} @@ -176,7 +170,7 @@ jobs: needs: - setup - specs - if: ${{ needs.setup.outputs.RUN_JAVA == 'true' }} + if: ${{ always() && needs.setup.outputs.RUN_JAVA == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.JAVA_MATRIX) }} steps: @@ -205,12 +199,6 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' run: yarn cli generate java ${{ matrix.client.name }} - - name: Check diff with pushed client - if: steps.cache.outputs.cache-hit != 'true' - run: | - git status - exit $(git status --porcelain ${{ matrix.client.folder }} | wc -l) - - name: Build '${{ matrix.client.name }}' client if: steps.cache.outputs.cache-hit != 'true' run: yarn cli build clients java ${{ matrix.client.name }} @@ -221,7 +209,7 @@ jobs: needs: - setup - specs - if: ${{ needs.setup.outputs.RUN_PHP == 'true' }} + if: ${{ always() && needs.setup.outputs.RUN_PHP == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.PHP_MATRIX) }} steps: @@ -233,16 +221,22 @@ jobs: job: client language: php + - name: Cache '${{ matrix.client.name }}' client + id: cache + uses: actions/cache@v2 + with: + path: ${{ matrix.client.folder }} + key: | + ${{ env.CACHE_VERSION }}-${{ + hashFiles( + format('{0}/**', matrix.client.folder), + format('specs/bundled/{0}.yml', matrix.client.name) + )}} + - name: Generate '${{ matrix.client.name }}' client if: steps.cache.outputs.cache-hit != 'true' run: yarn cli generate php ${{ matrix.client.name }} - - name: Check diff with pushed client - if: steps.cache.outputs.cache-hit != 'true' - run: | - git status - exit $(git status --porcelain ${{ matrix.client.folder }} | wc -l) - - name: Build '${{ matrix.client.name }}' client if: steps.cache.outputs.cache-hit != 'true' run: yarn cli build clients php ${{ matrix.client.name }} @@ -254,7 +248,7 @@ jobs: - client_javascript - client_java - client_php - if: ${{ needs.setup.outputs.RUN_CTS == 'true' }} + if: ${{ always() && needs.setup.outputs.RUN_CTS == 'true' }} steps: - uses: actions/checkout@v2 @@ -268,8 +262,30 @@ jobs: - name: Check diff with pushed CTS run: | - git status + git --no-pager diff exit $(git status --porcelain ./tests/output | wc -l) - name: Run run: yarn cli cts run + + codegen: + runs-on: ubuntu-20.04 + timeout-minutes: 10 + if: ${{ always() }} + needs: cts + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + + - name: Restore cache + uses: ./.github/actions/cache + with: + job: codegen + + - name: Push generated code + run: yarn workspace scripts pushGeneratedCode + env: + GITHUB_TOKEN: ${{ secrets.TOKEN_GENERATE_BOT }} + PR_NUMBER: ${{ github.event.number }} diff --git a/.github/workflows/codegen-cleanup.yml b/.github/workflows/codegen-cleanup.yml new file mode 100644 index 0000000000..02b40f7c00 --- /dev/null +++ b/.github/workflows/codegen-cleanup.yml @@ -0,0 +1,25 @@ +name: Codegen cleanup + +on: + pull_request: + types: [closed] + +env: + CACHE_VERSION: '9' + +jobs: + codegen: + runs-on: ubuntu-20.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: main + + - name: Setup + id: setup + uses: ./.github/actions/setup + + - name: Clean previously generated branch + run: yarn workspace scripts cleanGeneratedBranch ${{ github.head_ref }} diff --git a/.github/workflows/process-release.yml b/.github/workflows/process-release.yml index a1911ea3e2..f1b6a46113 100644 --- a/.github/workflows/process-release.yml +++ b/.github/workflows/process-release.yml @@ -6,7 +6,7 @@ on: - closed env: - CACHE_VERSION: '8' + CACHE_VERSION: '9' jobs: build: diff --git a/scripts/ci/codegen/cleanGeneratedBranch.ts b/scripts/ci/codegen/cleanGeneratedBranch.ts new file mode 100644 index 0000000000..e686243d7b --- /dev/null +++ b/scripts/ci/codegen/cleanGeneratedBranch.ts @@ -0,0 +1,31 @@ +/* eslint-disable no-console */ +import { run } from '../../common'; + +/** + * Deletes a branch for its `generated/${headRef}` name on origin. + */ +async function cleanGeneratedBranch(headRef: string): Promise { + const generatedCodeBranch = `generated/${headRef}`; + + if (!(await run(`git ls-remote --heads origin ${generatedCodeBranch}`))) { + console.log(`No branch named '${generatedCodeBranch}' was found.`); + + return; + } + + // Delete previous generations to avoid conflicts and out of date code + console.log(`Deleting generated branch: '${generatedCodeBranch}'`); + + await run(`git fetch origin ${generatedCodeBranch}`); + await run(`git push -d origin ${generatedCodeBranch}`); +} + +const args = process.argv.slice(2); + +if (!args || args.length === 0) { + throw new Error( + 'The base branch should be passed as a cli parameter of the `cleanGeneratedBranch` script.' + ); +} + +cleanGeneratedBranch(args[0]); diff --git a/scripts/ci/codegen/pushGeneratedCode.ts b/scripts/ci/codegen/pushGeneratedCode.ts new file mode 100644 index 0000000000..8fbdf7cd5e --- /dev/null +++ b/scripts/ci/codegen/pushGeneratedCode.ts @@ -0,0 +1,131 @@ +/* eslint-disable no-console */ +import { Octokit } from '@octokit/rest'; + +import { run } from '../../common'; +import { configureGitHubAuthor, OWNER, REPO } from '../../release/common'; + +if (!process.env.GITHUB_TOKEN) { + throw new Error('Environment variable `GITHUB_TOKEN` does not exist.'); +} + +const PR_NUMBER = parseInt(process.env.PR_NUMBER || '0', 10); +const FOLDERS_TO_CHECK = ['clients', 'specs/bundled']; +// this should be changed to the bot name once we have the logs +const BOT_NAME = 'shortcuts'; + +const octokit = new Octokit({ + auth: `token ${process.env.GITHUB_TOKEN}`, +}); + +async function getCommentBody(commit: string, branch: string): Promise { + const repoUrl = `https://github.com/${OWNER}/${REPO}`; + const generatedCommit = await run('git show -s --format=%H'); + const header = `✔️ codegen triggered on commit [${commit}](${repoUrl}/pull/${PR_NUMBER}/commits/${commit}).`; + const body = `🔍 Browse the generated code on branch [${branch}](${repoUrl}/tree/${branch}): [${generatedCommit}](${repoUrl}/commit/${generatedCommit}).`; + + return `${header} + +${body}`; +} + +/** + * Add or update comment to the current `PR_NUMBER`. + */ +async function upsertCommentToPullRequest( + baseCommit: string, + generatedCodeBranch: string +): Promise { + const baseOctokitConfig = { + owner: OWNER, + repo: REPO, + issue_number: PR_NUMBER, + }; + + try { + // Search for a previous comment from our bot. + const previousComment = await octokit.rest.issues + .listComments(baseOctokitConfig) + .then( + (res) => + res.data.filter( + (comment) => + comment.user?.login === BOT_NAME && + // this shouldn't be needed once we have a proper bot running + comment.body?.startsWith('✔️ codegen triggered on commit') + )[0] + ); + const commentBody = await getCommentBody(baseCommit, generatedCodeBranch); + + if (previousComment?.id) { + console.log(`Previous bot comment found ${previousComment.id}.`); + + await octokit.rest.issues.updateComment({ + ...baseOctokitConfig, + body: commentBody, + comment_id: previousComment.id, + }); + + return; + } + + console.log('Creating new comment.'); + await octokit.rest.issues.createComment({ + ...baseOctokitConfig, + body: commentBody, + }); + } catch (e) { + throw new Error(`Error with GitHub API: ${e}`); + } +} + +/** + * Push generated code for the current `JOB` and `CLIENT` on a `generated/` branch. + */ +async function pushGeneratedCode(): Promise { + await configureGitHubAuthor(); + + const baseBranch = await run('git branch --show-current'); + console.log(`Checking codegen status on '${baseBranch}'.`); + + // determine generated branch name based on current branch + const generatedCodeBranch = `generated/${baseBranch}`; + const generatedFolders = FOLDERS_TO_CHECK.join(' '); + + if ( + (await run( + `git status --porcelain ${generatedFolders} | wc -l | tr -d ' '` + )) === '0' + ) { + console.log(`No generated code changes found for '${baseBranch}'.`); + + return; + } + + await run(`yarn workspace scripts cleanGeneratedBranch ${baseBranch}`); + + const baseCommit = await run(`git show ${baseBranch} -s --format=%H`); + console.log( + `Codegen triggered on branch '${baseBranch}' for commit ${baseCommit}.` + ); + + console.log(`Creating branch for generated code: '${generatedCodeBranch}'`); + await run(`git branch ${generatedCodeBranch}`); + const commitMessage = + await run(`git show -s --format="Generated code for commit %H. + +Co-authored-by: %an <%ae>"`); + + console.log( + `Pushing code for folders '${generatedFolders}' to generated branch: ${generatedCodeBranch}` + ); + await run(`git checkout ${generatedCodeBranch}`); + await run(`git add ${generatedFolders}`); + await run(`git commit -m "${commitMessage}"`); + await run(`git push origin ${generatedCodeBranch}`); + + if (PR_NUMBER) { + await upsertCommentToPullRequest(baseCommit, generatedCodeBranch); + } +} + +pushGeneratedCode(); diff --git a/scripts/package.json b/scripts/package.json index e48daed9bb..1d1ec1c693 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -5,6 +5,8 @@ "build": "tsc", "createReleaseIssue": "yarn build && node dist/scripts/release/create-release-issue.js", "processRelease": "yarn build && node dist/scripts/release/process-release.js", + "pushGeneratedCode": "yarn build && node dist/scripts/ci/codegen/pushGeneratedCode.js", + "cleanGeneratedBranch": "yarn build && node dist/scripts/ci/codegen/cleanGeneratedBranch.js", "test": "jest" }, "devDependencies": { diff --git a/scripts/release/common.ts b/scripts/release/common.ts index 512159f826..e51dc0eeda 100644 --- a/scripts/release/common.ts +++ b/scripts/release/common.ts @@ -1,4 +1,5 @@ import config from '../../config/release.config.json'; +import { run } from '../common'; export const RELEASED_TAG = config.releasedTag; export const MAIN_BRANCH = config.mainBranch; @@ -28,3 +29,10 @@ export function getMarkdownSection(markdown: string, title: string): string { } return lines.slice(0, endIndex).join('\n'); } + +export async function configureGitHubAuthor(cwd?: string): Promise { + const { name, email } = getGitAuthor(); + + await run(`git config user.name "${name}"`, { cwd }); + await run(`git config user.email "${email}"`, { cwd }); +} diff --git a/scripts/release/process-release.ts b/scripts/release/process-release.ts index b0e3a4a1e2..239d703fb1 100755 --- a/scripts/release/process-release.ts +++ b/scripts/release/process-release.ts @@ -19,8 +19,8 @@ import { OWNER, REPO, getMarkdownSection, - getGitAuthor, getTargetBranch, + configureGitHubAuthor, } from './common'; import TEXT from './text'; @@ -97,13 +97,6 @@ async function updateOpenApiTools( ); } -async function configureGitHubAuthor(cwd?: string): Promise { - await run(`git config user.name "${getGitAuthor().name}"`, { cwd }); - await run(`git config user.email "${getGitAuthor().email}"`, { - cwd, - }); -} - async function processRelease(): Promise { if (!process.env.GITHUB_TOKEN) { throw new Error('Environment variable `GITHUB_TOKEN` does not exist.');