Skip to content

Commit dba57b3

Browse files
committed
Don't upload multiple times to same artifact in build workflow
The build workflow produces binaries for a range of target hosts. This is done by using a job matrix in the GitHub Actions workflow that produces each build in a parallel job. GitHub Actions workflow artifacts are used to transfer the generated files between sequential jobs in the workflow. The "actions/upload-artifact" action is used for this purpose. Previously, a single artifact was used for this purpose, with each of the parallel jobs uploading its own generated files to that artifact. However, support for uploading multiple times to a single artifact was dropped in version 4.0.0 of the "actions/upload-artifact" action. So it is now necessary to use a dedicated artifact for each of the builds. These can be downloaded in aggregate by using the artifact name globbing and merging features which were introduced in version 4.1.0 of the "actions/download-artifact" action.
1 parent 90d3d77 commit dba57b3

File tree

1 file changed

+51
-32
lines changed

1 file changed

+51
-32
lines changed

Diff for: .github/workflows/build.yml

+51-32
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,9 @@ env:
4848
GO_VERSION: '1.21'
4949
# See: https://github.com/actions/setup-node/#readme
5050
NODE_VERSION: '18.17'
51-
JOB_TRANSFER_ARTIFACT: build-artifacts
51+
JOB_TRANSFER_ARTIFACT_PREFIX: build-artifacts-
5252
CHANGELOG_ARTIFACTS: changelog
53-
STAGED_CHANNEL_FILES_ARTIFACT: staged-channel-files
53+
STAGED_CHANNEL_FILE_ARTIFACT_PREFIX: staged-channel-file-
5454
BASE_BUILD_DATA: |
5555
- config:
5656
# Human identifier for the job.
@@ -68,6 +68,8 @@ env:
6868
certificate-extension: pfx
6969
# Container for windows cert signing
7070
certificate-container: INSTALLER_CERT_WINDOWS_CONTAINER
71+
# Arbitrary identifier used to give the workflow artifact uploaded by each "build" matrix job a unique name.
72+
job-transfer-artifact-suffix: Windows_64bit
7173
# Quoting on the value is required here to allow the same comparison expression syntax to be used for this
7274
# and the companion needs.select-targets.outputs.merge-channel-files property (output values always have string
7375
# type).
@@ -91,6 +93,7 @@ env:
9193
{
9294
\"image\": \"ghcr.io/arduino/arduino-ide/linux:main\"
9395
}
96+
job-transfer-artifact-suffix: Linux_64bit
9497
mergeable-channel-file: 'false'
9598
artifacts:
9699
- path: '*Linux_64bit.zip'
@@ -107,6 +110,7 @@ env:
107110
certificate-secret: APPLE_SIGNING_CERTIFICATE_P12
108111
certificate-password-secret: KEYCHAIN_PASSWORD
109112
certificate-extension: p12
113+
job-transfer-artifact-suffix: macOS_64bit
110114
mergeable-channel-file: 'true'
111115
artifacts:
112116
- path: '*macOS_64bit.dmg'
@@ -121,6 +125,7 @@ env:
121125
certificate-secret: APPLE_SIGNING_CERTIFICATE_P12
122126
certificate-password-secret: KEYCHAIN_PASSWORD
123127
certificate-extension: p12
128+
job-transfer-artifact-suffix: macOS_arm64
124129
mergeable-channel-file: 'true'
125130
artifacts:
126131
- path: '*macOS_arm64.dmg'
@@ -233,7 +238,7 @@ jobs:
233238
) | \
234239
yq \
235240
--output-format json \
236-
'[.[].artifacts.[]]'
241+
'map(.artifacts[] + (.config | pick(["job-transfer-artifact-suffix"])))'
237242
)"
238243
239244
# The build matrix produces two macOS jobs (x86 and ARM) so the "channel update info files"
@@ -252,7 +257,7 @@ jobs:
252257
echo "${{ env.BASE_BUILD_DATA }}" | \
253258
yq \
254259
--output-format json \
255-
'[.[].artifacts.[]]'
260+
'map(.artifacts[] + (.config | pick(["job-transfer-artifact-suffix"])))'
256261
)"
257262
258263
merge_channel_files="false"
@@ -417,13 +422,13 @@ jobs:
417422
matrix.config.mergeable-channel-file == 'true'
418423
with:
419424
if-no-files-found: error
420-
name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }}
425+
name: ${{ env.STAGED_CHANNEL_FILE_ARTIFACT_PREFIX }}${{ matrix.config.job-transfer-artifact-suffix }}
421426
path: ${{ matrix.config.working-directory && format('{0}/{1}', matrix.config.working-directory, env.STAGED_CHANNEL_FILES_PATH) || env.STAGED_CHANNEL_FILES_PATH }}
422427

423-
- name: Upload [GitHub Actions]
428+
- name: Upload builds to job transfer artifact
424429
uses: actions/upload-artifact@v4
425430
with:
426-
name: ${{ env.JOB_TRANSFER_ARTIFACT }}
431+
name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}${{ matrix.config.job-transfer-artifact-suffix }}
427432
path: ${{ matrix.config.working-directory && format('{0}/{1}', matrix.config.working-directory, env.BUILD_ARTIFACTS_PATH) || env.BUILD_ARTIFACTS_PATH }}
428433

429434
- name: Manual Clean up for self-hosted runners
@@ -449,16 +454,17 @@ jobs:
449454
- name: Checkout
450455
uses: actions/checkout@v4
451456

452-
- name: Download staged-for-merge channel files artifact
457+
- name: Download staged-for-merge channel file artifacts
453458
uses: actions/download-artifact@v4
454459
with:
455-
name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }}
460+
merge-multiple: true
456461
path: ${{ env.CHANNEL_FILES_PATH }}
462+
pattern: ${{ env.STAGED_CHANNEL_FILE_ARTIFACT_PREFIX }}*
457463

458-
- name: Remove no longer needed artifact
464+
- name: Remove no longer needed artifacts
459465
uses: geekyeggo/delete-artifact@v5
460466
with:
461-
name: ${{ env.STAGED_CHANNEL_FILES_ARTIFACT }}
467+
name: ${{ env.STAGED_CHANNEL_FILE_ARTIFACT_PREFIX }}*
462468

463469
- name: Install Node.js
464470
uses: actions/setup-node@v4
@@ -488,11 +494,11 @@ jobs:
488494
--channel "${{ needs.build-type-determination.outputs.channel-name }}" \
489495
--input "${{ env.CHANNEL_FILES_PATH }}"
490496
491-
- name: Upload merged channel files to job transfer artifact
497+
- name: Upload merged channel files job transfer artifact
492498
uses: actions/upload-artifact@v4
493499
with:
494500
if-no-files-found: error
495-
name: ${{ env.JOB_TRANSFER_ARTIFACT }}
501+
name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}channel-files
496502
path: ${{ env.CHANNEL_FILES_PATH }}
497503

498504
artifacts:
@@ -503,22 +509,25 @@ jobs:
503509
if: always() && needs.build.result != 'skipped'
504510
runs-on: ubuntu-latest
505511

512+
env:
513+
BUILD_ARTIFACTS_FOLDER: build-artifacts
514+
506515
strategy:
507516
matrix:
508517
artifact: ${{ fromJson(needs.select-targets.outputs.artifact-matrix) }}
509518

510519
steps:
511-
- name: Download job transfer artifact
520+
- name: Download job transfer artifact that contains ${{ matrix.artifact.name }} tester build
512521
uses: actions/download-artifact@v4
513522
with:
514-
name: ${{ env.JOB_TRANSFER_ARTIFACT }}
515-
path: ${{ env.JOB_TRANSFER_ARTIFACT }}
523+
name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}${{ matrix.artifact.job-transfer-artifact-suffix }}
524+
path: ${{ env.BUILD_ARTIFACTS_FOLDER }}
516525

517526
- name: Upload tester build artifact
518527
uses: actions/upload-artifact@v4
519528
with:
520529
name: ${{ matrix.artifact.name }}
521-
path: ${{ env.JOB_TRANSFER_ARTIFACT }}/${{ matrix.artifact.path }}
530+
path: ${{ env.BUILD_ARTIFACTS_FOLDER }}/${{ matrix.artifact.path }}
522531

523532
changelog:
524533
needs:
@@ -561,11 +570,11 @@ jobs:
561570
562571
echo "$BODY" > CHANGELOG.txt
563572
564-
- name: Upload Changelog [GitHub Actions]
573+
- name: Upload changelog job transfer artifact
565574
if: needs.build-type-determination.outputs.is-nightly == 'true'
566575
uses: actions/upload-artifact@v4
567576
with:
568-
name: ${{ env.JOB_TRANSFER_ARTIFACT }}
577+
name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}changelog
569578
path: CHANGELOG.txt
570579

571580
publish:
@@ -584,18 +593,23 @@ jobs:
584593
needs.build-type-determination.outputs.publish-to-s3 == 'true' &&
585594
needs.build-type-determination.outputs.is-nightly == 'true'
586595
runs-on: ubuntu-latest
596+
597+
env:
598+
ARTIFACTS_FOLDER: build-artifacts
599+
587600
steps:
588-
- name: Download [GitHub Actions]
601+
- name: Download all job transfer artifacts
589602
uses: actions/download-artifact@v4
590603
with:
591-
name: ${{ env.JOB_TRANSFER_ARTIFACT }}
592-
path: ${{ env.JOB_TRANSFER_ARTIFACT }}
604+
merge-multiple: true
605+
path: ${{ env.ARTIFACTS_FOLDER }}
606+
pattern: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}*
593607

594608
- name: Publish Nightly [S3]
595609
uses: docker://plugins/s3
596610
env:
597-
PLUGIN_SOURCE: '${{ env.JOB_TRANSFER_ARTIFACT }}/*'
598-
PLUGIN_STRIP_PREFIX: '${{ env.JOB_TRANSFER_ARTIFACT }}/'
611+
PLUGIN_SOURCE: '${{ env.ARTIFACTS_FOLDER }}/*'
612+
PLUGIN_STRIP_PREFIX: '${{ env.ARTIFACTS_FOLDER }}/'
599613
PLUGIN_TARGET: '/arduino-ide/nightly'
600614
PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }}
601615
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -616,12 +630,17 @@ jobs:
616630
needs.changelog.result == 'success' &&
617631
needs.build-type-determination.outputs.is-release == 'true'
618632
runs-on: ubuntu-latest
633+
634+
env:
635+
ARTIFACTS_FOLDER: build-artifacts
636+
619637
steps:
620-
- name: Download [GitHub Actions]
638+
- name: Download all job transfer artifacts
621639
uses: actions/download-artifact@v4
622640
with:
623-
name: ${{ env.JOB_TRANSFER_ARTIFACT }}
624-
path: ${{ env.JOB_TRANSFER_ARTIFACT }}
641+
merge-multiple: true
642+
path: ${{ env.ARTIFACTS_FOLDER }}
643+
pattern: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}*
625644

626645
- name: Get Tag
627646
id: tag_name
@@ -633,7 +652,7 @@ jobs:
633652
with:
634653
repo_token: ${{ secrets.GITHUB_TOKEN }}
635654
release_name: ${{ steps.tag_name.outputs.TAG_NAME }}
636-
file: ${{ env.JOB_TRANSFER_ARTIFACT }}/*
655+
file: ${{ env.ARTIFACTS_FOLDER }}/*
637656
tag: ${{ github.ref }}
638657
file_glob: true
639658
body: ${{ needs.changelog.outputs.BODY }}
@@ -642,8 +661,8 @@ jobs:
642661
if: needs.build-type-determination.outputs.publish-to-s3 == 'true'
643662
uses: docker://plugins/s3
644663
env:
645-
PLUGIN_SOURCE: '${{ env.JOB_TRANSFER_ARTIFACT }}/*'
646-
PLUGIN_STRIP_PREFIX: '${{ env.JOB_TRANSFER_ARTIFACT }}/'
664+
PLUGIN_SOURCE: '${{ env.ARTIFACTS_FOLDER }}/*'
665+
PLUGIN_STRIP_PREFIX: '${{ env.ARTIFACTS_FOLDER }}/'
647666
PLUGIN_TARGET: '/arduino-ide'
648667
PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }}
649668
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -661,7 +680,7 @@ jobs:
661680
runs-on: ubuntu-latest
662681

663682
steps:
664-
- name: Remove unneeded job transfer artifact
683+
- name: Remove unneeded job transfer artifacts
665684
uses: geekyeggo/delete-artifact@v5
666685
with:
667-
name: ${{ env.JOB_TRANSFER_ARTIFACT }}
686+
name: ${{ env.JOB_TRANSFER_ARTIFACT_PREFIX }}*

0 commit comments

Comments
 (0)