diff --git a/build.gradle.kts b/build.gradle.kts index 25bbe249..28472b13 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -61,7 +61,7 @@ val isDevelopmentRelease = !hasProperty("finalRelease") val releaseVersion = releaseVersion() val releaseNotes = releaseNotes() val distributionVersion = distributionVersion() -val buildScanSummaryVersion = "1.0.3-2024.1" +val buildScanSummaryVersion = "1.0.4-2024.1" allprojects { version = releaseVersion.get() diff --git a/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh b/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh index 403776db..95a158a7 100755 --- a/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh +++ b/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh @@ -40,6 +40,7 @@ develocity_server='' interactive_mode='' ci_build_scan_url='' +remote_build_cache_type='' remote_build_cache_url='' mapping_file='' @@ -110,9 +111,9 @@ wizard_execute() { collect_gradle_details print_bl - explain_remote_build_cache_url + explain_collect_remote_build_cache print_bl - collect_remote_build_cache_url + collect_remote_build_cache explain_command_to_repeat_experiment_after_collecting_parameters print_bl @@ -136,6 +137,7 @@ wizard_execute() { map_additional_script_args() { ci_build_scan_url="${_arg_first_build_ci}" + remote_build_cache_type="${_arg_remote_build_cache_type}" remote_build_cache_url="${_arg_remote_build_cache_url}" mapping_file="${_arg_mapping_file}" } @@ -151,6 +153,10 @@ validate_required_args() { if [[ "${enable_develocity}" == "on" && -z "${develocity_server}" ]]; then _PRINT_HELP=yes die "ERROR: Missing required argument when enabling Develocity on a project not already connected: --develocity-server" "${INVALID_INPUT}" fi + + if [[ -n "${remote_build_cache_type}" && "${remote_build_cache_type}" != 'http' && "${remote_build_cache_type}" != 'gradle-enterprise' && "${remote_build_cache_type}" != 'develocity' ]]; then + _PRINT_HELP=yes die "ERROR: Invalid value for argument --remote-build-cache-type. Values are 'develocity', 'gradle-enterprise', or 'http'." "${INVALID_INPUT}" + fi } fetch_build_params_from_build_scan() { @@ -160,6 +166,9 @@ fetch_build_params_from_build_scan() { } read_build_params_from_build_scan_data() { + if [[ "${remote_build_cache_types[0]}" == "disabled" ]]; then + die "ERROR: Remote build cache was disabled for the first build. Enable the remote build cache in the build and restart the experiment." + fi if [ -z "${git_repo}" ]; then git_repo="${git_repos[0]}" project_name="$(basename -s .git "${git_repo}")" @@ -170,6 +179,9 @@ read_build_params_from_build_scan_data() { if [ -z "${git_commit_id}" ]; then git_commit_id="${git_commit_ids[0]}" fi + if [[ -z "${remote_build_cache_type}" && "${remote_build_cache_types[0]}" != "unknown" ]]; then + remote_build_cache_type="${remote_build_cache_types[0]}" + fi if [ -z "${remote_build_cache_url}" ]; then remote_build_cache_url="${remote_build_cache_urls[0]}" fi @@ -196,6 +208,9 @@ validate_build_config() { execute_build() { local args args=(--build-cache --init-script "${INIT_SCRIPTS_DIR}/configure-remote-build-caching.gradle") + if [ -n "${remote_build_cache_type}" ]; then + args+=("-Ddevelocity.build-validation.remoteBuildCacheType=${remote_build_cache_type}") + fi if [ -n "${remote_build_cache_url}" ]; then args+=("-Ddevelocity.build-validation.remoteBuildCacheUrl=${remote_build_cache_url}") fi @@ -360,7 +375,7 @@ EOF print_interactive_text "${text}" } -explain_remote_build_cache_url() { +explain_collect_remote_build_cache() { local text IFS='' read -r -d '' text <<EOF The local build will connect to the given remote build cache. The remote build @@ -369,11 +384,30 @@ EOF print_interactive_text "${text}" } +collect_remote_build_cache() { + collect_remote_build_cache_type + collect_remote_build_cache_url +} + +collect_remote_build_cache_type() { + local default_remote_cache_type="<project default>" + prompt_for_setting "What is the remote build cache connector type to use? [develocity, gradle-enterprise, http, or <BLANK>]" "${remote_build_cache_type}" "${default_remote_cache_type}" remote_build_cache_type + + if [[ -n "${remote_build_cache_type}" && "${remote_build_cache_type}" != 'http' && "${remote_build_cache_type}" != 'gradle-enterprise' && "${remote_build_cache_type}" != 'develocity' ]]; then + print_bl + die "ERROR: Invalid value for remote build cache connector type. Values are 'develocity', 'gradle-enterprise', 'http', or <BLANK> for project default." "${INVALID_INPUT}" + fi + + if [[ "${remote_build_cache_type}" == "${default_remote_cache_type}" ]]; then + remote_build_cache_type='' + fi +} + collect_remote_build_cache_url() { - local default_remote_cache="<project default>" - prompt_for_setting "What is the remote build cache url to use?" "${remote_build_cache_url}" "${default_remote_cache}" remote_build_cache_url + local default_remote_cache_url="<project default>" + prompt_for_setting "What is the remote build cache url to use?" "${remote_build_cache_url}" "${default_remote_cache_url}" remote_build_cache_url - if [[ "${remote_build_cache_url}" == "${default_remote_cache}" ]]; then + if [[ "${remote_build_cache_url}" == "${default_remote_cache_url}" ]]; then remote_build_cache_url='' fi } @@ -465,6 +499,10 @@ generate_command_to_repeat_experiment() { cmd+=("-m" "${mapping_file}") fi + if [ -n "${remote_build_cache_type}" ]; then + cmd+=("-y" "${remote_build_cache_type}") + fi + if [ -n "${remote_build_cache_url}" ]; then cmd+=("-u" "${remote_build_cache_url}") fi diff --git a/components/scripts/gradle/gradle-init-scripts/configure-remote-build-caching.gradle b/components/scripts/gradle/gradle-init-scripts/configure-remote-build-caching.gradle index fff09597..c51d1443 100644 --- a/components/scripts/gradle/gradle-init-scripts/configure-remote-build-caching.gradle +++ b/components/scripts/gradle/gradle-init-scripts/configure-remote-build-caching.gradle @@ -4,23 +4,134 @@ static getInputParam(Gradle gradle, String name) { return gradle.startParameter.systemPropertiesArgs[name] ?: System.getProperty(name) ?: System.getenv(envVarName) } +def isTopLevelBuild = !gradle.parent + +def expDir = getInputParam(gradle, 'develocity.build-validation.expDir') +def remoteBuildCacheType = getInputParam(gradle, 'develocity.build-validation.remoteBuildCacheType') def remoteBuildCacheUrl = getInputParam(gradle, 'develocity.build-validation.remoteBuildCacheUrl') -settingsEvaluated { settings -> +def docsRoot = 'https://docs.gradle.com/develocity/gradle-plugin' + +settingsEvaluated { Settings settings -> settings.buildCache { local { enabled = false } - remote(HttpBuildCache) { - enabled = true - push = false - if (remoteBuildCacheUrl) { - url = withPathTrailingSlash(new URI(remoteBuildCacheUrl)) + + if (remoteBuildCacheType) { + if (isInvalidRemoteBuildCacheType(remoteBuildCacheType)) { + failInvalidRemoteBuildCacheType(remoteBuildCacheType, expDir) + } + + if (isTopLevelBuild && missingRequiredPlugin(settings, remoteBuildCacheType)) { + failMissingRequiredPlugin(remoteBuildCacheType, expDir, docsRoot) + } + + def remoteBuildCacheImplementation = getRemoteBuildCacheImplementation(settings, remoteBuildCacheType) + if (remoteBuildCacheImplementation) { + logger.debug("Configuring remote build cache implementation for '${settings.rootProject.name}' as: ${remoteBuildCacheImplementation}") + remote(remoteBuildCacheImplementation) + } + } + + logger.debug("Remote build cache implementation for '${settings.rootProject.name}' is: ${remote?.class?.name}") + if (remote) { + remote { + enabled = true + } + + def remoteBuildCacheUri = remoteBuildCacheUrl ? withPathTrailingSlash(new URI(remoteBuildCacheUrl)) : null + if (remote instanceof HttpBuildCache) { + if (remoteBuildCacheUrl) { + remote.url = remoteBuildCacheUri + } else if (!remote.url) { + failMissingUrlForHttpBuildCache(expDir, docsRoot) + } + } else if (isBuildCacheImplementationFor(remote, 'com.gradle.develocity') || isBuildCacheImplementationFor(remote, 'com.gradle.enterprise')) { + if (remoteBuildCacheUri) { + remote.server = toServerPart(remoteBuildCacheUri) + remote.path = remoteBuildCacheUri.path + } } + } else if (isTopLevelBuild) { + failMissingRemoteBuildCacheConfiguration(expDir, docsRoot) } } } +static boolean isInvalidRemoteBuildCacheType(String remoteBuildCacheType) { + return !['develocity', 'gradle-enterprise', 'http'].contains(remoteBuildCacheType) +} + +static boolean missingRequiredPlugin(Settings settings, String type) { + return type == "develocity" && !settings.pluginManager.hasPlugin('com.gradle.develocity') + || type == "gradle-enterprise" && !settings.pluginManager.hasPlugin('com.gradle.enterprise') +} + +static Class<? extends BuildCache> getRemoteBuildCacheImplementation(Settings settings, String type) { + if (type == "develocity") { + return settings.develocity.buildCache + } else if (type == "gradle-enterprise") { + return settings.gradleEnterprise.buildCache + } else if (type == "http") { + return HttpBuildCache + } + return null +} + +static boolean isBuildCacheImplementationFor(BuildCache buildCache, String implementation) { + return buildCache.class.name.startsWith(implementation) +} + static URI withPathTrailingSlash(URI uri) { - uri.path.endsWith("/") ? uri : new URI(uri.scheme, uri.userInfo, uri.host, uri.port, uri.path + "/", uri.query, uri.fragment) + return uri.path.endsWith("/") ? uri : new URI(uri.scheme, uri.userInfo, uri.host, uri.port, uri.path + "/", uri.query, uri.fragment) +} + +static String toServerPart(URI uri) { + return new URI(uri.scheme, uri.userInfo, uri.host, uri.port, null, uri.query, uri.fragment) +} + +// The scripts already fail if the value of --remote-build-cache-type isn't valid. +// This is for the sake of completeness since this init script assumes it's valid. +static void failInvalidRemoteBuildCacheType(String remoteBuildCacheType, String expDir) { + def errorFile = new File(expDir, 'errors.txt') + def message = "Invalid value '${remoteBuildCacheType}' for remote build cache connector type. Values are 'develocity', 'gradle-enterprise', or 'http'." + errorFile.text = message + throw new IllegalStateException(message) +} + +// Included builds may not have the necessary plugin applied. +// Only fail if the top-level build is missing the required extension. +static void failMissingRequiredPlugin(String remoteBuildCacheType, String expDir, String docsRoot) { + def errorFile = new File(expDir, 'errors.txt') + errorFile.text = "Remote build cache connector type '${remoteBuildCacheType}' requested, but the required plugin is not applied." + if (remoteBuildCacheType == 'develocity') { + throw new IllegalStateException("Remote build cache connector type 'develocity' requested,\n" + + "but the Develocity Gradle plugin is not applied.\n" + + "Either apply it directly (see $docsRoot/current/#applying_the_plugin),\n" + + "use --enable-develocity to enable the plugin,\n" + + "or use --remote-build-cache-type to choose a different remote build cache connector type\n" + + "when running the build validation script.") + } else { + throw new IllegalStateException("Remote build cache connector type 'gradle-enterprise' requested,\n" + + "but the Gradle Enterprise Gradle plugin is not applied (see $docsRoot/legacy/#applying_the_plugin).") + } +} + +// Gradle already fails in this case, but handling it here means we can fail the experiment more +// gracefully and provide guidance the user. +static void failMissingUrlForHttpBuildCache(String expDir, String docsRoot) { + def errorFile = new File(expDir, 'errors.txt') + errorFile.text = 'A remote build cache URL has not been configured in the project or on the command line.' + throw new IllegalStateException("A remote build cache URL is not configured.\n" + + "Either configure it directly (see $docsRoot/current/#using_gradles_built_in_http_connector) in the project,\n" + + "or use --remote-build-cache-url when running the build validation script.") +} + +static void failMissingRemoteBuildCacheConfiguration(String expDir, String docsRoot) { + def errorFile = new File(expDir, 'errors.txt') + errorFile.text = "Remote build cache is not configured for the project." + throw new IllegalStateException("Remote build cache is not configured for the project.\n" + + "Either configure it directly (see $docsRoot/current/#using_the_develocity_connector),\n" + + "or use --remote-build-cache-type when running the build validation script.") } diff --git a/components/scripts/lib/build-scan-parse.sh b/components/scripts/lib/build-scan-parse.sh index d472c553..df4a77ff 100644 --- a/components/scripts/lib/build-scan-parse.sh +++ b/components/scripts/lib/build-scan-parse.sh @@ -10,6 +10,8 @@ git_branches=() git_commit_ids=() requested_tasks=() build_outcomes=() +remote_build_cache_types=() +remote_build_cache_class_names=() remote_build_cache_urls=() remote_build_cache_shards=() @@ -77,7 +79,7 @@ parse_build_scan_row() { local run_num - while IFS=, read -r run_num field_1 field_2 field_3 field_4 field_5 field_6 field_7 field_8 field_9 field_10 field_11 field_12 field_13 field_14 field_15 field_16 field_17 field_18 field_19 field_20; do + while IFS=, read -r run_num field_1 field_2 field_3 field_4 field_5 field_6 field_7 field_8 field_9 field_10 field_11 field_12 field_13 field_14 field_15 field_16 field_17 field_18 field_19 field_20 field_21 field_22; do debug "Build Scan $field_4 is for build $run_num" # The project_name should be overridden by Build Scan data if it is @@ -127,21 +129,23 @@ parse_build_scan_row() { # The below fields are always set by Build Scan data regardless of their # previous value and are always safe to override. - remote_build_cache_urls[run_num]="${field_10}" - remote_build_cache_shards[run_num]="${field_11}" + remote_build_cache_types[run_num]="${field_10}" + remote_build_cache_class_names[run_num]="${field_11}" + remote_build_cache_urls[run_num]="${field_12}" + remote_build_cache_shards[run_num]="${field_13}" # Build caching performance metrics - avoided_up_to_date_num_tasks[run_num]="${field_12}" - avoided_up_to_date_avoidance_savings[run_num]="${field_13}" - avoided_from_cache_num_tasks[run_num]="${field_14}" - avoided_from_cache_avoidance_savings[run_num]="${field_15}" - executed_cacheable_num_tasks[run_num]="${field_16}" - executed_cacheable_duration[run_num]="${field_17}" - executed_not_cacheable_num_tasks[run_num]="${field_18}" - executed_not_cacheable_duration[run_num]="${field_19}" + avoided_up_to_date_num_tasks[run_num]="${field_14}" + avoided_up_to_date_avoidance_savings[run_num]="${field_15}" + avoided_from_cache_num_tasks[run_num]="${field_16}" + avoided_from_cache_avoidance_savings[run_num]="${field_17}" + executed_cacheable_num_tasks[run_num]="${field_18}" + executed_cacheable_duration[run_num]="${field_19}" + executed_not_cacheable_num_tasks[run_num]="${field_20}" + executed_not_cacheable_duration[run_num]="${field_21}" # Other build metrics - serialization_factors[run_num]="${field_20}" + serialization_factors[run_num]="${field_22}" done <<< "${build_scan_row}" } diff --git a/components/scripts/lib/cli-parsers/gradle/05-cli-parser.m4 b/components/scripts/lib/cli-parsers/gradle/05-cli-parser.m4 index 568a9ce4..47ca6b80 100755 --- a/components/scripts/lib/cli-parsers/gradle/05-cli-parser.m4 +++ b/components/scripts/lib/cli-parsers/gradle/05-cli-parser.m4 @@ -3,6 +3,7 @@ # Created by argbash-init v2.10.0 # ARG_OPTIONAL_SINGLE([first-build-ci],[1],[]) # ARG_OPTIONAL_SINGLE([mapping-file],[m],[]) +# ARG_OPTIONAL_SINGLE([remote-build-cache-type],[y],[]) # ARG_OPTIONAL_SINGLE([remote-build-cache-url],[u],[]) # ARG_OPTIONAL_BOOLEAN([fail-if-not-fully-cacheable],[f],[]) # ARG_HELP([This function is overridden later on.]) @@ -30,6 +31,7 @@ function print_help() { print_option_usage -p print_option_usage -t print_option_usage -a + print_option_usage "-y, --remote-build-cache-type" "Specifies the remote build cache connector type to use in the second build run locally. Values are 'develocity', 'gradle-enterprise', or 'http'." print_option_usage "-u, --remote-build-cache-url" "Specifies the URL for the remote build cache to access in the second build run locally." print_option_usage -s print_option_usage -e diff --git a/components/scripts/lib/summary.sh b/components/scripts/lib/summary.sh index fec3cc6f..f31928a7 100644 --- a/components/scripts/lib/summary.sh +++ b/components/scripts/lib/summary.sh @@ -119,6 +119,12 @@ detect_warnings_from_build_scans() { if [ -z "${build_outcomes[i]}" ]; then warnings+=("Failed to fetch build scan data for the ${ORDINALS[i]} build.") fi + if [ "${remote_build_cache_types[i]}" == "unknown" ]; then + # "Develocity Build Validation Scripts" is specifically mentioned as to + # not imply Develocity itself does not work with other remote build cache + # implementations. + warnings+=("The ${ORDINALS[i]} build ran using a remote build cache implementation not officially supported by the experiment.") + fi done local value_mismatch=false @@ -126,13 +132,21 @@ detect_warnings_from_build_scans() { [[ "${git_repos[0]}" != "${git_repos[1]}" ]] || [[ "${git_branches[0]}" != "${git_branches[1]}" ]] || [[ "${git_commit_ids[0]}" != "${git_commit_ids[1]}" ]] || - [[ "${requested_tasks[0]}" != "${requested_tasks[1]}" ]]; then + [[ "${requested_tasks[0]}" != "${requested_tasks[1]}" ]] || + [[ "${remote_build_cache_urls[0]}" != "${remote_build_cache_urls[1]}" ]] || + [[ "${remote_build_cache_class_names[0]}" != "${remote_build_cache_class_names[1]}" ]]; then value_mismatch=true fi if [[ "${value_mismatch}" == "true" ]]; then warnings+=("Differences were detected between the two builds. This may skew the outcome of the experiment.") fi + if [ "${remote_build_cache_urls[0]}" != "${remote_build_cache_urls[1]}" ]; then + warnings+=("The two builds ran with different remote build cache URLs configured.") + fi + if [ "${remote_build_cache_class_names[0]}" != "${remote_build_cache_class_names[1]}" ]; then + warnings+=("The two builds ran using different remote build cache implementations.") + fi if [[ "${unknown_values}" == "true" ]]; then warnings+=("Some of the build properties could not be determined. This makes it uncertain if the experiment has run correctly.") fi diff --git a/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh b/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh index 8ad8b5f1..dac9d8d9 100755 --- a/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh +++ b/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh @@ -158,6 +158,9 @@ fetch_build_params_from_build_scan() { } read_build_params_from_build_scan_data() { + if [[ "${remote_build_cache_types[0]}" == "disabled" ]]; then + die "ERROR: Remote build cache was disabled for the first build. Enable the remote build cache in the build and restart the experiment." + fi if [ -z "${git_repo}" ]; then git_repo="${git_repos[0]}" project_name="$(basename -s .git "${git_repo}")" diff --git a/release/changes.md b/release/changes.md index 2a59dcbc..b5c7cba1 100644 --- a/release/changes.md +++ b/release/changes.md @@ -1,6 +1,8 @@ > [!IMPORTANT] > The distributions of the Develocity Build Validation Scripts prefixed with `gradle-enterprise` are deprecated and will be removed in a future release. Migrate to the distributions prefixed with `develocity` instead. +- [NEW] Support Develocity and Gradle Enterprise remote build cache connectors in the Gradle CI/Local experiment +- [NEW] Better handling of remote build cache misconfigurations - [FIX] Scripts do not wait long enough for build scans to become available when `--fail-if-not-fully-cacheable` is used - [FIX] Successful exit code returned when performance characteristics are unknown and `--fail-if-not-fully-cacheable` is used - [FIX] Gradle experiments do not disable background Build Scan publication