Develocity and Gradle Enterprise remote build cache connectors are supported

build.gradle.kts

@@ -61,7 +61,7 @@ val isDevelopmentRelease = !hasProperty("finalRelease")
 val releaseVersion = releaseVersion()
 val releaseNotes = releaseNotes()
 val distributionVersion = distributionVersion()
-val buildScanSummaryVersion = "1.0.3-2024.1"
+val buildScanSummaryVersion = "1.0.4-2024.1"
 
 allprojects {
     version = releaseVersion.get()

components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh

@@ -40,6 +40,7 @@ develocity_server=''
 interactive_mode=''
 
 ci_build_scan_url=''
+remote_build_cache_type=''
 remote_build_cache_url=''
 mapping_file=''
 
@@ -110,9 +111,9 @@ wizard_execute() {
   collect_gradle_details
 
   print_bl
-  explain_remote_build_cache_url
+  explain_collect_remote_build_cache
   print_bl
-  collect_remote_build_cache_url
+  collect_remote_build_cache
   explain_command_to_repeat_experiment_after_collecting_parameters
 
   print_bl
@@ -136,6 +137,7 @@ wizard_execute() {
 
 map_additional_script_args() {
   ci_build_scan_url="${_arg_first_build_ci}"
+  remote_build_cache_type="${_arg_remote_build_cache_type}"
   remote_build_cache_url="${_arg_remote_build_cache_url}"
   mapping_file="${_arg_mapping_file}"
 }
@@ -151,6 +153,10 @@ validate_required_args() {
   if [[ "${enable_develocity}" == "on" && -z "${develocity_server}" ]]; then
     _PRINT_HELP=yes die "ERROR: Missing required argument when enabling Develocity on a project not already connected: --develocity-server" "${INVALID_INPUT}"
   fi
+
+  if [[ -n "${remote_build_cache_type}" && "${remote_build_cache_type}" != 'http' && "${remote_build_cache_type}" != 'gradle-enterprise' && "${remote_build_cache_type}" != 'develocity' ]]; then
+    _PRINT_HELP=yes die "ERROR: Invalid value for argument --remote-build-cache-type. Values are 'develocity', 'gradle-enterprise', or 'http'." "${INVALID_INPUT}"
+  fi
 }
 
 fetch_build_params_from_build_scan() {
@@ -160,6 +166,9 @@ fetch_build_params_from_build_scan() {
 }
 
 read_build_params_from_build_scan_data() {
+  if [[ "${remote_build_cache_types[0]}" == "disabled" ]]; then
+    die "ERROR: Remote build cache was disabled for the first build. Enable the remote build cache in the build and restart the experiment."
+  fi
   if [ -z "${git_repo}" ]; then
     git_repo="${git_repos[0]}"
     project_name="$(basename -s .git "${git_repo}")"
@@ -170,6 +179,9 @@ read_build_params_from_build_scan_data() {
   if [ -z "${git_commit_id}" ]; then
     git_commit_id="${git_commit_ids[0]}"
   fi
+  if [[ -z "${remote_build_cache_type}" && "${remote_build_cache_types[0]}" != "unknown" ]]; then
+    remote_build_cache_type="${remote_build_cache_types[0]}"
+  fi
   if [ -z "${remote_build_cache_url}" ]; then
     remote_build_cache_url="${remote_build_cache_urls[0]}"
   fi
@@ -196,6 +208,9 @@ validate_build_config() {
 execute_build() {
   local args
   args=(--build-cache --init-script "${INIT_SCRIPTS_DIR}/configure-remote-build-caching.gradle")
+  if [ -n "${remote_build_cache_type}" ]; then
+    args+=("-Ddevelocity.build-validation.remoteBuildCacheType=${remote_build_cache_type}")
+  fi
   if [ -n "${remote_build_cache_url}" ]; then
     args+=("-Ddevelocity.build-validation.remoteBuildCacheUrl=${remote_build_cache_url}")
   fi
@@ -360,7 +375,7 @@ EOF
   print_interactive_text "${text}"
 }
 
-explain_remote_build_cache_url() {
+explain_collect_remote_build_cache() {
   local text
   IFS='' read -r -d '' text <<EOF
 The local build will connect to the given remote build cache. The remote build
@@ -369,11 +384,30 @@ EOF
   print_interactive_text "${text}"
 }
 
+collect_remote_build_cache() {
+  collect_remote_build_cache_type
+  collect_remote_build_cache_url
+}
+
+collect_remote_build_cache_type() {
+  local default_remote_cache_type="<project default>"
+  prompt_for_setting "What is the remote build cache connector type to use? [develocity, gradle-enterprise, http, or <BLANK>]" "${remote_build_cache_type}" "${default_remote_cache_type}" remote_build_cache_type
+
+  if [[ -n "${remote_build_cache_type}" && "${remote_build_cache_type}" != 'http' && "${remote_build_cache_type}" != 'gradle-enterprise' && "${remote_build_cache_type}" != 'develocity' ]]; then
+    print_bl
+    die "ERROR: Invalid value for remote build cache connector type. Values are 'develocity', 'gradle-enterprise', 'http', or <BLANK> for project default." "${INVALID_INPUT}"
+  fi
+
+  if [[ "${remote_build_cache_type}" == "${default_remote_cache_type}" ]]; then
+    remote_build_cache_type=''
+  fi
+}
+
 collect_remote_build_cache_url() {
-  local default_remote_cache="<project default>"
-  prompt_for_setting "What is the remote build cache url to use?" "${remote_build_cache_url}" "${default_remote_cache}" remote_build_cache_url
+  local default_remote_cache_url="<project default>"
+  prompt_for_setting "What is the remote build cache url to use?" "${remote_build_cache_url}" "${default_remote_cache_url}" remote_build_cache_url
 
-  if [[ "${remote_build_cache_url}" == "${default_remote_cache}" ]]; then
+  if [[ "${remote_build_cache_url}" == "${default_remote_cache_url}" ]]; then
     remote_build_cache_url=''
   fi
 }
@@ -465,6 +499,10 @@ generate_command_to_repeat_experiment() {
     cmd+=("-m" "${mapping_file}")
   fi
 
+  if [ -n "${remote_build_cache_type}" ]; then
+    cmd+=("-y" "${remote_build_cache_type}")
+  fi
+
   if [ -n "${remote_build_cache_url}" ]; then
     cmd+=("-u" "${remote_build_cache_url}")
   fi

components/scripts/gradle/gradle-init-scripts/configure-remote-build-caching.gradle

@@ -4,23 +4,134 @@ static getInputParam(Gradle gradle, String name) {
     return gradle.startParameter.systemPropertiesArgs[name] ?: System.getProperty(name) ?: System.getenv(envVarName)
 }
 
+def isTopLevelBuild = !gradle.parent
+
+def expDir = getInputParam(gradle, 'develocity.build-validation.expDir')
+def remoteBuildCacheType = getInputParam(gradle, 'develocity.build-validation.remoteBuildCacheType')
 def remoteBuildCacheUrl = getInputParam(gradle, 'develocity.build-validation.remoteBuildCacheUrl')
 
-settingsEvaluated { settings ->
+def docsRoot = 'https://docs.gradle.com/develocity/gradle-plugin'
+
+settingsEvaluated { Settings settings ->
     settings.buildCache {
         local {
             enabled = false
         }
-        remote(HttpBuildCache) {
-            enabled = true
-            push = false
-            if (remoteBuildCacheUrl) {
-                url = withPathTrailingSlash(new URI(remoteBuildCacheUrl))
+
+        if (remoteBuildCacheType) {
+            if (isInvalidRemoteBuildCacheType(remoteBuildCacheType)) {
+                failInvalidRemoteBuildCacheType(remoteBuildCacheType, expDir)
+            }
+
+            if (isTopLevelBuild && missingRequiredPlugin(settings, remoteBuildCacheType)) {
+                failMissingRequiredPlugin(remoteBuildCacheType, expDir, docsRoot)
+            }
+
+            def remoteBuildCacheImplementation = getRemoteBuildCacheImplementation(settings, remoteBuildCacheType)
+            if (remoteBuildCacheImplementation) {
+                logger.debug("Configuring remote build cache implementation for '${settings.rootProject.name}' as: ${remoteBuildCacheImplementation}")
+                remote(remoteBuildCacheImplementation)
+            }
+        }
+
+        logger.debug("Remote build cache implementation for '${settings.rootProject.name}' is: ${remote?.class?.name}")
+        if (remote) {
+            remote {
+                enabled = true
+            }
+
+            def remoteBuildCacheUri = remoteBuildCacheUrl ? withPathTrailingSlash(new URI(remoteBuildCacheUrl)) : null
+            if (remote instanceof HttpBuildCache) {
+                if (remoteBuildCacheUrl) {
+                    remote.url = remoteBuildCacheUri
+                } else if (!remote.url) {
+                    failMissingUrlForHttpBuildCache(expDir, docsRoot)
+                }
+            } else if (isBuildCacheImplementationFor(remote, 'com.gradle.develocity') || isBuildCacheImplementationFor(remote, 'com.gradle.enterprise')) {
+                if (remoteBuildCacheUri) {
+                    remote.server = toServerPart(remoteBuildCacheUri)
+                    remote.path = remoteBuildCacheUri.path
+                }
             }
+        } else if (isTopLevelBuild) {
+            failMissingRemoteBuildCacheConfiguration(expDir, docsRoot)
         }
     }
 }
 
+static boolean isInvalidRemoteBuildCacheType(String remoteBuildCacheType) {
+    return !['develocity', 'gradle-enterprise', 'http'].contains(remoteBuildCacheType)
+}
+
+static boolean missingRequiredPlugin(Settings settings, String type) {
+    return type == "develocity" && !settings.pluginManager.hasPlugin('com.gradle.develocity')
+        || type == "gradle-enterprise" && !settings.pluginManager.hasPlugin('com.gradle.enterprise')
+}
+
+static Class<? extends BuildCache> getRemoteBuildCacheImplementation(Settings settings, String type) {
+    if (type == "develocity") {
+        return settings.develocity.buildCache
+    } else if (type == "gradle-enterprise") {
+        return settings.gradleEnterprise.buildCache
+    } else if (type == "http") {
+        return HttpBuildCache
+    }
+    return null
+}
+
+static boolean isBuildCacheImplementationFor(BuildCache buildCache, String implementation) {
+    return buildCache.class.name.startsWith(implementation)
+}
+
 static URI withPathTrailingSlash(URI uri) {
-    uri.path.endsWith("/") ? uri : new URI(uri.scheme, uri.userInfo, uri.host, uri.port, uri.path + "/", uri.query, uri.fragment)
+    return uri.path.endsWith("/") ? uri : new URI(uri.scheme, uri.userInfo, uri.host, uri.port, uri.path + "/", uri.query, uri.fragment)
+}
+
+static String toServerPart(URI uri) {
+    return new URI(uri.scheme, uri.userInfo, uri.host, uri.port, null, uri.query, uri.fragment)
+}
+
+// The scripts already fail if the value of --remote-build-cache-type isn't valid.
+// This is for the sake of completeness since this init script assumes it's valid.
+static void failInvalidRemoteBuildCacheType(String remoteBuildCacheType, String expDir) {
+    def errorFile = new File(expDir, 'errors.txt')
+    def message = "Invalid value '${remoteBuildCacheType}' for remote build cache connector type. Values are 'develocity', 'gradle-enterprise', or 'http'."
+    errorFile.text = message
+    throw new IllegalStateException(message)
+}
+
+// Included builds may not have the necessary plugin applied.
+// Only fail if the top-level build is missing the required extension.
+static void failMissingRequiredPlugin(String remoteBuildCacheType, String expDir, String docsRoot) {
+    def errorFile = new File(expDir, 'errors.txt')
+    errorFile.text = "Remote build cache connector type '${remoteBuildCacheType}' requested, but the required plugin is not applied."
+    if (remoteBuildCacheType == 'develocity') {
+        throw new IllegalStateException("Remote build cache connector type 'develocity' requested,\n" +
+            "but the Develocity Gradle plugin is not applied.\n" +
+            "Either apply it directly (see $docsRoot/current/#applying_the_plugin),\n" +
+            "use --enable-develocity to enable the plugin,\n" +
+            "or use --remote-build-cache-type to choose a different remote build cache connector type\n" +
+            "when running the build validation script.")
+    } else {
+        throw new IllegalStateException("Remote build cache connector type 'gradle-enterprise' requested,\n" +
+            "but the Gradle Enterprise Gradle plugin is not applied (see $docsRoot/legacy/#applying_the_plugin).")
+    }
+}
+
+// Gradle already fails in this case, but handling it here means we can fail the experiment more
+// gracefully and provide guidance the user.
+static void failMissingUrlForHttpBuildCache(String expDir, String docsRoot) {
+    def errorFile = new File(expDir, 'errors.txt')
+    errorFile.text = 'A remote build cache URL has not been configured in the project or on the command line.'
+    throw new IllegalStateException("A remote build cache URL is not configured.\n"
+        + "Either configure it directly (see $docsRoot/current/#using_gradles_built_in_http_connector) in the project,\n"
+        + "or use --remote-build-cache-url when running the build validation script.")
+}
+
+static void failMissingRemoteBuildCacheConfiguration(String expDir, String docsRoot) {
+    def errorFile = new File(expDir, 'errors.txt')
+    errorFile.text = "Remote build cache is not configured for the project."
+    throw new IllegalStateException("Remote build cache is not configured for the project.\n" +
+        "Either configure it directly (see $docsRoot/current/#using_the_develocity_connector),\n" +
+        "or use --remote-build-cache-type when running the build validation script.")
 }

components/scripts/lib/build-scan-parse.sh

@@ -10,6 +10,8 @@ git_branches=()
 git_commit_ids=()
 requested_tasks=()
 build_outcomes=()
+remote_build_cache_types=()
+remote_build_cache_class_names=()
 remote_build_cache_urls=()
 remote_build_cache_shards=()
 
@@ -77,7 +79,7 @@ parse_build_scan_row() {
 
   local run_num
 
-  while IFS=, read -r run_num field_1 field_2 field_3 field_4 field_5 field_6 field_7 field_8 field_9 field_10 field_11 field_12 field_13 field_14 field_15 field_16 field_17 field_18 field_19 field_20; do
+  while IFS=, read -r run_num field_1 field_2 field_3 field_4 field_5 field_6 field_7 field_8 field_9 field_10 field_11 field_12 field_13 field_14 field_15 field_16 field_17 field_18 field_19 field_20 field_21 field_22; do
     debug "Build Scan $field_4 is for build $run_num"
 
     # The project_name should be overridden by Build Scan data if it is
@@ -127,21 +129,23 @@ parse_build_scan_row() {
     # The below fields are always set by Build Scan data regardless of their
     # previous value and are always safe to override.
 
-    remote_build_cache_urls[run_num]="${field_10}"
-    remote_build_cache_shards[run_num]="${field_11}"
+    remote_build_cache_types[run_num]="${field_10}"
+    remote_build_cache_class_names[run_num]="${field_11}"
+    remote_build_cache_urls[run_num]="${field_12}"
+    remote_build_cache_shards[run_num]="${field_13}"
 
     # Build caching performance metrics
-    avoided_up_to_date_num_tasks[run_num]="${field_12}"
-    avoided_up_to_date_avoidance_savings[run_num]="${field_13}"
-    avoided_from_cache_num_tasks[run_num]="${field_14}"
-    avoided_from_cache_avoidance_savings[run_num]="${field_15}"
-    executed_cacheable_num_tasks[run_num]="${field_16}"
-    executed_cacheable_duration[run_num]="${field_17}"
-    executed_not_cacheable_num_tasks[run_num]="${field_18}"
-    executed_not_cacheable_duration[run_num]="${field_19}"
+    avoided_up_to_date_num_tasks[run_num]="${field_14}"
+    avoided_up_to_date_avoidance_savings[run_num]="${field_15}"
+    avoided_from_cache_num_tasks[run_num]="${field_16}"
+    avoided_from_cache_avoidance_savings[run_num]="${field_17}"
+    executed_cacheable_num_tasks[run_num]="${field_18}"
+    executed_cacheable_duration[run_num]="${field_19}"
+    executed_not_cacheable_num_tasks[run_num]="${field_20}"
+    executed_not_cacheable_duration[run_num]="${field_21}"
 
     # Other build metrics
-    serialization_factors[run_num]="${field_20}"
+    serialization_factors[run_num]="${field_22}"
   done <<< "${build_scan_row}"
 }
 

components/scripts/lib/cli-parsers/gradle/05-cli-parser.m4

@@ -3,6 +3,7 @@
 # Created by argbash-init v2.10.0
 # ARG_OPTIONAL_SINGLE([first-build-ci],[1],[])
 # ARG_OPTIONAL_SINGLE([mapping-file],[m],[])
+# ARG_OPTIONAL_SINGLE([remote-build-cache-type],[y],[])
 # ARG_OPTIONAL_SINGLE([remote-build-cache-url],[u],[])
 # ARG_OPTIONAL_BOOLEAN([fail-if-not-fully-cacheable],[f],[])
 # ARG_HELP([This function is overridden later on.])
@@ -30,6 +31,7 @@ function print_help() {
   print_option_usage -p
   print_option_usage -t
   print_option_usage -a
+  print_option_usage "-y, --remote-build-cache-type" "Specifies the remote build cache connector type to use in the second build run locally. Values are 'develocity', 'gradle-enterprise', or 'http'."
   print_option_usage "-u, --remote-build-cache-url" "Specifies the URL for the remote build cache to access in the second build run locally."
   print_option_usage -s
   print_option_usage -e

components/scripts/lib/summary.sh

@@ -119,20 +119,34 @@ detect_warnings_from_build_scans() {
     if [ -z "${build_outcomes[i]}" ]; then
       warnings+=("Failed to fetch build scan data for the ${ORDINALS[i]} build.")
     fi
+    if [ "${remote_build_cache_types[i]}" == "unknown" ]; then
+      # "Develocity Build Validation Scripts" is specifically mentioned as to
+      # not imply Develocity itself does not work with other remote build cache
+      # implementations.
+      warnings+=("The ${ORDINALS[i]} build ran using a remote build cache implementation not officially supported by the experiment.")
+    fi
   done
 
   local value_mismatch=false
   if [[ "${project_names[0]}" != "${project_names[1]}" ]] ||
      [[ "${git_repos[0]}" != "${git_repos[1]}" ]] ||
      [[ "${git_branches[0]}" != "${git_branches[1]}" ]] ||
      [[ "${git_commit_ids[0]}" != "${git_commit_ids[1]}" ]] ||
-     [[ "${requested_tasks[0]}" != "${requested_tasks[1]}" ]]; then
+     [[ "${requested_tasks[0]}" != "${requested_tasks[1]}" ]] ||
+     [[ "${remote_build_cache_urls[0]}" != "${remote_build_cache_urls[1]}" ]] ||
+     [[ "${remote_build_cache_class_names[0]}" != "${remote_build_cache_class_names[1]}" ]]; then
     value_mismatch=true
   fi
 
   if [[ "${value_mismatch}" == "true" ]]; then
     warnings+=("Differences were detected between the two builds. This may skew the outcome of the experiment.")
   fi
+  if [ "${remote_build_cache_urls[0]}" != "${remote_build_cache_urls[1]}" ]; then
+    warnings+=("The two builds ran with different remote build cache URLs configured.")
+  fi
+  if [ "${remote_build_cache_class_names[0]}" != "${remote_build_cache_class_names[1]}" ]; then
+    warnings+=("The two builds ran using different remote build cache implementations.")
+  fi
   if [[ "${unknown_values}" == "true" ]]; then
     warnings+=("Some of the build properties could not be determined. This makes it uncertain if the experiment has run correctly.")
   fi

components/scripts/maven/04-validate-remote-build-caching-ci-local.sh

@@ -158,6 +158,9 @@ fetch_build_params_from_build_scan() {
 }
 
 read_build_params_from_build_scan_data() {
+  if [[ "${remote_build_cache_types[0]}" == "disabled" ]]; then
+    die "ERROR: Remote build cache was disabled for the first build. Enable the remote build cache in the build and restart the experiment."
+  fi
   if [ -z "${git_repo}" ]; then
     git_repo="${git_repos[0]}"
     project_name="$(basename -s .git "${git_repo}")"

release/changes.md

@@ -1,6 +1,8 @@
 > [!IMPORTANT]
 > The distributions of the Develocity Build Validation Scripts prefixed with `gradle-enterprise` are deprecated and will be removed in a future release. Migrate to the distributions prefixed with `develocity` instead.
 
+- [NEW] Support Develocity and Gradle Enterprise remote build cache connectors in the Gradle CI/Local experiment  
+- [NEW] Better handling of remote build cache misconfigurations
 - [FIX] Scripts do not wait long enough for build scans to become available when `--fail-if-not-fully-cacheable` is used
 - [FIX] Successful exit code returned when performance characteristics are unknown and `--fail-if-not-fully-cacheable` is used
 - [FIX] Gradle experiments do not disable background Build Scan publication