Skip to content

Commit 0b1ec01

Browse files
erichaagdeverichaagdev
committed
Use --query argument when calling summary tool
Calling the summary tool with the --query argument will allow all build scan data to be fetched at once. This is possible when both build scans are published during script execution. When that's not the case, the --query argument is not used.
1 parent 59336a4 commit 0b1ec01

12 files changed

+87
-52
lines changed

‎build.gradle.kts

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ val isDevelopmentRelease = !hasProperty("finalRelease")
6161
val releaseVersion = releaseVersion()
6262
val releaseNotes = releaseNotes()
6363
val distributionVersion = distributionVersion()
64-
val buildScanSummaryVersion = "1.0.4-2024.1"
64+
val buildScanSummaryVersion = "1.0.4-2024.1" // todo update to 1.0.5-2024.1 once released
6565

6666
allprojects {
6767
version = releaseVersion.get()

‎components/scripts/gradle/01-validate-incremental-building.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ execute() {
6262
rename_project_dir "build_${project_name}" "second-build_${project_name}"
6363

6464
print_bl
65-
fetch_build_cache_metrics
65+
process_build_scan_data
6666

6767
print_bl
6868
print_summary
@@ -109,7 +109,7 @@ wizard_execute() {
109109
print_bl
110110
explain_measure_build_results
111111
print_bl
112-
fetch_build_cache_metrics
112+
process_build_scan_data
113113
print_bl
114114
explain_and_print_summary
115115
}
@@ -139,8 +139,8 @@ print_gradle_command() {
139139
info "./gradlew --no-build-cache -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false $*$(print_extra_args)"
140140
}
141141

142-
fetch_build_cache_metrics() {
143-
process_build_scan_data_online
142+
process_build_scan_data() {
143+
process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
144144
}
145145

146146
# Overrides summary.sh#print_performance_characteristics

‎components/scripts/gradle/02-validate-local-build-caching-same-location.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ execute() {
6363
rename_project_dir "build_${project_name}" "second-build_${project_name}"
6464

6565
print_bl
66-
fetch_build_cache_metrics
66+
process_build_scan_data
6767

6868
print_bl
6969
print_summary
@@ -111,7 +111,7 @@ wizard_execute() {
111111
print_bl
112112
explain_measure_build_results
113113
print_bl
114-
fetch_build_cache_metrics
114+
process_build_scan_data
115115
print_bl
116116
explain_and_print_summary
117117
}
@@ -144,8 +144,8 @@ print_gradle_command() {
144144
info "./gradlew --build-cache -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false clean ${tasks}$(print_extra_args)"
145145
}
146146

147-
fetch_build_cache_metrics() {
148-
process_build_scan_data_online
147+
process_build_scan_data() {
148+
process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
149149
}
150150

151151
print_introduction() {

‎components/scripts/gradle/03-validate-local-build-caching-different-locations.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ execute() {
6464
execute_second_build
6565

6666
print_bl
67-
fetch_build_cache_metrics
67+
process_build_scan_data
6868

6969
print_bl
7070
print_summary
@@ -115,7 +115,7 @@ wizard_execute() {
115115
print_bl
116116
explain_measure_build_results
117117
print_bl
118-
fetch_build_cache_metrics
118+
process_build_scan_data
119119
print_bl
120120
explain_and_print_summary
121121
}
@@ -149,8 +149,8 @@ print_gradle_command() {
149149
info "./gradlew --build-cache -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false clean ${tasks}$(print_extra_args)"
150150
}
151151

152-
fetch_build_cache_metrics() {
153-
process_build_scan_data_online
152+
process_build_scan_data() {
153+
process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
154154
}
155155

156156
print_introduction() {

‎components/scripts/gradle/04-validate-remote-build-caching-ci-ci.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ main() {
5252
}
5353

5454
execute() {
55-
fetch_build_scans
55+
process_build_scan_data
5656
make_experiment_dir
5757

5858
print_bl
@@ -91,7 +91,7 @@ wizard_execute() {
9191
explain_command_to_repeat_experiment_after_collecting_parameters
9292

9393
print_bl
94-
fetch_build_scans
94+
process_build_scan_data
9595
make_experiment_dir
9696

9797
print_bl
@@ -116,8 +116,8 @@ validate_required_args() {
116116
fi
117117
}
118118

119-
fetch_build_scans() {
120-
fetch_build_scans_and_build_time_metrics 'verbose_logging' "${build_scan_urls[@]}"
119+
process_build_scan_data() {
120+
process_build_scan_data_online "$LOGGING_VERBOSE" "$(query_none)"
121121
}
122122

123123
# Overrides summary.sh#print_experiment_specific_summary_info

‎components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh

+6-2
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ execute() {
6666
execute_build
6767

6868
print_bl
69-
process_build_scan_data_online
69+
process_build_scan_data
7070

7171
print_bl
7272
print_summary
@@ -130,7 +130,7 @@ wizard_execute() {
130130
print_bl
131131
explain_measure_build_results
132132
print_bl
133-
process_build_scan_data_online
133+
process_build_scan_data
134134
print_bl
135135
explain_and_print_summary
136136
}
@@ -224,6 +224,10 @@ execute_build() {
224224
invoke_gradle 1 "${args[@]}"
225225
}
226226

227+
process_build_scan_data() {
228+
process_build_scan_data_online "$LOGGING_BRIEF" "$(query_none)"
229+
}
230+
227231
# Overrides summary.sh#print_experiment_specific_summary_info
228232
print_experiment_specific_summary_info() {
229233
summary_row "Custom value mapping file:" "${mapping_file:-<none>}"

‎components/scripts/lib/build-scan-online.sh

+45-19
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,35 @@
11
#!/usr/bin/env bash
22

3-
# This is a helper function for the common pattern of reading Build Scan metadata
4-
# from the build-scans.csv file, then retrieving build metrics using the
5-
# Develocity API.
3+
readonly LOGGING_BRIEF='brief_logging'
4+
readonly LOGGING_VERBOSE='verbose_logging'
5+
6+
# Main entrypoint for processing data online using the Build Scan summary tool.
7+
# All scripts should call this function to fetch Build Scan data used in the
8+
# experiment summary.
9+
#
10+
# USAGE: fetch_build_scans_and_build_time_metrics <logging_level> <query>
11+
#
12+
# <logging_level> should be set using either LOGGING_BRIEF or LOGGING_VERBOSE
13+
# constant, e.g., "$LOGGING_BRIEF"
14+
# <query> should be set using either query_none or query_run_id function, e.g.,
15+
# "$(query_none)"
616
process_build_scan_data_online() {
17+
local logging_level="$1"
18+
local query="$2"
19+
20+
# Always call since it will only read if the metadata file exists
721
read_build_scan_metadata
8-
fetch_build_scans_and_build_time_metrics 'brief_logging' "${build_scan_urls[@]}"
22+
23+
fetch_build_scans_and_build_time_metrics "$logging_level" "$query"
24+
}
25+
26+
query_none() {
27+
# Intentionally no-op so query definitions can appear consistent
28+
true
29+
}
30+
31+
query_run_id() {
32+
echo "value:\"Experiment run id=$RUN_ID\""
933
}
1034

1135
read_build_scan_metadata() {
@@ -46,38 +70,36 @@ is_build_scan_metadata_missing() {
4670
return 0
4771
}
4872

73+
# Used by CI / Local experiments to fetch data for the first CI build.
4974
fetch_single_build_scan() {
5075
local build_scan_url="$1"
5176

5277
local build_scan_data
53-
build_scan_data="$(fetch_build_scan_data 'verbose_logging' "${build_scan_url}")"
78+
build_scan_data="$(fetch_build_scan_data 'verbose_logging' "$(query_none)" "${build_scan_url}")"
5479

5580
parse_single_build_scan "${build_scan_data}"
5681
}
5782

5883
# The value of logging_level should be either 'brief_logging' or
59-
# 'verbose_logging'
84+
# 'verbose_logging'.
85+
#
86+
# WARNING: Experiment scripts should not call this function directly and instead
87+
# use one of the 'process_X' functions.
6088
fetch_build_scans_and_build_time_metrics() {
61-
local logging_level="$1"
62-
shift
63-
local build_scan_urls=("$@")
64-
65-
if [[ "${logging_level}" == 'verbose_logging' ]]; then
66-
info "Fetching build scan data"
67-
fi
68-
6989
local build_scan_data
70-
build_scan_data="$(fetch_build_scan_data "${logging_level}" "${build_scan_urls[@]}")"
7190

91+
build_scan_data="$(fetch_build_scan_data "${@}")"
7292
parse_build_scans_and_build_time_metrics "${build_scan_data}"
7393
}
7494

75-
# Note: Callers of this function require stdout to be clean. No logging can be
76-
# done inside this function.
95+
# WARNING: Experiment scripts should not call this function directly and instead
96+
# use one of the 'process_X' functions.
97+
#
98+
# WARNING: Callers of this function require stdout to be clean. No logging can
99+
# be done inside this function.
77100
fetch_build_scan_data() {
78101
local logging_level="$1"
79-
shift
80-
local build_scan_urls=("$@")
102+
local query="$2"
81103

82104
if [[ "${debug_mode}" == "on" ]]; then
83105
args+=("--debug")
@@ -99,6 +121,10 @@ fetch_build_scan_data() {
99121
args+=("--build-scan-availability-wait-timeout" "60")
100122
fi
101123

124+
if [[ -n "${query}" ]]; then
125+
args+=("--query" "$query")
126+
fi
127+
102128
for run_num in "${!build_scan_urls[@]}"; do
103129
args+=( "${run_num},${build_scan_urls[run_num]}" )
104130
done

‎components/scripts/maven/01-validate-local-build-caching-same-location.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ execute() {
6262
rename_project_dir "build_${project_name}" "second-build_${project_name}"
6363

6464
print_bl
65-
fetch_build_cache_metrics
65+
process_build_scan_data
6666

6767
print_bl
6868
print_summary
@@ -110,7 +110,7 @@ wizard_execute() {
110110
print_bl
111111
explain_measure_build_results
112112
print_bl
113-
fetch_build_cache_metrics
113+
process_build_scan_data
114114
print_bl
115115
explain_and_print_summary
116116
}
@@ -149,8 +149,8 @@ print_maven_command() {
149149
info "./mvnw -Dscan -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false clean ${tasks}$(print_extra_args)"
150150
}
151151

152-
fetch_build_cache_metrics() {
153-
process_build_scan_data_online
152+
process_build_scan_data() {
153+
process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
154154
}
155155

156156
print_introduction() {

‎components/scripts/maven/02-validate-local-build-caching-different-locations.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ execute() {
6262
execute_second_build
6363

6464
print_bl
65-
fetch_build_cache_metrics
65+
process_build_scan_data
6666

6767
print_bl
6868
print_summary
@@ -113,7 +113,7 @@ wizard_execute() {
113113
print_bl
114114
explain_measure_build_results
115115
print_bl
116-
fetch_build_cache_metrics
116+
process_build_scan_data
117117
print_bl
118118
explain_and_print_summary
119119
}
@@ -153,8 +153,8 @@ print_maven_command() {
153153
info "./mvnw -Dscan -Dscan.tag.${EXP_SCAN_TAG} -Dscan.value.runId=${RUN_ID} -Dpts.enabled=false clean ${tasks}$(print_extra_args)"
154154
}
155155

156-
fetch_build_cache_metrics() {
157-
process_build_scan_data_online
156+
process_build_scan_data() {
157+
process_build_scan_data_online "$LOGGING_BRIEF" "$(query_run_id)"
158158
}
159159

160160
print_introduction() {

‎components/scripts/maven/03-validate-remote-build-caching-ci-ci.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ main() {
5050
}
5151

5252
execute() {
53-
fetch_build_scans
53+
process_build_scan_data
5454
make_experiment_dir
5555

5656
print_bl
@@ -89,7 +89,7 @@ wizard_execute() {
8989
explain_command_to_repeat_experiment_after_collecting_parameters
9090

9191
print_bl
92-
fetch_build_scans
92+
process_build_scan_data
9393
make_experiment_dir
9494

9595
print_bl
@@ -114,8 +114,8 @@ validate_required_args() {
114114
fi
115115
}
116116

117-
fetch_build_scans() {
118-
fetch_build_scans_and_build_time_metrics 'verbose_logging' "${build_scan_urls[@]}"
117+
process_build_scan_data() {
118+
process_build_scan_data_online "$LOGGING_VERBOSE" "$(query_none)"
119119
}
120120

121121
# Overrides summary.sh#print_experiment_specific_summary_info

‎components/scripts/maven/04-validate-remote-build-caching-ci-local.sh

+6-2
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ execute() {
6363
execute_build
6464

6565
print_bl
66-
process_build_scan_data_online
66+
process_build_scan_data
6767

6868
print_bl
6969
print_summary
@@ -127,7 +127,7 @@ wizard_execute() {
127127
print_bl
128128
explain_measure_build_results
129129
print_bl
130-
process_build_scan_data_online
130+
process_build_scan_data
131131
print_bl
132132
explain_and_print_summary
133133
}
@@ -219,6 +219,10 @@ execute_build() {
219219
invoke_maven 1 "${args[@]}"
220220
}
221221

222+
process_build_scan_data() {
223+
process_build_scan_data_online "$LOGGING_BRIEF" "$(query_none)"
224+
}
225+
222226
# Overrides summary.sh#print_experiment_specific_summary_info
223227
print_experiment_specific_summary_info() {
224228
summary_row "Custom value mapping file:" "${mapping_file:-<none>}"

‎release/changes.md

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
44
- [NEW] Support Develocity and Gradle Enterprise remote build cache connectors in the Gradle CI/Local experiment
55
- [NEW] Better handling of remote build cache misconfigurations
6+
- [NEW] Fewer calls are made to the Develocity API when possible, resulting in faster and more stable execution
67
- [FIX] Scripts do not wait long enough for build scans to become available when `--fail-if-not-fully-cacheable` is used
78
- [FIX] Successful exit code returned when performance characteristics are unknown and `--fail-if-not-fully-cacheable` is used
89
- [FIX] Gradle experiments do not disable background Build Scan publication

0 commit comments

Comments
 (0)