Skip to content

Commit 1cb71ec

Browse files
jasontedorjasontedor
committed
Merge remote-tracking branch 'elastic/master' into sync-retention-leases
* elastic/master: Optimize warning header de-duplication (elastic#37725) Bubble exceptions up in ClusterApplierService (elastic#37729) SQL: Improve handling of invalid args for PERCENTILE/PERCENTILE_RANK (elastic#37803) Remove unused ThreadBarrier class (elastic#37666) Add built-in user and role for code plugin (elastic#37030) Consolidate testclusters tests into a single project (elastic#37362) Fix docs for MappingUpdatedAction SQL: Introduce SQL DATE data type (elastic#37693) disabling bwc test while backporting elastic#37639 Mute ClusterDisruptionIT testAckedIndexing Set acking timeout to 0 on dynamic mapping update (elastic#31140) Remove index audit output type (elastic#37707) Mute FollowerFailOverIT testReadRequestsReturnsLatestMappingVersion [ML] Increase close job timeout and lower the max number (elastic#37770) Remove Custom Listeners from SnapshotsService (elastic#37629) Use m_m_nodes from Zen1 master for Zen2 bootstrap (elastic#37701) Fix index filtering in follow info api. (elastic#37752) Use project dependency instead of substitutions for distributions (elastic#37730) Update authenticate to allow unknown fields (elastic#37713) Deprecate HLRC EmptyResponse used by security (elastic#37540)
2 parents 0496d83 + 7517e3a commit 1cb71ec

File tree

129 files changed

+1996
-4993
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

129 files changed

+1996
-4993
lines changed

‎build.gradle

Lines changed: 2 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -159,8 +159,8 @@ task verifyVersions {
159159
* the enabled state of every bwc task. It should be set back to true
160160
* after the backport of the backcompat code is complete.
161161
*/
162-
final boolean bwc_tests_enabled = true
163-
final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
162+
final boolean bwc_tests_enabled = false
163+
final String bwc_tests_disabled_issue = "backporting https://github.com/elastic/elasticsearch/pull/37639" /* place a PR link here when committing bwc changes */
164164
if (bwc_tests_enabled == false) {
165165
if (bwc_tests_disabled_issue.isEmpty()) {
166166
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
@@ -221,14 +221,6 @@ allprojects {
221221
"org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
222222
"org.elasticsearch.test:framework:${version}": ':test:framework',
223223
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
224-
"downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
225-
"downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
226-
"downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
227-
"downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
228-
"downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
229-
"downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
230-
"downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
231-
"downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
232224
"org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
233225
"org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
234226
// for transport client
@@ -240,19 +232,6 @@ allprojects {
240232
"org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
241233
"org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
242234
]
243-
// substitute unreleased versions with projects that check out and build locally
244-
bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
245-
Version unreleased = unreleasedVersion.version
246-
String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
247-
ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
248-
ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
249-
ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
250-
if (unreleased.onOrAfter('6.3.0')) {
251-
ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
252-
ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
253-
ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
254-
}
255-
}
256235

257236
/*
258237
* Gradle only resolve project substitutions during dependency resolution but

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
2323
import org.elasticsearch.gradle.BuildPlugin
2424
import org.elasticsearch.gradle.LoggedExec
2525
import org.elasticsearch.gradle.Version
26+
import org.elasticsearch.gradle.VersionCollection
2627
import org.elasticsearch.gradle.VersionProperties
2728
import org.elasticsearch.gradle.plugin.PluginBuildPlugin
2829
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
@@ -171,6 +172,12 @@ class ClusterFormationTasks {
171172

172173
/** Adds a dependency on the given distribution */
173174
static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
175+
if (distro.equals("integ-test-zip")) {
176+
// short circuit integ test so it doesn't complicate the rest of the distribution setup below
177+
project.dependencies.add(configuration.name,
178+
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip")
179+
return
180+
}
174181
// TEMP HACK
175182
// The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
176183
if (distro.equals('oss-zip')) {
@@ -180,22 +187,31 @@ class ClusterFormationTasks {
180187
distro = 'default'
181188
}
182189
// END TEMP HACK
183-
if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
190+
if (['oss', 'default'].contains(distro) == false) {
184191
throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
185192
}
186193
Version version = Version.fromString(elasticsearchVersion)
187-
if (version.before('6.3.0') && distro.startsWith('oss-')) {
188-
distro = distro.substring('oss-'.length())
189-
}
190-
String group = "downloads.zip"
191-
if (distro.equals("integ-test-zip")) {
192-
group = "org.elasticsearch.distribution.integ-test-zip"
193-
}
194+
String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo
194195
String artifactName = 'elasticsearch'
195196
if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
196197
artifactName += '-oss'
197198
}
198-
project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
199+
String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip'
200+
Object dependency
201+
boolean internalBuild = project.hasProperty('bwcVersions')
202+
VersionCollection.UnreleasedVersionInfo unreleasedInfo = null
203+
if (project.hasProperty('bwcVersions')) {
204+
// NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
205+
unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
206+
}
207+
if (unreleasedInfo != null) {
208+
dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
209+
} else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
210+
dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
211+
} else {
212+
dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip"
213+
}
214+
project.dependencies.add(configuration.name, dependency)
199215
}
200216

201217
/** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
44
import org.elasticsearch.gradle.FileContentsTask
55
import org.elasticsearch.gradle.LoggedExec
66
import org.elasticsearch.gradle.Version
7+
import org.elasticsearch.gradle.VersionCollection
78
import org.gradle.api.*
89
import org.gradle.api.artifacts.dsl.RepositoryHandler
910
import org.gradle.api.execution.TaskExecutionAdapter
@@ -184,22 +185,38 @@ class VagrantTestPlugin implements Plugin<Project> {
184185
upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
185186
}
186187

188+
List<Object> dependencies = new ArrayList<>()
187189
DISTRIBUTIONS.each {
188190
// Adds a dependency for the current version
189-
project.dependencies.add(PACKAGING_CONFIGURATION,
190-
project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
191+
dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
191192
}
192193

193-
UPGRADE_FROM_ARCHIVES.each {
194-
// The version of elasticsearch that we upgrade *from*
195-
project.dependencies.add(PACKAGING_CONFIGURATION,
196-
"downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
197-
if (upgradeFromVersion.onOrAfter('6.3.0')) {
198-
project.dependencies.add(PACKAGING_CONFIGURATION,
199-
"downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
194+
// The version of elasticsearch that we upgrade *from*
195+
VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion)
196+
if (unreleasedInfo != null) {
197+
// handle snapshots pointing to bwc build
198+
UPGRADE_FROM_ARCHIVES.each {
199+
dependencies.add(project.dependencies.project(
200+
path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it))
201+
if (upgradeFromVersion.onOrAfter('6.3.0')) {
202+
dependencies.add(project.dependencies.project(
203+
path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}"))
204+
}
205+
}
206+
} else {
207+
UPGRADE_FROM_ARCHIVES.each {
208+
// The version of elasticsearch that we upgrade *from*
209+
dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
210+
if (upgradeFromVersion.onOrAfter('6.3.0')) {
211+
dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
212+
}
200213
}
201214
}
202215

216+
for (Object dependency : dependencies) {
217+
project.dependencies.add(PACKAGING_CONFIGURATION, dependency)
218+
}
219+
203220
project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
204221
}
205222

‎buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020

2121
import java.util.ArrayList;
2222
import java.util.Collection;
23+
import java.util.Collections;
24+
import java.util.HashMap;
2325
import java.util.HashSet;
2426
import java.util.List;
2527
import java.util.Map;
@@ -86,6 +88,7 @@ public class VersionCollection {
8688

8789
private final Version currentVersion;
8890
private final Map<Integer, List<Version>> groupByMajor;
91+
private final Map<Version, UnreleasedVersionInfo> unreleased;
8992

9093
public class UnreleasedVersionInfo {
9194
public final Version version;
@@ -129,6 +132,16 @@ protected VersionCollection(List<String> versionLines, Version currentVersionPro
129132
assertCurrentVersionMatchesParsed(currentVersionProperty);
130133

131134
assertNoOlderThanTwoMajors();
135+
136+
Map<Version, UnreleasedVersionInfo> unreleased = new HashMap<>();
137+
for (Version unreleasedVersion : getUnreleased()) {
138+
if (unreleasedVersion.equals(currentVersion)) {
139+
continue;
140+
}
141+
unreleased.put(unreleasedVersion,
142+
new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion)));
143+
}
144+
this.unreleased = Collections.unmodifiableMap(unreleased);
132145
}
133146

134147
private void assertNoOlderThanTwoMajors() {
@@ -150,6 +163,13 @@ private void assertCurrentVersionMatchesParsed(Version currentVersionProperty) {
150163
}
151164
}
152165

166+
/**
167+
* Returns info about the unreleased version, or {@code null} if the version is released.
168+
*/
169+
public UnreleasedVersionInfo unreleasedInfo(Version version) {
170+
return unreleased.get(version);
171+
}
172+
153173
public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
154174
getUnreleased().stream()
155175
.filter(version -> version.equals(currentVersion) == false)

‎buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java

Lines changed: 42 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -37,34 +37,34 @@ public void testListClusters() {
3737
}
3838

3939
public void testUseClusterByOne() {
40-
BuildResult result = getTestClustersRunner("user1").build();
40+
BuildResult result = getTestClustersRunner(":user1").build();
4141
assertTaskSuccessful(result, ":user1");
4242
assertStartedAndStoppedOnce(result);
4343
}
4444

4545
public void testUseClusterByOneWithDryRun() {
46-
BuildResult result = getTestClustersRunner("--dry-run", "user1").build();
46+
BuildResult result = getTestClustersRunner("--dry-run", ":user1").build();
4747
assertNull(result.task(":user1"));
4848
assertNotStarted(result);
4949
}
5050

5151
public void testUseClusterByTwo() {
52-
BuildResult result = getTestClustersRunner("user1", "user2").build();
52+
BuildResult result = getTestClustersRunner(":user1", ":user2").build();
5353
assertTaskSuccessful(result, ":user1", ":user2");
5454
assertStartedAndStoppedOnce(result);
5555
}
5656

5757
public void testUseClusterByUpToDateTask() {
5858
// Run it once, ignoring the result and again to make sure it's considered up to date.
5959
// Gradle randomly considers tasks without inputs and outputs as as up-to-date or success on the first run
60-
getTestClustersRunner("upToDate1", "upToDate2").build();
61-
BuildResult result = getTestClustersRunner("upToDate1", "upToDate2").build();
60+
getTestClustersRunner(":upToDate1", ":upToDate2").build();
61+
BuildResult result = getTestClustersRunner(":upToDate1", ":upToDate2").build();
6262
assertTaskUpToDate(result, ":upToDate1", ":upToDate2");
6363
assertNotStarted(result);
6464
}
6565

6666
public void testUseClusterBySkippedTask() {
67-
BuildResult result = getTestClustersRunner("skipped1", "skipped2").build();
67+
BuildResult result = getTestClustersRunner(":skipped1", ":skipped2").build();
6868
assertTaskSkipped(result, ":skipped1", ":skipped2");
6969
assertNotStarted(result);
7070
}
@@ -82,17 +82,44 @@ public void testUseClusterBySkippedAndWorkingTask() {
8282
}
8383

8484
public void testMultiProject() {
85-
BuildResult result = GradleRunner.create()
86-
.withProjectDir(getProjectDir("testclusters_multiproject"))
87-
.withArguments("user1", "user2", "-s", "-i", "--parallel", "-Dlocal.repo.path=" + getLocalTestRepoPath())
88-
.withPluginClasspath()
89-
.build();
90-
assertTaskSuccessful(result, ":user1", ":user2");
85+
BuildResult result = getTestClustersRunner(
86+
"user1", "user2", "-s", "-i", "--parallel", "-Dlocal.repo.path=" + getLocalTestRepoPath()
87+
).build();
88+
89+
assertTaskSuccessful(
90+
result,
91+
":user1", ":user2", ":alpha:user1", ":alpha:user2", ":bravo:user1", ":bravo:user2"
92+
);
93+
assertStartedAndStoppedOnce(result);
94+
assertOutputOnlyOnce(
95+
result.getOutput(),
96+
"Starting `node{:alpha:myTestCluster}`",
97+
"Stopping `node{::myTestCluster}`"
98+
);
99+
assertOutputOnlyOnce(
100+
result.getOutput(),
101+
"Starting `node{::myTestCluster}`",
102+
"Stopping `node{:bravo:myTestCluster}`"
103+
);
104+
}
105+
106+
public void testIncremental() {
107+
BuildResult result = getTestClustersRunner("clean", ":user1").build();
108+
assertTaskSuccessful(result, ":user1");
109+
assertStartedAndStoppedOnce(result);
110+
111+
result = getTestClustersRunner(":user1").build();
112+
assertTaskSuccessful(result, ":user1");
113+
assertStartedAndStoppedOnce(result);
114+
115+
result = getTestClustersRunner("clean", ":user1").build();
116+
assertTaskSuccessful(result, ":user1");
117+
assertStartedAndStoppedOnce(result);
91118
assertStartedAndStoppedOnce(result);
92119
}
93120

94121
public void testUseClusterByFailingOne() {
95-
BuildResult result = getTestClustersRunner("itAlwaysFails").buildAndFail();
122+
BuildResult result = getTestClustersRunner(":itAlwaysFails").buildAndFail();
96123
assertTaskFailed(result, ":itAlwaysFails");
97124
assertStartedAndStoppedOnce(result);
98125
assertOutputContains(
@@ -103,7 +130,7 @@ public void testUseClusterByFailingOne() {
103130
}
104131

105132
public void testUseClusterByFailingDependency() {
106-
BuildResult result = getTestClustersRunner("dependsOnFailed").buildAndFail();
133+
BuildResult result = getTestClustersRunner(":dependsOnFailed").buildAndFail();
107134
assertTaskFailed(result, ":itAlwaysFails");
108135
assertNull(result.task(":dependsOnFailed"));
109136
assertStartedAndStoppedOnce(result);
@@ -115,7 +142,7 @@ public void testUseClusterByFailingDependency() {
115142
}
116143

117144
public void testConfigurationLocked() {
118-
BuildResult result = getTestClustersRunner("illegalConfigAlter").buildAndFail();
145+
BuildResult result = getTestClustersRunner(":illegalConfigAlter").buildAndFail();
119146
assertTaskFailed(result, ":illegalConfigAlter");
120147
assertOutputContains(
121148
result.getOutput(),

‎buildSrc/src/testKit/testclusters/build.gradle

Lines changed: 34 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,45 @@
11
plugins {
22
id 'elasticsearch.testclusters'
3+
id 'base'
34
}
45

5-
testClusters {
6-
myTestCluster {
7-
distribution = 'ZIP'
8-
version = System.getProperty("test.version_under_test")
6+
allprojects { all ->
7+
repositories {
8+
maven {
9+
url System.getProperty("local.repo.path")
10+
}
11+
String luceneSnapshotRevision = System.getProperty("test.lucene-snapshot-revision")
12+
if (luceneSnapshotRevision != null) {
13+
maven {
14+
url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + luceneSnapshotRevision
15+
}
16+
}
17+
jcenter()
918
}
10-
}
1119

12-
repositories {
13-
maven {
14-
url System.getProperty("local.repo.path")
15-
}
16-
}
20+
if (project == rootProject || project.name == "alpha" || project.name == "bravo") {
21+
apply plugin: 'elasticsearch.testclusters'
1722

18-
task user1 {
19-
useCluster testClusters.myTestCluster
20-
doLast {
21-
println "$path: Cluster running @ ${testClusters.myTestCluster.httpSocketURI}"
22-
}
23-
}
23+
all.testClusters {
24+
myTestCluster {
25+
distribution = 'ZIP'
26+
version = System.getProperty("test.version_under_test")
27+
javaHome = file(System.getProperty('java.home'))
28+
}
29+
}
2430

25-
task user2 {
26-
useCluster testClusters.myTestCluster
27-
doLast {
28-
println "$path: Cluster running @ ${testClusters.myTestCluster.httpSocketURI}"
31+
task user1 {
32+
useCluster testClusters.myTestCluster
33+
doFirst {
34+
println "$path: Cluster running @ ${testClusters.myTestCluster.httpSocketURI}"
35+
}
36+
}
37+
task user2 {
38+
useCluster testClusters.myTestCluster
39+
doFirst {
40+
println "$path: Cluster running @ ${testClusters.myTestCluster.httpSocketURI}"
41+
}
42+
}
2943
}
3044
}
3145

‎buildSrc/src/testKit/testclusters/settings.gradle

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
include 'dummyPlugin'
2+
include ':alpha'
3+
include ':bravo'
4+
include ':charlie'

0 commit comments

Comments
 (0)