Skip to content

Commit b442673

Browse files
dakronedakrone
committed
Merge remote-tracking branch 'origin/master' into slm-retention
2 parents 5faa8f9 + ed3da66 commit b442673

File tree

280 files changed

+7986
-2983
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

280 files changed

+7986
-2983
lines changed

‎.ci/build-cache.gradle

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎.ci/bwcVersions

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@ BWC_VERSION:
66
- "7.2.0"
77
- "7.2.1"
88
- "7.3.0"
9+
- "7.3.1"
910
- "7.4.0"
1011
- "8.0.0"

‎.ci/init.gradle

Lines changed: 83 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,92 @@
1-
if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFACTORY_TOKEN == null) {
2-
throw new GradleException("Using init script without configuration")
3-
} else {
4-
logger.info("Using elastic artifactory repos")
5-
settingsEvaluated { settings ->
6-
settings.pluginManagement {
1+
import com.bettercloud.vault.VaultConfig;
2+
import com.bettercloud.vault.Vault;
3+
4+
initscript {
5+
repositories {
6+
mavenCentral()
7+
}
8+
dependencies {
9+
classpath 'com.bettercloud:vault-java-driver:4.1.0'
10+
}
11+
}
12+
13+
['VAULT_ADDR', 'VAULT_ROLE_ID', 'VAULT_SECRET_ID'].each {
14+
if (System.env."$it" == null) {
15+
throw new GradleException("$it must be set!")
16+
17+
}
18+
}
19+
20+
final String vaultToken = new Vault(
21+
new VaultConfig()
22+
.address(System.env.VAULT_ADDR)
23+
.engineVersion(1)
24+
.build()
25+
)
26+
.withRetries(5, 1000)
27+
.auth()
28+
.loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
29+
.getAuthClientToken();
30+
31+
final Vault vault = new Vault(
32+
new VaultConfig()
33+
.address(System.env.VAULT_ADDR)
34+
.engineVersion(1)
35+
.token(vaultToken)
36+
.build()
37+
)
38+
.withRetries(5, 1000)
39+
40+
final Map<String,String> artifactoryCredentials = vault.logical()
41+
.read("secret/elasticsearch-ci/artifactory.elstc.co")
42+
.getData();
43+
44+
logger.info("Using elastic artifactory repos")
45+
Closure configCache = {
46+
return {
47+
name "artifactory-gradle-release"
48+
url "https://artifactory.elstc.co/artifactory/gradle-release"
49+
credentials {
50+
username artifactoryCredentials.get("username")
51+
password artifactoryCredentials.get("token")
52+
}
53+
}
54+
}
55+
settingsEvaluated { settings ->
56+
settings.pluginManagement {
57+
repositories {
58+
maven configCache()
59+
}
60+
}
61+
}
62+
projectsLoaded {
63+
allprojects {
64+
buildscript {
765
repositories {
8-
maven {
9-
name "artifactory-gradle-plugins"
10-
url "https://artifactory.elstc.co/artifactory/gradle-plugins"
11-
credentials {
12-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
13-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
14-
}
15-
}
16-
gradlePluginPortal()
66+
maven configCache()
1767
}
1868
}
69+
repositories {
70+
maven configCache()
71+
}
1972
}
20-
projectsLoaded {
21-
allprojects {
22-
buildscript {
23-
repositories {
24-
maven {
25-
name "artifactory-gradle-release"
26-
url "https://artifactory.elstc.co/artifactory/gradle-release/"
27-
credentials {
28-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
29-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
30-
}
31-
}
32-
}
33-
}
34-
repositories {
35-
maven {
36-
name "artifactory-gradle-release"
37-
url "https://artifactory.elstc.co/artifactory/gradle-release/"
38-
credentials {
39-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
40-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
41-
}
73+
}
74+
75+
if (System.env.GRADLE_BUILD_CACHE_URL != null) {
76+
final Map<String,String> buildCacheCredentials = vault.logical()
77+
.read("secret/elasticsearch-ci/gradle-build-cache")
78+
.getData();
79+
gradle.settingsEvaluated { settings ->
80+
settings.buildCache {
81+
remote(HttpBuildCache) {
82+
url = System.getenv('GRADLE_BUILD_CACHE_URL')
83+
push = Boolean.valueOf(System.getenv('GRADLE_BUILD_CACHE_PUSH') ?: 'false')
84+
credentials {
85+
username = buildCacheCredentials.get("username")
86+
password = buildCacheCredentials.get("password")
4287
}
4388
}
4489
}
4590
}
4691
}
92+

‎README.textile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,9 +155,9 @@ Complete control on the index level is allowed. As an example, in the above case
155155
<pre>
156156
curl -XPUT http://localhost:9200/another_user?pretty -H 'Content-Type: application/json' -d '
157157
{
158-
"index" : {
159-
"number_of_shards" : 2,
160-
"number_of_replicas" : 1
158+
"settings" : {
159+
"index.number_of_shards" : 2,
160+
"index.number_of_replicas" : 1
161161
}
162162
}'
163163
</pre>

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
2424
import groovy.transform.CompileDynamic
2525
import groovy.transform.CompileStatic
2626
import org.apache.commons.io.IOUtils
27-
import org.eclipse.jgit.lib.Constants
28-
import org.eclipse.jgit.lib.RepositoryBuilder
2927
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
3028
import org.elasticsearch.gradle.info.GlobalInfoExtension
3129
import org.elasticsearch.gradle.info.JavaHome
@@ -711,26 +709,12 @@ class BuildPlugin implements Plugin<Project> {
711709
// after the doFirst added by the info plugin, and we can override attributes
712710
JavaVersion compilerJavaVersion = ext.get('compilerJavaVersion') as JavaVersion
713711
jarTask.manifest.attributes(
712+
'Change': ext.get('gitRevision'),
714713
'X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch,
715714
'X-Compile-Lucene-Version': VersionProperties.lucene,
716715
'X-Compile-Elasticsearch-Snapshot': VersionProperties.isElasticsearchSnapshot(),
717-
'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),
716+
'Build-Date': ext.get('buildDate'),
718717
'Build-Java-Version': compilerJavaVersion)
719-
if (jarTask.manifest.attributes.containsKey('Change') == false) {
720-
jarTask.logger.warn('Building without git revision id.')
721-
jarTask.manifest.attributes('Change': 'Unknown')
722-
} else {
723-
/*
724-
* The info-scm plugin assumes that if GIT_COMMIT is set it was set by Jenkins to the commit hash for this build.
725-
* However, that assumption is wrong as this build could be a sub-build of another Jenkins build for which GIT_COMMIT
726-
* is the commit hash for that build. Therefore, if GIT_COMMIT is set we calculate the commit hash ourselves.
727-
*/
728-
if (System.getenv("GIT_COMMIT") != null) {
729-
final String hash = new RepositoryBuilder().findGitDir(project.buildDir).build().resolve(Constants.HEAD).name
730-
final String shortHash = hash?.substring(0, 7)
731-
jarTask.manifest.attributes('Change': shortHash)
732-
}
733-
}
734718
}
735719

736720
// add license/notice files

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,10 @@
1818
*/
1919
package org.elasticsearch.gradle.test
2020

21-
21+
import org.elasticsearch.gradle.VersionProperties
2222
import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
2323
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
24+
import org.elasticsearch.gradle.tool.ClasspathUtils
2425
import org.gradle.api.DefaultTask
2526
import org.gradle.api.Task
2627
import org.gradle.api.execution.TaskExecutionAdapter
@@ -244,7 +245,8 @@ class RestIntegTestTask extends DefaultTask {
244245
restSpec
245246
}
246247
project.dependencies {
247-
restSpec project.project(':rest-api-spec')
248+
restSpec ClasspathUtils.isElasticsearchProject() ? project.project(':rest-api-spec') :
249+
"org.elasticsearch.rest-api-spec:${VersionProperties.elasticsearch}"
248250
}
249251
Task copyRestSpec = project.tasks.findByName('copyRestSpec')
250252
if (copyRestSpec != null) {

‎buildSrc/src/main/java/org/elasticsearch/gradle/ReaperService.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,21 @@ public void registerCommand(String serviceId, String... command) {
6363
ensureReaperStarted();
6464

6565
try {
66-
Files.writeString(inputDir.resolve(serviceId + ".cmd"), String.join(" ", command));
66+
Files.writeString(getCmdFile(serviceId), String.join(" ", command));
6767
} catch (IOException e) {
6868
throw new UncheckedIOException(e);
6969
}
7070
}
7171

72+
private Path getCmdFile(String serviceId) {
73+
return inputDir.resolve(
74+
serviceId.replaceAll("[^a-zA-Z0-9]","-") + ".cmd"
75+
);
76+
}
77+
7278
public void unregister(String serviceId) {
7379
try {
74-
Files.deleteIfExists(inputDir.resolve(serviceId + ".cmd"));
80+
Files.deleteIfExists(getCmdFile(serviceId));
7581
} catch (IOException e) {
7682
throw new UncheckedIOException(e);
7783
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,4 +273,5 @@ private String runJavaAsScript(File javaHome, String script) {
273273
}
274274
return stdout.toString(UTF_8).trim();
275275
}
276+
276277
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import org.gradle.api.Project;
88
import org.gradle.api.plugins.ExtraPropertiesExtension;
99
import org.gradle.internal.jvm.Jvm;
10+
import org.gradle.process.ExecResult;
1011

1112
import java.io.BufferedReader;
1213
import java.io.ByteArrayOutputStream;
@@ -15,13 +16,17 @@
1516
import java.io.IOException;
1617
import java.io.InputStreamReader;
1718
import java.io.UncheckedIOException;
19+
import java.time.ZoneOffset;
20+
import java.time.ZonedDateTime;
1821
import java.util.ArrayList;
1922
import java.util.Arrays;
2023
import java.util.HashMap;
2124
import java.util.List;
2225
import java.util.Map;
2326
import java.util.stream.Collectors;
2427

28+
import static java.nio.charset.StandardCharsets.UTF_8;
29+
2530
public class GlobalBuildInfoPlugin implements Plugin<Project> {
2631
private static final String GLOBAL_INFO_EXTENSION_NAME = "globalInfo";
2732
private static Integer _defaultParallel = null;
@@ -87,6 +92,8 @@ public void apply(Project project) {
8792
ext.set("minimumCompilerVersion", minimumCompilerVersion);
8893
ext.set("minimumRuntimeVersion", minimumRuntimeVersion);
8994
ext.set("gradleJavaVersion", Jvm.current().getJavaVersion());
95+
ext.set("gitRevision", gitRevision(project));
96+
ext.set("buildDate", ZonedDateTime.now(ZoneOffset.UTC));
9097
});
9198
}
9299

@@ -195,4 +202,22 @@ private static int findDefaultParallel(Project project) {
195202

196203
return _defaultParallel;
197204
}
205+
206+
private String gitRevision(final Project project) {
207+
final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
208+
final ByteArrayOutputStream stderr = new ByteArrayOutputStream();
209+
final ExecResult result = project.exec(spec -> {
210+
spec.setExecutable("git");
211+
spec.setArgs(Arrays.asList("rev-parse", "HEAD"));
212+
spec.setStandardOutput(stdout);
213+
spec.setErrorOutput(stderr);
214+
spec.setIgnoreExitValue(true);
215+
});
216+
217+
if (result.getExitValue() != 0) {
218+
return "unknown";
219+
}
220+
return stdout.toString(UTF_8).trim();
221+
}
222+
198223
}

‎buildSrc/version.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
elasticsearch = 8.0.0
22
lucene = 8.2.0
33

4-
bundled_jdk = 12.0.1+12@69cfe15208a647278a19ef0990eea691
4+
bundled_jdk = 12.0.2+10@e482c34c86bd4bf8b56c0b35558996b9
55

66
# optional dependencies
77
spatial4j = 0.7

‎client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.elasticsearch.common.xcontent.XContentHelper;
5555
import org.elasticsearch.common.xcontent.XContentType;
5656
import org.elasticsearch.common.xcontent.json.JsonXContent;
57+
import org.elasticsearch.index.seqno.ReplicationTracker;
5758
import org.elasticsearch.test.rest.yaml.ObjectPath;
5859
import org.junit.Before;
5960

@@ -260,7 +261,9 @@ public void testForgetFollower() throws IOException {
260261
final Map<?, ?> shardStatsAsMap = (Map<?, ?>) shardStats.get(0);
261262
final Map<?, ?> retentionLeasesStats = (Map<?, ?>) shardStatsAsMap.get("retention_leases");
262263
final List<?> leases = (List<?>) retentionLeasesStats.get("leases");
263-
assertThat(leases, empty());
264+
for (final Object lease : leases) {
265+
assertThat(((Map<?, ?>) lease).get("source"), equalTo(ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE));
266+
}
264267
}
265268
}
266269

‎client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@
6969
import static org.hamcrest.Matchers.containsString;
7070
import static org.hamcrest.Matchers.empty;
7171
import static org.hamcrest.Matchers.equalTo;
72-
import static org.hamcrest.Matchers.greaterThan;
7372
import static org.hamcrest.Matchers.hasKey;
7473
import static org.hamcrest.Matchers.hasSize;
7574
import static org.hamcrest.Matchers.is;
@@ -387,10 +386,6 @@ public void testGetStats() throws Exception {
387386
assertNotEquals(zeroIndexerStats, stateAndStats.getIndexerStats());
388387
assertThat(stateAndStats.getTaskState(),
389388
is(oneOf(DataFrameTransformTaskState.STARTED, DataFrameTransformTaskState.STOPPED)));
390-
assertNotNull(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress());
391-
assertThat(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress().getPercentComplete(), equalTo(100.0));
392-
assertThat(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress().getTotalDocs(), greaterThan(0L));
393-
assertThat(stateAndStats.getCheckpointingInfo().getNext().getCheckpointProgress().getRemainingDocs(), equalTo(0L));
394389
assertThat(stateAndStats.getReason(), is(nullValue()));
395390
});
396391
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1045,6 +1045,8 @@ public void testForceMerge() throws IOException {
10451045
assertThat(forceMergeResponse.getSuccessfulShards(), equalTo(1));
10461046
assertThat(forceMergeResponse.getFailedShards(), equalTo(0));
10471047
assertThat(forceMergeResponse.getShardFailures(), equalTo(BroadcastResponse.EMPTY));
1048+
1049+
assertThat(forceMergeRequest.getDescription(), containsString(index));
10481050
}
10491051
{
10501052
String nonExistentIndex = "non_existent_index";
@@ -1053,6 +1055,8 @@ public void testForceMerge() throws IOException {
10531055
ElasticsearchException exception = expectThrows(ElasticsearchException.class,
10541056
() -> execute(forceMergeRequest, highLevelClient().indices()::forcemerge, highLevelClient().indices()::forcemergeAsync));
10551057
assertEquals(RestStatus.NOT_FOUND, exception.status());
1058+
1059+
assertThat(forceMergeRequest.getDescription(), containsString(nonExistentIndex));
10561060
}
10571061
}
10581062

‎client/rest-high-level/src/test/java/org/elasticsearch/client/core/MainResponseTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ protected void assertInstances(org.elasticsearch.action.main.MainResponse server
5858
assertThat(serverTestInstance.getNodeName(), equalTo(clientInstance.getNodeName()));
5959
assertThat("You Know, for Search", equalTo(clientInstance.getTagline()));
6060

61-
assertThat(serverTestInstance.getBuild().shortHash(), equalTo(clientInstance.getVersion().getBuildHash()));
61+
assertThat(serverTestInstance.getBuild().hash(), equalTo(clientInstance.getVersion().getBuildHash()));
6262
assertThat(serverTestInstance.getVersion().toString(), equalTo(clientInstance.getVersion().getNumber()));
6363
assertThat(serverTestInstance.getBuild().date(), equalTo(clientInstance.getVersion().getBuildDate()));
6464
assertThat(serverTestInstance.getBuild().flavor().displayName(), equalTo(clientInstance.getVersion().getBuildFlavor()));

0 commit comments

Comments
 (0)