Skip to content

Commit 1d78248

Browse files
Merge remote-tracking branch 'origin/master' into reindex_v2
2 parents e12b58f + 807a4fb commit 1d78248

File tree

1,582 files changed

+19460
-16183
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,582 files changed

+19460
-16183
lines changed

.ci/build.sh

-12
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,5 @@
11
#!/bin/bash
22

3-
JAVA_HOME=${JAVA_HOME:-$HOME/.java/openjdk13}
4-
RUNTIME_JAVA_HOME=${RUNTIME_JAVA_HOME:-$HOME/.java/openjdk11}
5-
6-
JAVA7_HOME=$HOME/.java/java7
7-
JAVA8_HOME=$HOME/.java/java8
8-
JAVA9_HOME=$HOME/.java/java9
9-
JAVA10_HOME=$HOME/.java/java10
10-
JAVA11_HOME=$HOME/.java/java11
11-
JAVA12_HOME=$HOME/.java/openjdk12
12-
JAVA13_HOME=$HOME/.java/openjdk13
13-
143
# drop page cache and kernel slab objects on linux
154
[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
165

@@ -36,7 +25,6 @@ if pwd | grep -v -q ^/dev/shm ; then
3625
MAX_WORKERS=$(($MAX_WORKERS*2/3))
3726
fi
3827

39-
export GRADLE_OPTS="-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m"
4028
set -e
4129
./gradlew --parallel --scan \
4230
-Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \

.ci/jobs.t/defaults.yml

+15-1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,20 @@
3636
- ansicolor
3737
- timestamps
3838
- gradle-build-scan
39+
- inject:
40+
properties-file: '.ci/java-versions.properties'
41+
properties-content: |
42+
WORKSPACE=/dev/shm/$JOB_NAME
43+
JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
44+
RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
45+
JAVA7_HOME=$HOME/.java/java7
46+
JAVA8_HOME=$HOME/.java/java8
47+
JAVA9_HOME=$HOME/.java/java9
48+
JAVA10_HOME=$HOME/.java/java10
49+
JAVA11_HOME=$HOME/.java/java11
50+
JAVA12_HOME=$HOME/.java/openjdk12
51+
JAVA13_HOME=$HOME/.java/openjdk13
52+
GRADLE_OPTS=-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m
3953
properties:
4054
- github:
4155
url: https://github.com/elastic/elasticsearch/
@@ -76,4 +90,4 @@
7690
7791
7892
79-
93+
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
- job:
2+
name: elastic+elasticsearch+%BRANCH%+docker-test-ramdisk
3+
workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+docker-test
4+
display-name: "elastic / elasticsearch # %BRANCH% - docker test ramdisk (experimental)"
5+
description: "Experimental test job to shake out issues with Docker in CI\n"
6+
node: "(debian-10 || sles-12 || debian-8 || sles-15) && immutable"
7+
triggers: []
8+
builders:
9+
- shell: |
10+
#!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push
11+
.ci/build.sh -Dbwc.checkout.align=true -p test/fixtures composeUp

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

+7
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,13 @@ private static boolean shouldRunDockerTests(Project project) {
498498
return true;
499499

500500
case LINUX:
501+
// We don't attempt to check the current flavor and version of Linux unless we're
502+
// running in CI, because we don't want to stop people running the Docker tests in
503+
// their own environments if they really want to.
504+
if (BuildParams.isCi() == false) {
505+
return true;
506+
}
507+
501508
// Only some hosts in CI are configured with Docker. We attempt to work out the OS
502509
// and version, so that we know whether to expect to find Docker. We don't attempt
503510
// to probe for whether Docker is available, because that doesn't tell us whether

buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -271,10 +271,10 @@ public static String gitRevision(File rootDir) {
271271
Path refFile = gitDir.resolve(refName);
272272
if (Files.exists(refFile)) {
273273
revision = readFirstLine(refFile);
274-
} else if (Files.exists(dotGit.resolve("packed-refs"))) {
274+
} else if (Files.exists(gitDir.resolve("packed-refs"))) {
275275
// Check packed references for commit ID
276276
Pattern p = Pattern.compile("^([a-f0-9]{40}) " + refName + "$");
277-
try (Stream<String> lines = Files.lines(dotGit.resolve("packed-refs"))) {
277+
try (Stream<String> lines = Files.lines(gitDir.resolve("packed-refs"))) {
278278
revision = lines.map(p::matcher)
279279
.filter(Matcher::matches)
280280
.map(m -> m.group(1))

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

+5
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,11 @@ public void keystore(String key, FileSupplier valueSupplier) {
179179
nodes.all(each -> each.keystore(key, valueSupplier));
180180
}
181181

182+
@Override
183+
public void cliSetup(String binTool, CharSequence... args) {
184+
nodes.all(each -> each.cliSetup(binTool, args));
185+
}
186+
182187
@Override
183188
public void setting(String key, String value) {
184189
nodes.all(each -> each.setting(key, value));

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

+44-5
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
124124
private final LazyPropertyMap<String, CharSequence> settings = new LazyPropertyMap<>("Settings", this);
125125
private final LazyPropertyMap<String, CharSequence> keystoreSettings = new LazyPropertyMap<>("Keystore", this);
126126
private final LazyPropertyMap<String, File> keystoreFiles = new LazyPropertyMap<>("Keystore files", this, FileEntry::new);
127+
private final LazyPropertyList<CliEntry> cliSetup = new LazyPropertyList<>("CLI setup commands", this);
127128
private final LazyPropertyMap<String, CharSequence> systemProperties = new LazyPropertyMap<>("System properties", this);
128129
private final LazyPropertyMap<String, CharSequence> environment = new LazyPropertyMap<>("Environment", this);
129130
private final LazyPropertyList<CharSequence> jvmArgs = new LazyPropertyList<>("JVM arguments", this);
@@ -301,6 +302,11 @@ public void keystore(String key, FileSupplier valueSupplier) {
301302
keystoreFiles.put(key, valueSupplier);
302303
}
303304

305+
@Override
306+
public void cliSetup(String binTool, CharSequence... args) {
307+
cliSetup.add(new CliEntry(binTool, args));
308+
}
309+
304310
@Override
305311
public void setting(String key, String value) {
306312
settings.put(key, value);
@@ -475,6 +481,14 @@ public synchronized void start() {
475481
));
476482
}
477483

484+
if (cliSetup.isEmpty() == false) {
485+
logToProcessStdout("Running " + cliSetup.size() + " setup commands");
486+
487+
for (CliEntry entry : cliSetup) {
488+
runElasticsearchBinScript(entry.executable, entry.args);
489+
}
490+
}
491+
478492
logToProcessStdout("Starting Elasticsearch process");
479493
startElasticsearchProcess();
480494
}
@@ -623,7 +637,7 @@ public void user(Map<String, String> userSpec) {
623637
credentials.add(cred);
624638
}
625639

626-
private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
640+
private void runElasticsearchBinScriptWithInput(String input, String tool, CharSequence... args) {
627641
if (
628642
Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false &&
629643
Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false
@@ -642,12 +656,12 @@ private void runElasticsearchBinScriptWithInput(String input, String tool, Strin
642656
.supply()
643657
);
644658
spec.args(
645-
OS.<List<String>>conditional()
659+
OS.<List<CharSequence>>conditional()
646660
.onWindows(() -> {
647-
ArrayList<String> result = new ArrayList<>();
661+
ArrayList<CharSequence> result = new ArrayList<>();
648662
result.add("/c");
649663
result.add("bin\\" + tool + ".bat");
650-
for (String arg : args) {
664+
for (CharSequence arg : args) {
651665
result.add(arg);
652666
}
653667
return result;
@@ -663,7 +677,7 @@ private void runElasticsearchBinScriptWithInput(String input, String tool, Strin
663677
}
664678
}
665679

666-
private void runElasticsearchBinScript(String tool, String... args) {
680+
private void runElasticsearchBinScript(String tool, CharSequence... args) {
667681
runElasticsearchBinScriptWithInput("", tool, args);
668682
}
669683

@@ -1205,6 +1219,11 @@ public List<?> getKeystoreFiles() {
12051219
return keystoreFiles.getNormalizedCollection();
12061220
}
12071221

1222+
@Nested
1223+
public List<?> getCliSetup() {
1224+
return cliSetup.getNormalizedCollection();
1225+
}
1226+
12081227
@Nested
12091228
public List<?> getSettings() {
12101229
return settings.getNormalizedCollection();
@@ -1376,4 +1395,24 @@ public File getFile() {
13761395
return file;
13771396
}
13781397
}
1398+
1399+
private static class CliEntry {
1400+
private String executable;
1401+
private CharSequence[] args;
1402+
1403+
CliEntry(String executable, CharSequence[] args) {
1404+
this.executable = executable;
1405+
this.args = args;
1406+
}
1407+
1408+
@Input
1409+
public String getExecutable() {
1410+
return executable;
1411+
}
1412+
1413+
@Input
1414+
public CharSequence[] getArgs() {
1415+
return args;
1416+
}
1417+
}
13791418
}

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

+2
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ public interface TestClusterConfiguration {
5858

5959
void keystore(String key, FileSupplier valueSupplier);
6060

61+
void cliSetup(String binTool, CharSequence... args);
62+
6163
void setting(String key, String value);
6264

6365
void setting(String key, String value, PropertyNormalization normalization);

buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs

+6
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,9 @@ org.eclipse.jdt.core.formatter.comment.line_length=140
2222
org.eclipse.jdt.core.formatter.lineSplit=140
2323
org.eclipse.jdt.core.formatter.tabulation.char=space
2424
org.eclipse.jdt.core.formatter.tabulation.size=4
25+
26+
# Silence warnings about references to jdk internals. We intentionally use some
27+
# and have much tighter control of the warnings in the forbidden APIs gradle
28+
# task which is the definitive list of allowed references.
29+
org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
30+
org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore

buildSrc/version.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
elasticsearch = 8.0.0
2-
lucene = 8.4.0-snapshot-08b8d116f8f
2+
lucene = 8.4.0
33

44
bundled_jdk_vendor = adoptopenjdk
55
bundled_jdk = 13.0.1+9

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

+12-3
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@
2929
import org.elasticsearch.client.RequestConverters.EndpointBuilder;
3030
import org.elasticsearch.client.core.PageParams;
3131
import org.elasticsearch.client.ml.CloseJobRequest;
32-
import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
33-
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
3432
import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
3533
import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
3634
import org.elasticsearch.client.ml.DeleteCalendarRequest;
@@ -41,7 +39,9 @@
4139
import org.elasticsearch.client.ml.DeleteForecastRequest;
4240
import org.elasticsearch.client.ml.DeleteJobRequest;
4341
import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
42+
import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
4443
import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
44+
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
4545
import org.elasticsearch.client.ml.FindFileStructureRequest;
4646
import org.elasticsearch.client.ml.FlushJobRequest;
4747
import org.elasticsearch.client.ml.ForecastJobRequest;
@@ -692,7 +692,16 @@ static Request deleteDataFrameAnalytics(DeleteDataFrameAnalyticsRequest deleteRe
692692
.addPathPartAsIs("_ml", "data_frame", "analytics")
693693
.addPathPart(deleteRequest.getId())
694694
.build();
695-
return new Request(HttpDelete.METHOD_NAME, endpoint);
695+
696+
Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
697+
698+
RequestConverters.Params params = new RequestConverters.Params();
699+
if (deleteRequest.getForce() != null) {
700+
params.putParam("force", Boolean.toString(deleteRequest.getForce()));
701+
}
702+
request.addParameters(params.asMap());
703+
704+
return request;
696705
}
697706

698707
static Request evaluateDataFrame(EvaluateDataFrameRequest evaluateRequest) throws IOException {

client/rest-high-level/src/main/java/org/elasticsearch/client/SyncedFlushResponse.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ public static final class ShardFailure implements ToXContentFragment {
282282
private Map<String, Object> routing;
283283

284284
@SuppressWarnings("unchecked")
285-
static ConstructingObjectParser<ShardFailure, Void> PARSER = new ConstructingObjectParser<>(
285+
static final ConstructingObjectParser<ShardFailure, Void> PARSER = new ConstructingObjectParser<>(
286286
"shardfailure",
287287
a -> new ShardFailure((Integer)a[0], (String)a[1], (Map<String, Object>)a[2])
288288
);

client/rest-high-level/src/main/java/org/elasticsearch/client/core/MainResponse.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,7 @@ public int hashCode() {
9999
}
100100

101101
public static class Version {
102-
@SuppressWarnings("unchecked")
103-
private static ConstructingObjectParser<Version, Void> PARSER =
102+
private static final ConstructingObjectParser<Version, Void> PARSER =
104103
new ConstructingObjectParser<>(Version.class.getName(), true,
105104
args -> {
106105
return new Version((String) args[0], (String) args[1], (String) args[2], (String) args[3], (String) args[4],

client/rest-high-level/src/main/java/org/elasticsearch/client/core/MultiTermVectorsResponse.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public MultiTermVectorsResponse(List<TermVectorsResponse> responses) {
3636
this.responses = responses;
3737
}
3838

39-
private static ConstructingObjectParser<MultiTermVectorsResponse, Void> PARSER =
39+
private static final ConstructingObjectParser<MultiTermVectorsResponse, Void> PARSER =
4040
new ConstructingObjectParser<>("multi_term_vectors", true,
4141
args -> {
4242
// as the response comes from server, we are sure that args[0] will be a list of TermVectorsResponse

client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsResponse.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public TermVectorsResponse(String index, String id, long version, boolean found,
4848
this.termVectorList = termVectorList;
4949
}
5050

51-
private static ConstructingObjectParser<TermVectorsResponse, Void> PARSER = new ConstructingObjectParser<>("term_vectors", true,
51+
private static final ConstructingObjectParser<TermVectorsResponse, Void> PARSER = new ConstructingObjectParser<>("term_vectors", true,
5252
args -> {
5353
// as the response comes from server, we are sure that args[5] will be a list of TermVector
5454
@SuppressWarnings("unchecked") List<TermVector> termVectorList = (List<TermVector>) args[5];
@@ -145,7 +145,7 @@ public int hashCode() {
145145

146146
public static final class TermVector {
147147

148-
private static ConstructingObjectParser<TermVector, String> PARSER = new ConstructingObjectParser<>("term_vector", true,
148+
private static final ConstructingObjectParser<TermVector, String> PARSER = new ConstructingObjectParser<>("term_vector", true,
149149
(args, ctxFieldName) -> {
150150
// as the response comes from server, we are sure that args[1] will be a list of Term
151151
@SuppressWarnings("unchecked") List<Term> terms = (List<Term>) args[1];
@@ -218,7 +218,7 @@ public int hashCode() {
218218
// Class containing a general field statistics for the field
219219
public static final class FieldStatistics {
220220

221-
private static ConstructingObjectParser<FieldStatistics, Void> PARSER = new ConstructingObjectParser<>(
221+
private static final ConstructingObjectParser<FieldStatistics, Void> PARSER = new ConstructingObjectParser<>(
222222
"field_statistics", true,
223223
args -> {
224224
return new FieldStatistics((long) args[0], (int) args[1], (long) args[2]);
@@ -282,7 +282,7 @@ public int hashCode() {
282282

283283

284284
public static final class Term {
285-
private static ConstructingObjectParser<Term, String> PARSER = new ConstructingObjectParser<>("token", true,
285+
private static final ConstructingObjectParser<Term, String> PARSER = new ConstructingObjectParser<>("token", true,
286286
(args, ctxTerm) -> {
287287
// as the response comes from server, we are sure that args[4] will be a list of Token
288288
@SuppressWarnings("unchecked") List<Token> tokens = (List<Token>) args[4];
@@ -393,7 +393,7 @@ public int hashCode() {
393393

394394
public static final class Token {
395395

396-
private static ConstructingObjectParser<Token, Void> PARSER = new ConstructingObjectParser<>("token", true,
396+
private static final ConstructingObjectParser<Token, Void> PARSER = new ConstructingObjectParser<>("token", true,
397397
args -> {
398398
return new Token((Integer) args[0], (Integer) args[1], (Integer) args[2], (String) args[3]);
399399
});

client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java

+2-6
Original file line numberDiff line numberDiff line change
@@ -278,18 +278,14 @@ public PutIndexTemplateRequest source(Map<String, Object> templateSource) {
278278
Map<String, Object> source = templateSource;
279279
for (Map.Entry<String, Object> entry : source.entrySet()) {
280280
String name = entry.getKey();
281-
if (name.equals("template")) {
282-
if(entry.getValue() instanceof String) {
283-
patterns(Collections.singletonList((String) entry.getValue()));
284-
}
285-
} else if (name.equals("index_patterns")) {
281+
if (name.equals("index_patterns")) {
286282
if(entry.getValue() instanceof String) {
287283
patterns(Collections.singletonList((String) entry.getValue()));
288284
} else if (entry.getValue() instanceof List) {
289285
List<String> elements = ((List<?>) entry.getValue()).stream().map(Object::toString).collect(Collectors.toList());
290286
patterns(elements);
291287
} else {
292-
throw new IllegalArgumentException("Malformed [template] value, should be a string or a list of strings");
288+
throw new IllegalArgumentException("Malformed [index_patterns] value, should be a string or a list of strings");
293289
}
294290
} else if (name.equals("order")) {
295291
order(XContentMapValues.nodeIntegerValue(entry.getValue(), order()));

0 commit comments

Comments
 (0)