Skip to content

Commit 0e5d42c

Browse files
jasontedorjasontedor
committed
Merge branch 'master' into ccr
* master: Adjust BWC version on mapping version Token API supports the client_credentials grant (#33106) Build: forked compiler max memory matches jvmArgs (#33138) Introduce mapping version to index metadata (#33147) SQL: Enable aggregations to create a separate bucket for missing values (#32832) Fix grammar in contributing docs SECURITY: Fix Compile Error in ReservedRealmTests (#33166) APM server monitoring (#32515) Support only string `format` in date, root object & date range (#28117) [Rollup] Move toBuilders() methods out of rollup config objects (#32585) Fix forbiddenapis on java 11 (#33116) Apply publishing to genreate pom (#33094) Have circuit breaker succeed on unknown mem usage Do not lose default mapper on metadata updates (#33153) Fix a mappings update test (#33146) Reload Secure Settings REST specs & docs (#32990) Refactor CachingUsernamePassword realm (#32646)
2 parents 47e9e72 + 318df2a commit 0e5d42c

File tree

76 files changed

+1591
-456
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+1591
-456
lines changed

‎CONTRIBUTING.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ Contributing to the Elasticsearch codebase
9595
JDK 10 is required to build Elasticsearch. You must have a JDK 10 installation
9696
with the environment variable `JAVA_HOME` referencing the path to Java home for
9797
your JDK 10 installation. By default, tests use the same runtime as `JAVA_HOME`.
98-
However, since Elasticsearch, supports JDK 8 the build supports compiling with
98+
However, since Elasticsearch supports JDK 8, the build supports compiling with
9999
JDK 10 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
100100
pointing to the Java home of a JDK 8 installation. Note that this mechanism can
101101
be used to test against other JDKs as well, this is not only limited to JDK 8.

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -601,7 +601,6 @@ class BuildPlugin implements Plugin<Project> {
601601
} else {
602602
options.fork = true
603603
options.forkOptions.javaHome = compilerJavaHomeFile
604-
options.forkOptions.memoryMaximumSize = "512m"
605604
}
606605
if (targetCompatibilityVersion == JavaVersion.VERSION_1_8) {
607606
// compile with compact 3 profile by default

‎buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import org.gradle.api.DefaultTask;
2424
import org.gradle.api.JavaVersion;
2525
import org.gradle.api.file.FileCollection;
26+
import org.gradle.api.logging.Logger;
27+
import org.gradle.api.logging.Logging;
2628
import org.gradle.api.tasks.Input;
2729
import org.gradle.api.tasks.InputFiles;
2830
import org.gradle.api.tasks.OutputFile;
@@ -41,6 +43,7 @@
4143

4244
public class ForbiddenApisCliTask extends DefaultTask {
4345

46+
private final Logger logger = Logging.getLogger(ForbiddenApisCliTask.class);
4447
private FileCollection signaturesFiles;
4548
private List<String> signatures = new ArrayList<>();
4649
private Set<String> bundledSignatures = new LinkedHashSet<>();
@@ -49,12 +52,21 @@ public class ForbiddenApisCliTask extends DefaultTask {
4952
private FileCollection classesDirs;
5053
private Action<JavaExecSpec> execAction;
5154

55+
@Input
5256
public JavaVersion getTargetCompatibility() {
5357
return targetCompatibility;
5458
}
5559

5660
public void setTargetCompatibility(JavaVersion targetCompatibility) {
57-
this.targetCompatibility = targetCompatibility;
61+
if (targetCompatibility.compareTo(JavaVersion.VERSION_1_10) > 0) {
62+
logger.warn(
63+
"Target compatibility is set to {} but forbiddenapis only supports up to 10. Will cap at 10.",
64+
targetCompatibility
65+
);
66+
this.targetCompatibility = JavaVersion.VERSION_1_10;
67+
} else {
68+
this.targetCompatibility = targetCompatibility;
69+
}
5870
}
5971

6072
public Action<JavaExecSpec> getExecAction() {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -685,6 +685,7 @@ public void testApiNamingConventions() throws Exception {
685685
"nodes.stats",
686686
"nodes.hot_threads",
687687
"nodes.usage",
688+
"nodes.reload_secure_settings",
688689
"search_shards",
689690
};
690691
Set<String> deprecatedMethods = new HashSet<>();

‎docs/reference/cluster/nodes-reload-secure-settings.asciidoc

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
[[cluster-nodes-reload-secure-settings]]
2+
== Nodes Reload Secure Settings
3+
4+
The cluster nodes reload secure settings API is used to re-read the
5+
local node's encrypted keystore. Specifically, it will prompt the keystore
6+
decryption and reading accross the cluster. The keystore's plain content is
7+
used to reinitialize all compatible plugins. A compatible plugin can be
8+
reinitilized without restarting the node. The operation is
9+
complete when all compatible plugins have finished reinitilizing. Subsequently,
10+
the keystore is closed and any changes to it will not be reflected on the node.
11+
12+
[source,js]
13+
--------------------------------------------------
14+
POST _nodes/reload_secure_settings
15+
POST _nodes/nodeId1,nodeId2/reload_secure_settings
16+
--------------------------------------------------
17+
// CONSOLE
18+
// TEST[setup:node]
19+
// TEST[s/nodeId1,nodeId2/*/]
20+
21+
The first command reloads the keystore on each node. The seconds allows
22+
to selectively target `nodeId1` and `nodeId2`. The node selection options are
23+
detailed <<cluster-nodes,here>>.
24+
25+
Note: It is an error if secure settings are inconsistent across the cluster
26+
nodes, yet this consistency is not enforced whatsoever. Hence, reloading specific
27+
nodes is not standard. It is only justifiable when retrying failed reload operations.
28+
29+
[float]
30+
[[rest-reload-secure-settings]]
31+
==== REST Reload Secure Settings Response
32+
33+
The response contains the `nodes` object, which is a map, keyed by the
34+
node id. Each value has the node `name` and an optional `reload_exception`
35+
field. The `reload_exception` field is a serialization of the exception
36+
that was thrown during the reload process, if any.
37+
38+
[source,js]
39+
--------------------------------------------------
40+
{
41+
"_nodes": {
42+
"total": 1,
43+
"successful": 1,
44+
"failed": 0
45+
},
46+
"cluster_name": "my_cluster",
47+
"nodes": {
48+
"pQHNt5rXTTWNvUgOrdynKg": {
49+
"name": "node-0"
50+
}
51+
}
52+
}
53+
--------------------------------------------------
54+
// TESTRESPONSE[s/"my_cluster"/$body.cluster_name/]
55+
// TESTRESPONSE[s/"pQHNt5rXTTWNvUgOrdynKg"/\$node_name/]

‎docs/reference/commands/setup-passwords.asciidoc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
== elasticsearch-setup-passwords
55

66
The `elasticsearch-setup-passwords` command sets the passwords for the built-in
7-
`elastic`, `kibana`, `logstash_system`, and `beats_system` users.
7+
`elastic`, `kibana`, `logstash_system`, `beats_system`, and `apm_system` users.
88

99
[float]
1010
=== Synopsis

‎docs/reference/monitoring/exporters.asciidoc

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -105,12 +105,12 @@ route monitoring data:
105105

106106
[options="header"]
107107
|=======================
108-
| Template | Purpose
109-
| `.monitoring-alerts` | All cluster alerts for monitoring data.
110-
| `.monitoring-beats` | All Beats monitoring data.
111-
| `.monitoring-es` | All {es} monitoring data.
112-
| `.monitoring-kibana` | All {kib} monitoring data.
113-
| `.monitoring-logstash` | All Logstash monitoring data.
108+
| Template | Purpose
109+
| `.monitoring-alerts` | All cluster alerts for monitoring data.
110+
| `.monitoring-beats` | All Beats monitoring data.
111+
| `.monitoring-es` | All {es} monitoring data.
112+
| `.monitoring-kibana` | All {kib} monitoring data.
113+
| `.monitoring-logstash` | All Logstash monitoring data.
114114
|=======================
115115

116116
The templates are ordinary {es} templates that control the default settings and

‎gradle.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
org.gradle.daemon=false
22
org.gradle.jvmargs=-Xmx2g
3+
options.forkOptions.memoryMaximumSize=2g

‎rest-api-spec/src/main/resources/rest-api-spec/api/nodes.reload_secure_settings.json

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"nodes.reload_secure_settings": {
3+
"documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-reload-secure-settings.html",
4+
"methods": ["POST"],
5+
"url": {
6+
"path": "/_nodes/reload_secure_settings",
7+
"paths": ["/_nodes/reload_secure_settings", "/_nodes/{node_id}/reload_secure_settings"],
8+
"parts": {
9+
"node_id": {
10+
"type": "list",
11+
"description": "A comma-separated list of node IDs to span the reload/reinit call. Should stay empty because reloading usually involves all cluster nodes."
12+
}
13+
},
14+
"params": {
15+
"timeout": {
16+
"type" : "time",
17+
"description" : "Explicit operation timeout"
18+
}
19+
}
20+
},
21+
"body": null
22+
}
23+
}

‎rest-api-spec/src/main/resources/rest-api-spec/test/nodes.reload_secure_settings/10_basic.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
"node_reload_secure_settings test":
3+
4+
- do:
5+
nodes.reload_secure_settings: {}
6+
7+
- is_true: nodes
8+
- is_true: cluster_name

‎server/src/main/java/org/elasticsearch/cluster/ClusterState.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ public String toString() {
284284
final String TAB = " ";
285285
for (IndexMetaData indexMetaData : metaData) {
286286
sb.append(TAB).append(indexMetaData.getIndex());
287-
sb.append(": v[").append(indexMetaData.getVersion()).append("]\n");
287+
sb.append(": v[").append(indexMetaData.getVersion()).append("], mv[").append(indexMetaData.getMappingVersion()).append("]\n");
288288
for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
289289
sb.append(TAB).append(TAB).append(shard).append(": ");
290290
sb.append("p_term [").append(indexMetaData.primaryTerm(shard)).append("], ");

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.carrotsearch.hppc.cursors.ObjectCursor;
2525
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
2626

27+
import org.elasticsearch.Assertions;
2728
import org.elasticsearch.Version;
2829
import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
2930
import org.elasticsearch.action.support.ActiveShardCount;
@@ -291,6 +292,7 @@ public Iterator<Setting<Integer>> settings() {
291292

292293
public static final String KEY_IN_SYNC_ALLOCATIONS = "in_sync_allocations";
293294
static final String KEY_VERSION = "version";
295+
static final String KEY_MAPPING_VERSION = "mapping_version";
294296
static final String KEY_ROUTING_NUM_SHARDS = "routing_num_shards";
295297
static final String KEY_SETTINGS = "settings";
296298
static final String KEY_STATE = "state";
@@ -309,6 +311,9 @@ public Iterator<Setting<Integer>> settings() {
309311

310312
private final Index index;
311313
private final long version;
314+
315+
private final long mappingVersion;
316+
312317
private final long[] primaryTerms;
313318

314319
private final State state;
@@ -336,7 +341,7 @@ public Iterator<Setting<Integer>> settings() {
336341
private final ActiveShardCount waitForActiveShards;
337342
private final ImmutableOpenMap<String, RolloverInfo> rolloverInfos;
338343

339-
private IndexMetaData(Index index, long version, long[] primaryTerms, State state, int numberOfShards, int numberOfReplicas, Settings settings,
344+
private IndexMetaData(Index index, long version, long mappingVersion, long[] primaryTerms, State state, int numberOfShards, int numberOfReplicas, Settings settings,
340345
ImmutableOpenMap<String, MappingMetaData> mappings, ImmutableOpenMap<String, AliasMetaData> aliases,
341346
ImmutableOpenMap<String, Custom> customs, ImmutableOpenIntMap<Set<String>> inSyncAllocationIds,
342347
DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters,
@@ -345,6 +350,8 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat
345350

346351
this.index = index;
347352
this.version = version;
353+
assert mappingVersion >= 0 : mappingVersion;
354+
this.mappingVersion = mappingVersion;
348355
this.primaryTerms = primaryTerms;
349356
assert primaryTerms.length == numberOfShards;
350357
this.state = state;
@@ -394,6 +401,9 @@ public long getVersion() {
394401
return this.version;
395402
}
396403

404+
public long getMappingVersion() {
405+
return mappingVersion;
406+
}
397407

398408
/**
399409
* The term of the current selected primary. This is a non-negative number incremented when
@@ -644,6 +654,7 @@ private static class IndexMetaDataDiff implements Diff<IndexMetaData> {
644654
private final String index;
645655
private final int routingNumShards;
646656
private final long version;
657+
private final long mappingVersion;
647658
private final long[] primaryTerms;
648659
private final State state;
649660
private final Settings settings;
@@ -656,6 +667,7 @@ private static class IndexMetaDataDiff implements Diff<IndexMetaData> {
656667
IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) {
657668
index = after.index.getName();
658669
version = after.version;
670+
mappingVersion = after.mappingVersion;
659671
routingNumShards = after.routingNumShards;
660672
state = after.state;
661673
settings = after.settings;
@@ -672,6 +684,11 @@ private static class IndexMetaDataDiff implements Diff<IndexMetaData> {
672684
index = in.readString();
673685
routingNumShards = in.readInt();
674686
version = in.readLong();
687+
if (in.getVersion().onOrAfter(Version.V_6_5_0)) {
688+
mappingVersion = in.readVLong();
689+
} else {
690+
mappingVersion = 1;
691+
}
675692
state = State.fromId(in.readByte());
676693
settings = Settings.readSettingsFromStream(in);
677694
primaryTerms = in.readVLongArray();
@@ -707,6 +724,9 @@ public void writeTo(StreamOutput out) throws IOException {
707724
out.writeString(index);
708725
out.writeInt(routingNumShards);
709726
out.writeLong(version);
727+
if (out.getVersion().onOrAfter(Version.V_6_5_0)) {
728+
out.writeVLong(mappingVersion);
729+
}
710730
out.writeByte(state.id);
711731
Settings.writeSettingsToStream(settings, out);
712732
out.writeVLongArray(primaryTerms);
@@ -723,6 +743,7 @@ public void writeTo(StreamOutput out) throws IOException {
723743
public IndexMetaData apply(IndexMetaData part) {
724744
Builder builder = builder(index);
725745
builder.version(version);
746+
builder.mappingVersion(mappingVersion);
726747
builder.setRoutingNumShards(routingNumShards);
727748
builder.state(state);
728749
builder.settings(settings);
@@ -739,6 +760,11 @@ public IndexMetaData apply(IndexMetaData part) {
739760
public static IndexMetaData readFrom(StreamInput in) throws IOException {
740761
Builder builder = new Builder(in.readString());
741762
builder.version(in.readLong());
763+
if (in.getVersion().onOrAfter(Version.V_6_5_0)) {
764+
builder.mappingVersion(in.readVLong());
765+
} else {
766+
builder.mappingVersion(1);
767+
}
742768
builder.setRoutingNumShards(in.readInt());
743769
builder.state(State.fromId(in.readByte()));
744770
builder.settings(readSettingsFromStream(in));
@@ -778,6 +804,9 @@ public static IndexMetaData readFrom(StreamInput in) throws IOException {
778804
public void writeTo(StreamOutput out) throws IOException {
779805
out.writeString(index.getName()); // uuid will come as part of settings
780806
out.writeLong(version);
807+
if (out.getVersion().onOrAfter(Version.V_6_5_0)) {
808+
out.writeVLong(mappingVersion);
809+
}
781810
out.writeInt(routingNumShards);
782811
out.writeByte(state.id());
783812
writeSettingsToStream(settings, out);
@@ -821,6 +850,7 @@ public static class Builder {
821850
private String index;
822851
private State state = State.OPEN;
823852
private long version = 1;
853+
private long mappingVersion = 1;
824854
private long[] primaryTerms = null;
825855
private Settings settings = Settings.Builder.EMPTY_SETTINGS;
826856
private final ImmutableOpenMap.Builder<String, MappingMetaData> mappings;
@@ -843,6 +873,7 @@ public Builder(IndexMetaData indexMetaData) {
843873
this.index = indexMetaData.getIndex().getName();
844874
this.state = indexMetaData.state;
845875
this.version = indexMetaData.version;
876+
this.mappingVersion = indexMetaData.mappingVersion;
846877
this.settings = indexMetaData.getSettings();
847878
this.primaryTerms = indexMetaData.primaryTerms.clone();
848879
this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings);
@@ -1009,6 +1040,15 @@ public Builder version(long version) {
10091040
return this;
10101041
}
10111042

1043+
public long mappingVersion() {
1044+
return mappingVersion;
1045+
}
1046+
1047+
public Builder mappingVersion(final long mappingVersion) {
1048+
this.mappingVersion = mappingVersion;
1049+
return this;
1050+
}
1051+
10121052
/**
10131053
* returns the primary term for the given shard.
10141054
* See {@link IndexMetaData#primaryTerm(int)} for more information.
@@ -1136,7 +1176,7 @@ public IndexMetaData build() {
11361176

11371177
final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
11381178

1139-
return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(),
1179+
return new IndexMetaData(new Index(index, uuid), version, mappingVersion, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(),
11401180
tmpAliases.build(), customs.build(), filledInSyncAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters,
11411181
indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards, rolloverInfos.build());
11421182
}
@@ -1145,6 +1185,7 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build
11451185
builder.startObject(indexMetaData.getIndex().getName());
11461186

11471187
builder.field(KEY_VERSION, indexMetaData.getVersion());
1188+
builder.field(KEY_MAPPING_VERSION, indexMetaData.getMappingVersion());
11481189
builder.field(KEY_ROUTING_NUM_SHARDS, indexMetaData.getRoutingNumShards());
11491190
builder.field(KEY_STATE, indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));
11501191

@@ -1218,6 +1259,7 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti
12181259
if (token != XContentParser.Token.START_OBJECT) {
12191260
throw new IllegalArgumentException("expected object but got a " + token);
12201261
}
1262+
boolean mappingVersion = false;
12211263
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
12221264
if (token == XContentParser.Token.FIELD_NAME) {
12231265
currentFieldName = parser.currentName();
@@ -1316,6 +1358,9 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti
13161358
builder.state(State.fromString(parser.text()));
13171359
} else if (KEY_VERSION.equals(currentFieldName)) {
13181360
builder.version(parser.longValue());
1361+
} else if (KEY_MAPPING_VERSION.equals(currentFieldName)) {
1362+
mappingVersion = true;
1363+
builder.mappingVersion(parser.longValue());
13191364
} else if (KEY_ROUTING_NUM_SHARDS.equals(currentFieldName)) {
13201365
builder.setRoutingNumShards(parser.intValue());
13211366
} else {
@@ -1325,6 +1370,9 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti
13251370
throw new IllegalArgumentException("Unexpected token " + token);
13261371
}
13271372
}
1373+
if (Assertions.ENABLED && Version.indexCreated(builder.settings).onOrAfter(Version.V_6_5_0)) {
1374+
assert mappingVersion : "mapping version should be present for indices created on or after 6.5.0";
1375+
}
13281376
return builder.build();
13291377
}
13301378
}

0 commit comments

Comments
 (0)