Skip to content

Commit fd1046c

Browse files
tlrxtlrx
committed
Merge branch 'master' into replicated-closed-indices
2 parents 46a6fba + 1d82a6d commit fd1046c

File tree

392 files changed

+4189
-2302
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

392 files changed

+4189
-2302
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/migration/DeprecationInfoResponse.java

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,16 +37,19 @@ public class DeprecationInfoResponse {
3737
private static final ParseField CLUSTER_SETTINGS = new ParseField("cluster_settings");
3838
private static final ParseField NODE_SETTINGS = new ParseField("node_settings");
3939
private static final ParseField INDEX_SETTINGS = new ParseField("index_settings");
40+
private static final ParseField ML_SETTINGS = new ParseField("ml_settings");
4041

4142
private final List<DeprecationIssue> clusterSettingsIssues;
4243
private final List<DeprecationIssue> nodeSettingsIssues;
4344
private final Map<String, List<DeprecationIssue>> indexSettingsIssues;
45+
private final List<DeprecationIssue> mlSettingsIssues;
4446

4547
public DeprecationInfoResponse(List<DeprecationIssue> clusterSettingsIssues, List<DeprecationIssue> nodeSettingsIssues,
46-
Map<String, List<DeprecationIssue>> indexSettingsIssues) {
48+
Map<String, List<DeprecationIssue>> indexSettingsIssues, List<DeprecationIssue> mlSettingsIssues) {
4749
this.clusterSettingsIssues = Objects.requireNonNull(clusterSettingsIssues, "cluster settings issues cannot be null");
4850
this.nodeSettingsIssues = Objects.requireNonNull(nodeSettingsIssues, "node settings issues cannot be null");
4951
this.indexSettingsIssues = Objects.requireNonNull(indexSettingsIssues, "index settings issues cannot be null");
52+
this.mlSettingsIssues = Objects.requireNonNull(mlSettingsIssues, "ml settings issues cannot be null");
5053
}
5154

5255
public List<DeprecationIssue> getClusterSettingsIssues() {
@@ -61,6 +64,10 @@ public Map<String, List<DeprecationIssue>> getIndexSettingsIssues() {
6164
return indexSettingsIssues;
6265
}
6366

67+
public List<DeprecationIssue> getMlSettingsIssues() {
68+
return mlSettingsIssues;
69+
}
70+
6471
private static List<DeprecationIssue> parseDeprecationIssues(XContentParser parser) throws IOException {
6572
List<DeprecationIssue> issues = new ArrayList<>();
6673
XContentParser.Token token = null;
@@ -76,6 +83,7 @@ public static DeprecationInfoResponse fromXContent(XContentParser parser) throws
7683
Map<String, List<DeprecationIssue>> indexSettings = new HashMap<>();
7784
List<DeprecationIssue> clusterSettings = new ArrayList<>();
7885
List<DeprecationIssue> nodeSettings = new ArrayList<>();
86+
List<DeprecationIssue> mlSettings = new ArrayList<>();
7987
String fieldName = null;
8088
XContentParser.Token token;
8189
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -85,6 +93,8 @@ public static DeprecationInfoResponse fromXContent(XContentParser parser) throws
8593
clusterSettings.addAll(parseDeprecationIssues(parser));
8694
} else if (NODE_SETTINGS.getPreferredName().equals(fieldName)) {
8795
nodeSettings.addAll(parseDeprecationIssues(parser));
96+
} else if (ML_SETTINGS.getPreferredName().equals(fieldName)) {
97+
mlSettings.addAll(parseDeprecationIssues(parser));
8898
} else if (INDEX_SETTINGS.getPreferredName().equals(fieldName)) {
8999
// parse out the key/value pairs
90100
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -96,7 +106,7 @@ public static DeprecationInfoResponse fromXContent(XContentParser parser) throws
96106
}
97107
}
98108
}
99-
return new DeprecationInfoResponse(clusterSettings, nodeSettings, indexSettings);
109+
return new DeprecationInfoResponse(clusterSettings, nodeSettings, indexSettings, mlSettings);
100110
}
101111

102112
@Override
@@ -106,17 +116,19 @@ public boolean equals(Object o) {
106116
DeprecationInfoResponse that = (DeprecationInfoResponse) o;
107117
return Objects.equals(clusterSettingsIssues, that.clusterSettingsIssues) &&
108118
Objects.equals(nodeSettingsIssues, that.nodeSettingsIssues) &&
119+
Objects.equals(mlSettingsIssues, that.mlSettingsIssues) &&
109120
Objects.equals(indexSettingsIssues, that.indexSettingsIssues);
110121
}
111122

112123
@Override
113124
public int hashCode() {
114-
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues);
125+
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues);
115126
}
116127

117128
@Override
118129
public String toString() {
119-
return clusterSettingsIssues.toString() + ":" + nodeSettingsIssues.toString() + ":" + indexSettingsIssues.toString();
130+
return clusterSettingsIssues.toString() + ":" + nodeSettingsIssues.toString() + ":" + indexSettingsIssues.toString() +
131+
":" + mlSettingsIssues.toString();
120132
}
121133

122134
/**

‎client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ public void testGetDeprecationInfo() throws IOException {
8585
assertThat(response.getClusterSettingsIssues().size(), equalTo(0));
8686
assertThat(response.getIndexSettingsIssues().size(), equalTo(0));
8787
assertThat(response.getNodeSettingsIssues().size(), equalTo(0));
88+
assertThat(response.getMlSettingsIssues().size(), equalTo(0));
8889
}
8990

9091
/**

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationClientDocumentationIT.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,8 @@ public void testGetDeprecationInfo() throws IOException, InterruptedException {
182182
deprecationInfoResponse.getNodeSettingsIssues(); // <2>
183183
Map<String, List<DeprecationInfoResponse.DeprecationIssue>> indexIssues =
184184
deprecationInfoResponse.getIndexSettingsIssues(); // <3>
185+
List<DeprecationInfoResponse.DeprecationIssue> mlIssues =
186+
deprecationInfoResponse.getMlSettingsIssues(); // <4>
185187
// end::get-deprecation-info-response
186188

187189
// tag::get-deprecation-info-execute-listener
@@ -195,6 +197,8 @@ public void onResponse(DeprecationInfoResponse deprecationInfoResponse1) { // <1
195197
deprecationInfoResponse.getNodeSettingsIssues();
196198
Map<String, List<DeprecationInfoResponse.DeprecationIssue>> indexIssues =
197199
deprecationInfoResponse.getIndexSettingsIssues();
200+
List<DeprecationInfoResponse.DeprecationIssue> mlIssues =
201+
deprecationInfoResponse.getMlSettingsIssues();
198202
}
199203

200204
@Override

‎client/rest-high-level/src/test/java/org/elasticsearch/client/migration/DeprecationInfoResponseTests.java

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ private void toXContent(DeprecationInfoResponse response, XContentBuilder builde
6565
}
6666
}
6767
builder.endObject();
68+
69+
builder.startArray("ml_settings");
70+
for (DeprecationInfoResponse.DeprecationIssue issue : response.getMlSettingsIssues()) {
71+
toXContent(issue, builder);
72+
}
73+
builder.endArray();
6874
}
6975
builder.endObject();
7076
}
@@ -105,12 +111,14 @@ private List<DeprecationInfoResponse.DeprecationIssue> createRandomIssues(boolea
105111
}
106112

107113
private DeprecationInfoResponse createInstance() {
108-
return new DeprecationInfoResponse(createRandomIssues(true), createRandomIssues(true), createIndexSettingsIssues());
114+
return new DeprecationInfoResponse(createRandomIssues(true), createRandomIssues(true), createIndexSettingsIssues(),
115+
createRandomIssues(true));
109116
}
110117

111118
private DeprecationInfoResponse copyInstance(DeprecationInfoResponse req) {
112119
return new DeprecationInfoResponse(new ArrayList<>(req.getClusterSettingsIssues()),
113-
new ArrayList<>(req.getNodeSettingsIssues()), new HashMap<>(req.getIndexSettingsIssues()));
120+
new ArrayList<>(req.getNodeSettingsIssues()), new HashMap<>(req.getIndexSettingsIssues()),
121+
new ArrayList<>(req.getMlSettingsIssues()));
114122
}
115123

116124
private DeprecationInfoResponse mutateInstance(DeprecationInfoResponse req) {
@@ -128,16 +136,21 @@ public void testFromXContent() throws IOException {
128136
}
129137

130138
public void testNullFailedIndices() {
131-
NullPointerException exception =
132-
expectThrows(NullPointerException.class, () -> new DeprecationInfoResponse(null, null, null));
139+
NullPointerException exception = expectThrows(NullPointerException.class,
140+
() -> new DeprecationInfoResponse(null, null, null, null));
133141
assertEquals("cluster settings issues cannot be null", exception.getMessage());
134142

135-
exception = expectThrows(NullPointerException.class, () -> new DeprecationInfoResponse(Collections.emptyList(), null, null));
143+
exception = expectThrows(NullPointerException.class,
144+
() -> new DeprecationInfoResponse(Collections.emptyList(), null, null, null));
136145
assertEquals("node settings issues cannot be null", exception.getMessage());
137146

138147
exception = expectThrows(NullPointerException.class,
139-
() -> new DeprecationInfoResponse(Collections.emptyList(), Collections.emptyList(), null));
148+
() -> new DeprecationInfoResponse(Collections.emptyList(), Collections.emptyList(), null, null));
140149
assertEquals("index settings issues cannot be null", exception.getMessage());
150+
151+
exception = expectThrows(NullPointerException.class,
152+
() -> new DeprecationInfoResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null));
153+
assertEquals("ml settings issues cannot be null", exception.getMessage());
141154
}
142155

143156
public void testEqualsAndHashCode() {

‎docs/java-rest/high-level/migration/get-deprecation-info.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,4 @@ include-tagged::{doc-tests-file}[{api}-response]
3333
<1> a List of Cluster deprecations
3434
<2> a List of Node deprecations
3535
<3> a Map of key IndexName, value List of deprecations for the index
36+
<4> a list of Machine Learning related deprecations

‎docs/reference/indices/flush.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ which returns something similar to:
104104
"sync_id" : "AVvFY-071siAOuFGEO9P", <1>
105105
"max_unsafe_auto_id_timestamp" : "-1",
106106
"min_retained_seq_no" : "0",
107-
"retention_leases" : "id:replica-0;retaining_seq_no:0;timestamp:1547235588;source:replica"
107+
"retention_leases" : "primary_term:1;version:1;id:replica-0;retaining_seq_no:0;timestamp:1547235588;source:replica"
108108
},
109109
"num_docs" : 0
110110
}
@@ -119,7 +119,7 @@ which returns something similar to:
119119
// TESTRESPONSE[s/"translog_uuid" : "hnOG3xFcTDeoI_kvvvOdNA"/"translog_uuid": $body.indices.twitter.shards.0.0.commit.user_data.translog_uuid/]
120120
// TESTRESPONSE[s/"history_uuid" : "XP7KDJGiS1a2fHYiFL5TXQ"/"history_uuid": $body.indices.twitter.shards.0.0.commit.user_data.history_uuid/]
121121
// TESTRESPONSE[s/"sync_id" : "AVvFY-071siAOuFGEO9P"/"sync_id": $body.indices.twitter.shards.0.0.commit.user_data.sync_id/]
122-
// TESTRESPONSE[s/"retention_leases" : "id:replica-0;retaining_seq_no:0;timestamp:1547235588;source:replica"/"retention_leases": $body.indices.twitter.shards.0.0.commit.user_data.retention_leases/]
122+
// TESTRESPONSE[s/"retention_leases" : "primary_term:1;version:1;id:replica-0;retaining_seq_no:0;timestamp:1547235588;source:replica"/"retention_leases": $body.indices.twitter.shards.0.0.commit.user_data.retention_leases/]
123123
<1> the `sync id` marker
124124

125125
[float]

‎docs/reference/mapping/types/geo-shape.asciidoc

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ type.
2121
|=======================================================================
2222
|Option |Description| Default
2323

24-
|`tree |deprecated[6.6, PrefixTrees no longer used] Name of the PrefixTree
24+
|`tree` |deprecated[6.6, PrefixTrees no longer used] Name of the PrefixTree
2525
implementation to be used: `geohash` for GeohashPrefixTree and `quadtree`
2626
for QuadPrefixTree. Note: This parameter is only relevant for `term` and
2727
`recursive` strategies.
@@ -127,6 +127,20 @@ the `tree` or `strategy` parameters according to the appropriate
127127
<<geo-shape-mapping-options>>. Note that these parameters are now deprecated
128128
and will be removed in a future version.
129129

130+
*IMPORTANT NOTES*
131+
132+
The following features are not yet supported with the new indexing approach:
133+
134+
* `geo_shape` query with `MultiPoint` geometry types - Elasticsearch currently prevents searching
135+
geo_shape fields with a MultiPoint geometry type to avoid a brute force linear search
136+
over each individual point. For now, if this is absolutely needed, this can be achieved
137+
using a `bool` query with each individual point.
138+
139+
* `CONTAINS` relation query - when using the new default vector indexing strategy, `geo_shape`
140+
queries with `relation` defined as `contains` are not yet supported. If this query relation
141+
is an absolute necessity, it is recommended to set `strategy` to `quadtree` and use the
142+
deprecated PrefixTree strategy indexing approach.
143+
130144
[[prefix-trees]]
131145
[float]
132146
==== Prefix trees

‎docs/reference/migration/apis/deprecation.asciidoc

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ Example response:
6868
"details" : "This index is named [logs:apache], which contains the illegal character ':'."
6969
}
7070
]
71-
}
71+
},
72+
"ml_settings" : [ ]
7273
}
7374
--------------------------------------------------
7475
// NOTCONSOLE
@@ -109,7 +110,8 @@ key. Similarly, any node-level warnings are found under `node_settings`. Since
109110
only a select subset of your nodes might incorporate these settings, it is
110111
important to read the `details` section for more information about which nodes
111112
are affected. Index warnings are sectioned off per index and can be filtered
112-
using an index-pattern in the query.
113+
using an index-pattern in the query. Machine Learning related deprecation
114+
warnings can be found under the `ml_settings` key.
113115

114116
The following example request shows only index-level deprecations of all
115117
`logstash-*` indices:

‎docs/reference/migration/migrate_7_0/settings.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,11 +138,11 @@ used.
138138

139139
TLS version 1.0 is now disabled by default as it suffers from
140140
https://www.owasp.org/index.php/Transport_Layer_Protection_Cheat_Sheet#Rule_-_Only_Support_Strong_Protocols[known security issues].
141-
The default protocols are now TLSv1.2 and TLSv1.1.
141+
The default protocols are now TLSv1.3 (if supported), TLSv1.2 and TLSv1.1.
142142
You can enable TLS v1.0 by configuring the relevant `ssl.supported_protocols` setting to include `"TLSv1"`, for example:
143143
[source,yaml]
144144
--------------------------------------------------
145-
xpack.security.http.ssl.supported_protocols: [ "TLSv1.2", "TLSv1.1", "TLSv1" ]
145+
xpack.security.http.ssl.supported_protocols: [ "TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1" ]
146146
--------------------------------------------------
147147

148148
[float]

‎docs/reference/ml/apis/find-file-structure.asciidoc

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -606,11 +606,11 @@ If the request does not encounter errors, you receive the following result:
606606
},
607607
"tpep_dropoff_datetime" : {
608608
"type" : "date",
609-
"format" : "8yyyy-MM-dd HH:mm:ss"
609+
"format" : "yyyy-MM-dd HH:mm:ss"
610610
},
611611
"tpep_pickup_datetime" : {
612612
"type" : "date",
613-
"format" : "8yyyy-MM-dd HH:mm:ss"
613+
"format" : "yyyy-MM-dd HH:mm:ss"
614614
},
615615
"trip_distance" : {
616616
"type" : "double"
@@ -624,7 +624,7 @@ If the request does not encounter errors, you receive the following result:
624624
"field" : "tpep_pickup_datetime",
625625
"timezone" : "{{ beat.timezone }}",
626626
"formats" : [
627-
"8yyyy-MM-dd HH:mm:ss"
627+
"yyyy-MM-dd HH:mm:ss"
628628
]
629629
}
630630
}
@@ -1398,7 +1398,7 @@ this:
13981398
"field" : "timestamp",
13991399
"timezone" : "{{ beat.timezone }}",
14001400
"formats" : [
1401-
"8yyyy-MM-dd'T'HH:mm:ss,SSS"
1401+
"yyyy-MM-dd'T'HH:mm:ss,SSS"
14021402
]
14031403
}
14041404
},
@@ -1558,7 +1558,7 @@ this:
15581558
"field" : "timestamp",
15591559
"timezone" : "{{ beat.timezone }}",
15601560
"formats" : [
1561-
"8yyyy-MM-dd'T'HH:mm:ss,SSS"
1561+
"yyyy-MM-dd'T'HH:mm:ss,SSS"
15621562
]
15631563
}
15641564
},

‎docs/reference/settings/security-settings.asciidoc

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -480,7 +480,8 @@ and `full`. Defaults to `full`.
480480
See <<ssl-tls-settings,`ssl.verification_mode`>> for an explanation of these values.
481481

482482
`ssl.supported_protocols`::
483-
Supported protocols for TLS/SSL (with versions). Defaults to `TLSv1.2,TLSv1.1`.
483+
Supported protocols for TLS/SSL (with versions). Defaults to `TLSv1.3,TLSv1.2,TLSv1.1` if
484+
the JVM supports TLSv1.3, otherwise `TLSv1.2,TLSv1.1`.
484485

485486
`ssl.cipher_suites`:: Specifies the cipher suites that should be supported when
486487
communicating with the LDAP server.
@@ -724,7 +725,8 @@ and `full`. Defaults to `full`.
724725
See <<ssl-tls-settings,`ssl.verification_mode`>> for an explanation of these values.
725726

726727
`ssl.supported_protocols`::
727-
Supported protocols for TLS/SSL (with versions). Defaults to `TLSv1.2, TLSv1.1`.
728+
Supported protocols for TLS/SSL (with versions). Defaults to `TLSv1.3,TLSv1.2,TLSv1.1` if
729+
the JVM supports TLSv1.3, otherwise `TLSv1.2,TLSv1.1`.
728730

729731
`ssl.cipher_suites`:: Specifies the cipher suites that should be supported when
730732
communicating with the Active Directory server.
@@ -1132,7 +1134,8 @@ Defaults to `full`.
11321134
See <<ssl-tls-settings,`ssl.verification_mode`>> for a more detailed explanation of these values.
11331135

11341136
`ssl.supported_protocols`::
1135-
Specifies the supported protocols for TLS/SSL.
1137+
Specifies the supported protocols for TLS/SSL. Defaults to `TLSv1.3,TLSv1.2,TLSv1.1` if
1138+
the JVM supports TLSv1.3, otherwise `TLSv1.2,TLSv1.1`.
11361139

11371140
`ssl.cipher_suites`::
11381141
Specifies the
@@ -1206,7 +1209,8 @@ settings. For more information, see
12061209

12071210
`ssl.supported_protocols`::
12081211
Supported protocols with versions. Valid protocols: `SSLv2Hello`,
1209-
`SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`. Defaults to `TLSv1.2`, `TLSv1.1`.
1212+
`SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`, `TLSv1.3`. Defaults to `TLSv1.3,TLSv1.2,TLSv1.1` if
1213+
the JVM supports TLSv1.3, otherwise `TLSv1.2,TLSv1.1`.
12101214
+
12111215
--
12121216
NOTE: If `xpack.security.fips_mode.enabled` is `true`, you cannot use `SSLv2Hello`

‎docs/reference/settings/ssl-settings.asciidoc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ endif::server[]
1111

1212
+{ssl-prefix}.ssl.supported_protocols+::
1313
Supported protocols with versions. Valid protocols: `SSLv2Hello`,
14-
`SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`. Defaults to `TLSv1.2`, `TLSv1.1`.
14+
`SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`, `TLSv1.3`. Defaults to `TLSv1.3,TLSv1.2,TLSv1.1` if
15+
the JVM supports TLSv1.3, otherwise `TLSv1.2,TLSv1.1`.
1516

1617

1718
ifdef::server[]

‎docs/reference/sql/limitations.asciidoc

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,18 @@ a field is an array (has multiple values) or not, so without reading all the dat
6767
=== Sorting by aggregation
6868

6969
When doing aggregations (`GROUP BY`) {es-sql} relies on {es}'s `composite` aggregation for its support for paginating results.
70-
But this type of aggregation does come with a limitation: sorting can only be applied on the key used for the aggregation's buckets. This
71-
means that queries like `SELECT * FROM test GROUP BY age ORDER BY COUNT(*)` are not possible.
70+
However this type of aggregation does come with a limitation: sorting can only be applied on the key used for the aggregation's buckets.
71+
{es-sql} overcomes this limitation by doing client-side sorting however as a safety measure, allows only up to *512* rows.
72+
73+
It is recommended to use `LIMIT` for queries that use sorting by aggregation, essentially indicating the top N results that are desired:
74+
75+
[source, sql]
76+
--------------------------------------------------
77+
SELECT * FROM test GROUP BY age ORDER BY COUNT(*) LIMIT 100;
78+
--------------------------------------------------
79+
80+
It is possible to run the same queries without a `LIMIT` however in that case if the maximum size (*512*) is passed, an exception will be
81+
returned as {es-sql} is unable to track (and sort) all the results returned.
7282

7383
[float]
7484
=== Using aggregation functions on top of scalar functions

0 commit comments

Comments
 (0)