Skip to content

Commit 99a2acf

Browse files
committed
Merge remote-tracking branch 'elastic/master' into pr/38140
* elastic/master: (54 commits) Introduce retention leases versioning (elastic#37951) Correctly disable tests for FIPS JVMs (elastic#38214) AwaitsFix testAbortedSnapshotDuringInitDoesNotStart (elastic#38227) Preserve ILM operation mode when creating new lifecycles (elastic#38134) Enable trace log in FollowerFailOverIT (elastic#38148) SnapshotShardsService Simplifications (elastic#38025) Default include_type_name to false in the yml test harness. (elastic#38058) Disable bwc preparing to backport of#37977, elastic#37857 and elastic#37872 (elastic#38126) Adding ml_settings entry to HLRC and Docs for deprecation_info (elastic#38118) Replace awaitBusy with assertBusy in atLeastDocsIndexed (elastic#38190) Adjust SearchRequest version checks (elastic#38181) AwaitsFix testClientSucceedsWithVerificationDisabled (elastic#38213) Zen2ify RareClusterStateIT (elastic#38184) ML: Fix error race condition on stop _all datafeeds and close _all jobs (elastic#38113) AwaitsFix PUT mapping with _doc on an index that has types (elastic#38204) Allow built-in monitoring_user role to call GET _xpack API (elastic#38060) Update geo_shape docs to include unsupported features (elastic#38138) [ML] Remove "8" prefixes from file structure finder timestamp formats (elastic#38016) Disable bwc tests while backporting elastic#38104 (elastic#38182) Enable TLSv1.3 by default for JDKs with support (elastic#38103) ...
2 parents 906da63 + f181e17 commit 99a2acf

File tree

384 files changed

+5998
-3163
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

384 files changed

+5998
-3163
lines changed

build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -159,8 +159,8 @@ task verifyVersions {
159159
* the enabled state of every bwc task. It should be set back to true
160160
* after the backport of the backcompat code is complete.
161161
*/
162-
final boolean bwc_tests_enabled = true
163-
final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
162+
final boolean bwc_tests_enabled = false
163+
final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37951" /* place a PR link here when committing bwc changes */
164164
if (bwc_tests_enabled == false) {
165165
if (bwc_tests_disabled_issue.isEmpty()) {
166166
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

+3
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@
9595
import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
9696
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
9797
import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
98+
import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid;
99+
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
98100
import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
99101
import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
100102
import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
@@ -1760,6 +1762,7 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
17601762
map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c));
17611763
map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c));
17621764
map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c));
1765+
map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c));
17631766
map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c));
17641767
map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c));
17651768
map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));

client/rest-high-level/src/main/java/org/elasticsearch/client/migration/DeprecationInfoResponse.java

+16-4
Original file line numberDiff line numberDiff line change
@@ -37,16 +37,19 @@ public class DeprecationInfoResponse {
3737
private static final ParseField CLUSTER_SETTINGS = new ParseField("cluster_settings");
3838
private static final ParseField NODE_SETTINGS = new ParseField("node_settings");
3939
private static final ParseField INDEX_SETTINGS = new ParseField("index_settings");
40+
private static final ParseField ML_SETTINGS = new ParseField("ml_settings");
4041

4142
private final List<DeprecationIssue> clusterSettingsIssues;
4243
private final List<DeprecationIssue> nodeSettingsIssues;
4344
private final Map<String, List<DeprecationIssue>> indexSettingsIssues;
45+
private final List<DeprecationIssue> mlSettingsIssues;
4446

4547
public DeprecationInfoResponse(List<DeprecationIssue> clusterSettingsIssues, List<DeprecationIssue> nodeSettingsIssues,
46-
Map<String, List<DeprecationIssue>> indexSettingsIssues) {
48+
Map<String, List<DeprecationIssue>> indexSettingsIssues, List<DeprecationIssue> mlSettingsIssues) {
4749
this.clusterSettingsIssues = Objects.requireNonNull(clusterSettingsIssues, "cluster settings issues cannot be null");
4850
this.nodeSettingsIssues = Objects.requireNonNull(nodeSettingsIssues, "node settings issues cannot be null");
4951
this.indexSettingsIssues = Objects.requireNonNull(indexSettingsIssues, "index settings issues cannot be null");
52+
this.mlSettingsIssues = Objects.requireNonNull(mlSettingsIssues, "ml settings issues cannot be null");
5053
}
5154

5255
public List<DeprecationIssue> getClusterSettingsIssues() {
@@ -61,6 +64,10 @@ public Map<String, List<DeprecationIssue>> getIndexSettingsIssues() {
6164
return indexSettingsIssues;
6265
}
6366

67+
public List<DeprecationIssue> getMlSettingsIssues() {
68+
return mlSettingsIssues;
69+
}
70+
6471
private static List<DeprecationIssue> parseDeprecationIssues(XContentParser parser) throws IOException {
6572
List<DeprecationIssue> issues = new ArrayList<>();
6673
XContentParser.Token token = null;
@@ -76,6 +83,7 @@ public static DeprecationInfoResponse fromXContent(XContentParser parser) throws
7683
Map<String, List<DeprecationIssue>> indexSettings = new HashMap<>();
7784
List<DeprecationIssue> clusterSettings = new ArrayList<>();
7885
List<DeprecationIssue> nodeSettings = new ArrayList<>();
86+
List<DeprecationIssue> mlSettings = new ArrayList<>();
7987
String fieldName = null;
8088
XContentParser.Token token;
8189
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -85,6 +93,8 @@ public static DeprecationInfoResponse fromXContent(XContentParser parser) throws
8593
clusterSettings.addAll(parseDeprecationIssues(parser));
8694
} else if (NODE_SETTINGS.getPreferredName().equals(fieldName)) {
8795
nodeSettings.addAll(parseDeprecationIssues(parser));
96+
} else if (ML_SETTINGS.getPreferredName().equals(fieldName)) {
97+
mlSettings.addAll(parseDeprecationIssues(parser));
8898
} else if (INDEX_SETTINGS.getPreferredName().equals(fieldName)) {
8999
// parse out the key/value pairs
90100
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -96,7 +106,7 @@ public static DeprecationInfoResponse fromXContent(XContentParser parser) throws
96106
}
97107
}
98108
}
99-
return new DeprecationInfoResponse(clusterSettings, nodeSettings, indexSettings);
109+
return new DeprecationInfoResponse(clusterSettings, nodeSettings, indexSettings, mlSettings);
100110
}
101111

102112
@Override
@@ -106,17 +116,19 @@ public boolean equals(Object o) {
106116
DeprecationInfoResponse that = (DeprecationInfoResponse) o;
107117
return Objects.equals(clusterSettingsIssues, that.clusterSettingsIssues) &&
108118
Objects.equals(nodeSettingsIssues, that.nodeSettingsIssues) &&
119+
Objects.equals(mlSettingsIssues, that.mlSettingsIssues) &&
109120
Objects.equals(indexSettingsIssues, that.indexSettingsIssues);
110121
}
111122

112123
@Override
113124
public int hashCode() {
114-
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues);
125+
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues);
115126
}
116127

117128
@Override
118129
public String toString() {
119-
return clusterSettingsIssues.toString() + ":" + nodeSettingsIssues.toString() + ":" + indexSettingsIssues.toString();
130+
return clusterSettingsIssues.toString() + ":" + nodeSettingsIssues.toString() + ":" + indexSettingsIssues.toString() +
131+
":" + mlSettingsIssues.toString();
120132
}
121133

122134
/**

client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java

+1
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ public void testClusterHealthYellowClusterLevel() throws IOException {
182182
assertThat(response.getIndices().size(), equalTo(0));
183183
}
184184

185+
@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/35450")
185186
public void testClusterHealthYellowIndicesLevel() throws IOException {
186187
createIndex("index", Settings.EMPTY);
187188
createIndex("index2", Settings.EMPTY);

client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java

+1
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ public void testGetDeprecationInfo() throws IOException {
8585
assertThat(response.getClusterSettingsIssues().size(), equalTo(0));
8686
assertThat(response.getIndexSettingsIssues().size(), equalTo(0));
8787
assertThat(response.getNodeSettingsIssues().size(), equalTo(0));
88+
assertThat(response.getMlSettingsIssues().size(), equalTo(0));
8889
}
8990

9091
/**

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationClientDocumentationIT.java

+4
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,8 @@ public void testGetDeprecationInfo() throws IOException, InterruptedException {
182182
deprecationInfoResponse.getNodeSettingsIssues(); // <2>
183183
Map<String, List<DeprecationInfoResponse.DeprecationIssue>> indexIssues =
184184
deprecationInfoResponse.getIndexSettingsIssues(); // <3>
185+
List<DeprecationInfoResponse.DeprecationIssue> mlIssues =
186+
deprecationInfoResponse.getMlSettingsIssues(); // <4>
185187
// end::get-deprecation-info-response
186188

187189
// tag::get-deprecation-info-execute-listener
@@ -195,6 +197,8 @@ public void onResponse(DeprecationInfoResponse deprecationInfoResponse1) { // <1
195197
deprecationInfoResponse.getNodeSettingsIssues();
196198
Map<String, List<DeprecationInfoResponse.DeprecationIssue>> indexIssues =
197199
deprecationInfoResponse.getIndexSettingsIssues();
200+
List<DeprecationInfoResponse.DeprecationIssue> mlIssues =
201+
deprecationInfoResponse.getMlSettingsIssues();
198202
}
199203

200204
@Override

client/rest-high-level/src/test/java/org/elasticsearch/client/migration/DeprecationInfoResponseTests.java

+19-6
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ private void toXContent(DeprecationInfoResponse response, XContentBuilder builde
6565
}
6666
}
6767
builder.endObject();
68+
69+
builder.startArray("ml_settings");
70+
for (DeprecationInfoResponse.DeprecationIssue issue : response.getMlSettingsIssues()) {
71+
toXContent(issue, builder);
72+
}
73+
builder.endArray();
6874
}
6975
builder.endObject();
7076
}
@@ -105,12 +111,14 @@ private List<DeprecationInfoResponse.DeprecationIssue> createRandomIssues(boolea
105111
}
106112

107113
private DeprecationInfoResponse createInstance() {
108-
return new DeprecationInfoResponse(createRandomIssues(true), createRandomIssues(true), createIndexSettingsIssues());
114+
return new DeprecationInfoResponse(createRandomIssues(true), createRandomIssues(true), createIndexSettingsIssues(),
115+
createRandomIssues(true));
109116
}
110117

111118
private DeprecationInfoResponse copyInstance(DeprecationInfoResponse req) {
112119
return new DeprecationInfoResponse(new ArrayList<>(req.getClusterSettingsIssues()),
113-
new ArrayList<>(req.getNodeSettingsIssues()), new HashMap<>(req.getIndexSettingsIssues()));
120+
new ArrayList<>(req.getNodeSettingsIssues()), new HashMap<>(req.getIndexSettingsIssues()),
121+
new ArrayList<>(req.getMlSettingsIssues()));
114122
}
115123

116124
private DeprecationInfoResponse mutateInstance(DeprecationInfoResponse req) {
@@ -128,16 +136,21 @@ public void testFromXContent() throws IOException {
128136
}
129137

130138
public void testNullFailedIndices() {
131-
NullPointerException exception =
132-
expectThrows(NullPointerException.class, () -> new DeprecationInfoResponse(null, null, null));
139+
NullPointerException exception = expectThrows(NullPointerException.class,
140+
() -> new DeprecationInfoResponse(null, null, null, null));
133141
assertEquals("cluster settings issues cannot be null", exception.getMessage());
134142

135-
exception = expectThrows(NullPointerException.class, () -> new DeprecationInfoResponse(Collections.emptyList(), null, null));
143+
exception = expectThrows(NullPointerException.class,
144+
() -> new DeprecationInfoResponse(Collections.emptyList(), null, null, null));
136145
assertEquals("node settings issues cannot be null", exception.getMessage());
137146

138147
exception = expectThrows(NullPointerException.class,
139-
() -> new DeprecationInfoResponse(Collections.emptyList(), Collections.emptyList(), null));
148+
() -> new DeprecationInfoResponse(Collections.emptyList(), Collections.emptyList(), null, null));
140149
assertEquals("index settings issues cannot be null", exception.getMessage());
150+
151+
exception = expectThrows(NullPointerException.class,
152+
() -> new DeprecationInfoResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null));
153+
assertEquals("ml settings issues cannot be null", exception.getMessage());
141154
}
142155

143156
public void testEqualsAndHashCode() {

docs/java-rest/high-level/migration/get-deprecation-info.asciidoc

+1
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,4 @@ include-tagged::{doc-tests-file}[{api}-response]
3333
<1> a List of Cluster deprecations
3434
<2> a List of Node deprecations
3535
<3> a Map of key IndexName, value List of deprecations for the index
36+
<4> a list of Machine Learning related deprecations
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
[[search-aggregations-bucket-geotilegrid-aggregation]]
2+
=== GeoTile Grid Aggregation
3+
4+
A multi-bucket aggregation that works on `geo_point` fields and groups points into
5+
buckets that represent cells in a grid. The resulting grid can be sparse and only
6+
contains cells that have matching data. Each cell corresponds to a
7+
https://en.wikipedia.org/wiki/Tiled_web_map[map tile] as used by many online map
8+
sites. Each cell is labeled using a "{zoom}/{x}/{y}" format, where zoom is equal
9+
to the user-specified precision.
10+
11+
* High precision keys have a larger range for x and y, and represent tiles that
12+
cover only a small area.
13+
* Low precision keys have a smaller range for x and y, and represent tiles that
14+
each cover a large area.
15+
16+
See https://wiki.openstreetmap.org/wiki/Zoom_levels[Zoom level documentation]
17+
on how precision (zoom) correlates to size on the ground. Precision for this
18+
aggregation can be between 0 and 29, inclusive.
19+
20+
WARNING: The highest-precision geotile of length 29 produces cells that cover
21+
less than a 10cm by 10cm of land and so high-precision requests can be very
22+
costly in terms of RAM and result sizes. Please see the example below on how
23+
to first filter the aggregation to a smaller geographic area before requesting
24+
high-levels of detail.
25+
26+
The specified field must be of type `geo_point` (which can only be set
27+
explicitly in the mappings) and it can also hold an array of `geo_point`
28+
fields, in which case all points will be taken into account during aggregation.
29+
30+
31+
==== Simple low-precision request
32+
33+
[source,js]
34+
--------------------------------------------------
35+
PUT /museums
36+
{
37+
"mappings": {
38+
"properties": {
39+
"location": {
40+
"type": "geo_point"
41+
}
42+
}
43+
}
44+
}
45+
46+
POST /museums/_bulk?refresh
47+
{"index":{"_id":1}}
48+
{"location": "52.374081,4.912350", "name": "NEMO Science Museum"}
49+
{"index":{"_id":2}}
50+
{"location": "52.369219,4.901618", "name": "Museum Het Rembrandthuis"}
51+
{"index":{"_id":3}}
52+
{"location": "52.371667,4.914722", "name": "Nederlands Scheepvaartmuseum"}
53+
{"index":{"_id":4}}
54+
{"location": "51.222900,4.405200", "name": "Letterenhuis"}
55+
{"index":{"_id":5}}
56+
{"location": "48.861111,2.336389", "name": "Musée du Louvre"}
57+
{"index":{"_id":6}}
58+
{"location": "48.860000,2.327000", "name": "Musée d'Orsay"}
59+
60+
POST /museums/_search?size=0
61+
{
62+
"aggregations" : {
63+
"large-grid" : {
64+
"geotile_grid" : {
65+
"field" : "location",
66+
"precision" : 8
67+
}
68+
}
69+
}
70+
}
71+
--------------------------------------------------
72+
// CONSOLE
73+
74+
Response:
75+
76+
[source,js]
77+
--------------------------------------------------
78+
{
79+
...
80+
"aggregations": {
81+
"large-grid": {
82+
"buckets": [
83+
{
84+
"key" : "8/131/84",
85+
"doc_count" : 3
86+
},
87+
{
88+
"key" : "8/129/88",
89+
"doc_count" : 2
90+
},
91+
{
92+
"key" : "8/131/85",
93+
"doc_count" : 1
94+
}
95+
]
96+
}
97+
}
98+
}
99+
--------------------------------------------------
100+
// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
101+
102+
==== High-precision requests
103+
104+
When requesting detailed buckets (typically for displaying a "zoomed in" map)
105+
a filter like <<query-dsl-geo-bounding-box-query,geo_bounding_box>> should be
106+
applied to narrow the subject area otherwise potentially millions of buckets
107+
will be created and returned.
108+
109+
[source,js]
110+
--------------------------------------------------
111+
POST /museums/_search?size=0
112+
{
113+
"aggregations" : {
114+
"zoomed-in" : {
115+
"filter" : {
116+
"geo_bounding_box" : {
117+
"location" : {
118+
"top_left" : "52.4, 4.9",
119+
"bottom_right" : "52.3, 5.0"
120+
}
121+
}
122+
},
123+
"aggregations":{
124+
"zoom1":{
125+
"geotile_grid" : {
126+
"field": "location",
127+
"precision": 22
128+
}
129+
}
130+
}
131+
}
132+
}
133+
}
134+
--------------------------------------------------
135+
// CONSOLE
136+
// TEST[continued]
137+
138+
[source,js]
139+
--------------------------------------------------
140+
{
141+
...
142+
"aggregations" : {
143+
"zoomed-in" : {
144+
"doc_count" : 3,
145+
"zoom1" : {
146+
"buckets" : [
147+
{
148+
"key" : "22/2154412/1378379",
149+
"doc_count" : 1
150+
},
151+
{
152+
"key" : "22/2154385/1378332",
153+
"doc_count" : 1
154+
},
155+
{
156+
"key" : "22/2154259/1378425",
157+
"doc_count" : 1
158+
}
159+
]
160+
}
161+
}
162+
}
163+
}
164+
--------------------------------------------------
165+
// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
166+
167+
168+
==== Options
169+
170+
[horizontal]
171+
field:: Mandatory. The name of the field indexed with GeoPoints.
172+
173+
precision:: Optional. The integer zoom of the key used to define
174+
cells/buckets in the results. Defaults to 7.
175+
Values outside of [0,29] will be rejected.
176+
177+
size:: Optional. The maximum number of geohash buckets to return
178+
(defaults to 10,000). When results are trimmed, buckets are
179+
prioritised based on the volumes of documents they contain.
180+
181+
shard_size:: Optional. To allow for more accurate counting of the top cells
182+
returned in the final result the aggregation defaults to
183+
returning `max(10,(size x number-of-shards))` buckets from each
184+
shard. If this heuristic is undesirable, the number considered
185+
from each shard can be over-ridden using this parameter.

0 commit comments

Comments
 (0)