Skip to content

Commit 097edf4

Browse files
DaveCTurnerDaveCTurner
committed
Merge branch 'master' into feature/searchable-snapshots
2 parents 5b115fb + e204894 commit 097edf4

File tree

880 files changed

+15655
-8525
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

880 files changed

+15655
-8525
lines changed

‎TESTING.asciidoc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,3 +638,9 @@ which you can use to measure the performance impact. It comes with a set of
638638
default benchmarks that we also
639639
https://elasticsearch-benchmarks.elastic.co/[run every night]. To get started,
640640
please see https://esrally.readthedocs.io/en/stable/[Rally's documentation].
641+
642+
== Test doc builds
643+
644+
The Elasticsearch docs are in AsciiDoc format. You can test and build the docs
645+
locally using the Elasticsearch documentation build process. See
646+
https://github.com/elastic/docs.

‎buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,13 +170,23 @@ public List<T> subList(int fromIndex, int toIndex) {
170170
}
171171

172172
@Override
173-
public List<? extends Object> getNormalizedCollection() {
173+
public List<? extends PropertyListEntry<T>> getNormalizedCollection() {
174174
return delegate.stream()
175175
.peek(this::validate)
176176
.filter(entry -> entry.getNormalization() != PropertyNormalization.IGNORE_VALUE)
177177
.collect(Collectors.toList());
178178
}
179179

180+
/**
181+
* Return a "flattened" collection. This should be used when the collection type is itself a complex type with properties
182+
* annotated as Gradle inputs rather than a simple type like {@link String}.
183+
*
184+
* @return a flattened collection filtered according to normalization strategy
185+
*/
186+
public List<? extends T> getFlatNormalizedCollection() {
187+
return getNormalizedCollection().stream().map(PropertyListEntry::getValue).collect(Collectors.toList());
188+
}
189+
180190
private void validate(PropertyListEntry<T> entry) {
181191
assertNotNull(entry.getValue(), "entry");
182192
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1163,7 +1163,7 @@ public List<?> getKeystoreFiles() {
11631163

11641164
@Nested
11651165
public List<?> getCliSetup() {
1166-
return cliSetup.getNormalizedCollection();
1166+
return cliSetup.getFlatNormalizedCollection();
11671167
}
11681168

11691169
@Nested

‎buildSrc/src/main/resources/minimumGradleVersion

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
6.1.1
1+
6.2

‎buildSrc/version.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ joda = 2.10.4
2929
# - plugins/ingest-attachment (transitive dependency, check the upstream POM)
3030
# - distribution/tools/plugin-cli
3131
# - x-pack/plugin/security
32-
bouncycastle = 1.61
32+
bouncycastle=1.64
3333
# test dependencies
3434
randomizedrunner = 2.7.6
3535
junit = 4.12

‎client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,6 @@
3939
import java.io.IOException;
4040
import java.util.List;
4141

42-
import static java.util.Arrays.asList;
43-
import static java.util.Collections.unmodifiableList;
4442
import static org.elasticsearch.rest.RestRequest.Method.POST;
4543
import static org.elasticsearch.rest.RestRequest.Method.PUT;
4644
import static org.elasticsearch.rest.RestStatus.OK;
@@ -49,11 +47,11 @@ public class RestNoopBulkAction extends BaseRestHandler {
4947

5048
@Override
5149
public List<Route> routes() {
52-
return unmodifiableList(asList(
50+
return List.of(
5351
new Route(POST, "/_noop_bulk"),
5452
new Route(PUT, "/_noop_bulk"),
5553
new Route(POST, "/{index}/_noop_bulk"),
56-
new Route(PUT, "/{index}/_noop_bulk")));
54+
new Route(PUT, "/{index}/_noop_bulk"));
5755
}
5856

5957
@Override

‎client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,20 +26,18 @@
2626

2727
import java.util.List;
2828

29-
import static java.util.Arrays.asList;
30-
import static java.util.Collections.unmodifiableList;
3129
import static org.elasticsearch.rest.RestRequest.Method.GET;
3230
import static org.elasticsearch.rest.RestRequest.Method.POST;
3331

3432
public class RestNoopSearchAction extends BaseRestHandler {
3533

3634
@Override
3735
public List<Route> routes() {
38-
return unmodifiableList(asList(
36+
return List.of(
3937
new Route(GET, "/_noop_search"),
4038
new Route(POST, "/_noop_search"),
4139
new Route(GET, "/{index}/_noop_search"),
42-
new Route(POST, "/{index}/_noop_search")));
40+
new Route(POST, "/{index}/_noop_search"));
4341
}
4442

4543
@Override

‎client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@
5454
import org.elasticsearch.action.support.master.AcknowledgedResponse;
5555
import org.elasticsearch.action.update.UpdateRequest;
5656
import org.elasticsearch.action.update.UpdateResponse;
57+
import org.elasticsearch.client.analytics.ParsedStringStats;
58+
import org.elasticsearch.client.analytics.ParsedTopMetrics;
59+
import org.elasticsearch.client.analytics.StringStatsAggregationBuilder;
60+
import org.elasticsearch.client.analytics.TopMetricsAggregationBuilder;
5761
import org.elasticsearch.client.core.CountRequest;
5862
import org.elasticsearch.client.core.CountResponse;
5963
import org.elasticsearch.client.core.GetSourceRequest;
@@ -1926,6 +1930,8 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
19261930
map.put(IpRangeAggregationBuilder.NAME, (p, c) -> ParsedBinaryRange.fromXContent(p, (String) c));
19271931
map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
19281932
map.put(CompositeAggregationBuilder.NAME, (p, c) -> ParsedComposite.fromXContent(p, (String) c));
1933+
map.put(StringStatsAggregationBuilder.NAME, (p, c) -> ParsedStringStats.PARSER.parse(p, (String) c));
1934+
map.put(TopMetricsAggregationBuilder.NAME, (p, c) -> ParsedTopMetrics.PARSER.parse(p, (String) c));
19291935
List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
19301936
.map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
19311937
.collect(Collectors.toList());

‎client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedStringStats.java

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.client.analytics;
21+
22+
import org.elasticsearch.common.ParseField;
23+
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
24+
import org.elasticsearch.common.xcontent.XContentBuilder;
25+
import org.elasticsearch.common.xcontent.XContentParser;
26+
import org.elasticsearch.search.aggregations.ParsedAggregation;
27+
28+
import java.io.IOException;
29+
import java.util.HashMap;
30+
import java.util.Map;
31+
32+
import static java.util.Collections.unmodifiableMap;
33+
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
34+
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
35+
36+
/**
37+
* Results from the {@code string_stats} aggregation.
38+
*/
39+
public class ParsedStringStats extends ParsedAggregation {
40+
private static final ParseField COUNT_FIELD = new ParseField("count");
41+
private static final ParseField MIN_LENGTH_FIELD = new ParseField("min_length");
42+
private static final ParseField MAX_LENGTH_FIELD = new ParseField("max_length");
43+
private static final ParseField AVG_LENGTH_FIELD = new ParseField("avg_length");
44+
private static final ParseField ENTROPY_FIELD = new ParseField("entropy");
45+
private static final ParseField DISTRIBUTION_FIELD = new ParseField("distribution");
46+
47+
private final long count;
48+
private final int minLength;
49+
private final int maxLength;
50+
private final double avgLength;
51+
private final double entropy;
52+
private final boolean showDistribution;
53+
private final Map<String, Double> distribution;
54+
55+
private ParsedStringStats(String name, long count, int minLength, int maxLength, double avgLength, double entropy,
56+
boolean showDistribution, Map<String, Double> distribution) {
57+
setName(name);
58+
this.count = count;
59+
this.minLength = minLength;
60+
this.maxLength = maxLength;
61+
this.avgLength = avgLength;
62+
this.entropy = entropy;
63+
this.showDistribution = showDistribution;
64+
this.distribution = distribution;
65+
}
66+
67+
/**
68+
* The number of non-empty fields counted.
69+
*/
70+
public long getCount() {
71+
return count;
72+
}
73+
74+
/**
75+
* The length of the shortest term.
76+
*/
77+
public int getMinLength() {
78+
return minLength;
79+
}
80+
81+
/**
82+
* The length of the longest term.
83+
*/
84+
public int getMaxLength() {
85+
return maxLength;
86+
}
87+
88+
/**
89+
* The average length computed over all terms.
90+
*/
91+
public double getAvgLength() {
92+
return avgLength;
93+
}
94+
95+
/**
96+
* The <a href="https://en.wikipedia.org/wiki/Entropy_(information_theory)">Shannon Entropy</a>
97+
* value computed over all terms collected by the aggregation.
98+
* Shannon entropy quantifies the amount of information contained in
99+
* the field. It is a very useful metric for measuring a wide range of
100+
* properties of a data set, such as diversity, similarity,
101+
* randomness etc.
102+
*/
103+
public double getEntropy() {
104+
return entropy;
105+
}
106+
107+
/**
108+
* The probability distribution for all characters. {@code null} unless
109+
* explicitly requested with {@link StringStatsAggregationBuilder#showDistribution(boolean)}.
110+
*/
111+
public Map<String, Double> getDistribution() {
112+
return distribution;
113+
}
114+
115+
@Override
116+
public String getType() {
117+
return StringStatsAggregationBuilder.NAME;
118+
}
119+
120+
private static final Object NULL_DISTRIBUTION_MARKER = new Object();
121+
public static final ConstructingObjectParser<ParsedStringStats, String> PARSER = new ConstructingObjectParser<>(
122+
StringStatsAggregationBuilder.NAME, true, (args, name) -> {
123+
long count = (long) args[0];
124+
boolean disributionWasExplicitNull = args[5] == NULL_DISTRIBUTION_MARKER;
125+
if (count == 0) {
126+
return new ParsedStringStats(name, count, 0, 0, 0, 0, disributionWasExplicitNull, null);
127+
}
128+
int minLength = (int) args[1];
129+
int maxLength = (int) args[2];
130+
double averageLength = (double) args[3];
131+
double entropy = (double) args[4];
132+
if (disributionWasExplicitNull) {
133+
return new ParsedStringStats(name, count, minLength, maxLength, averageLength, entropy,
134+
disributionWasExplicitNull, null);
135+
} else {
136+
@SuppressWarnings("unchecked")
137+
Map<String, Double> distribution = (Map<String, Double>) args[5];
138+
return new ParsedStringStats(name, count, minLength, maxLength, averageLength, entropy,
139+
distribution != null, distribution);
140+
}
141+
});
142+
static {
143+
PARSER.declareLong(constructorArg(), COUNT_FIELD);
144+
PARSER.declareIntOrNull(constructorArg(), 0, MIN_LENGTH_FIELD);
145+
PARSER.declareIntOrNull(constructorArg(), 0, MAX_LENGTH_FIELD);
146+
PARSER.declareDoubleOrNull(constructorArg(), 0, AVG_LENGTH_FIELD);
147+
PARSER.declareDoubleOrNull(constructorArg(), 0, ENTROPY_FIELD);
148+
PARSER.declareObjectOrNull(optionalConstructorArg(), (p, c) -> unmodifiableMap(p.map(HashMap::new, XContentParser::doubleValue)),
149+
NULL_DISTRIBUTION_MARKER, DISTRIBUTION_FIELD);
150+
ParsedAggregation.declareAggregationFields(PARSER);
151+
}
152+
153+
@Override
154+
protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
155+
builder.field(COUNT_FIELD.getPreferredName(), count);
156+
if (count == 0) {
157+
builder.nullField(MIN_LENGTH_FIELD.getPreferredName());
158+
builder.nullField(MAX_LENGTH_FIELD.getPreferredName());
159+
builder.nullField(AVG_LENGTH_FIELD.getPreferredName());
160+
builder.field(ENTROPY_FIELD.getPreferredName(), 0.0);
161+
} else {
162+
builder.field(MIN_LENGTH_FIELD.getPreferredName(), minLength);
163+
builder.field(MAX_LENGTH_FIELD.getPreferredName(), maxLength);
164+
builder.field(AVG_LENGTH_FIELD.getPreferredName(), avgLength);
165+
builder.field(ENTROPY_FIELD.getPreferredName(), entropy);
166+
}
167+
if (showDistribution) {
168+
builder.field(DISTRIBUTION_FIELD.getPreferredName(), distribution);
169+
}
170+
return builder;
171+
}
172+
}

0 commit comments

Comments
 (0)