Skip to content

Commit d35c156

Browse files
DaveCTurnerDaveCTurner
committed
Merge branch 'master' into feature/searchable-snapshots
2 parents 4bde03a + 4f48e05 commit d35c156

File tree

228 files changed

+4855
-1358
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

228 files changed

+4855
-1358
lines changed

‎.ci/matrix-runtime-javas.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
ES_RUNTIME_JAVA:
99
- java11
10-
- openjdk13
1110
- openjdk14
1211
- openjdk15
1312
- zulu11

‎buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
143143
repo.patternLayout(layout -> layout.artifact(artifactPattern));
144144
});
145145
repositories.exclusiveContent(exclusiveContentRepository -> {
146-
exclusiveContentRepository.filter(config -> config.includeGroup(jdk.getVendor()));
146+
exclusiveContentRepository.filter(config -> config.includeGroup(groupName(jdk)));
147147
exclusiveContentRepository.forRepositories(ivyRepo);
148148
});
149149
}
@@ -254,7 +254,11 @@ private static String dependencyNotation(Jdk jdk) {
254254
: jdk.getPlatform();
255255
String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
256256

257-
return jdk.getVendor() + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
257+
return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
258+
}
259+
260+
private static String groupName(Jdk jdk) {
261+
return jdk.getVendor() + "_" + jdk.getMajor();
258262
}
259263

260264
private static String configName(String... parts) {

‎buildSrc/version.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jts = 1.15.0
1111
# you should also inspect that version to see if it can be advanced along with
1212
# the com.maxmind.geoip2:geoip2 dependency
1313
jackson = 2.8.11
14-
jacksondatabind = 2.8.11.4
14+
jacksondatabind = 2.8.11.6
1515
snakeyaml = 1.17
1616
icu4j = 62.1
1717
supercsv = 2.4.0

‎client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@
2323

2424
public class ProxyModeInfo implements RemoteConnectionInfo.ModeInfo {
2525
static final String NAME = "proxy";
26-
static final String ADDRESS = "address";
27-
static final String NUM_SOCKETS_CONNECTED = "num_sockets_connected";
28-
static final String MAX_SOCKET_CONNECTIONS = "max_socket_connections";
26+
static final String PROXY_ADDRESS = "proxy_address";
27+
static final String NUM_PROXY_SOCKETS_CONNECTED = "num_proxy_sockets_connected";
28+
static final String MAX_PROXY_SOCKET_CONNECTIONS = "max_proxy_socket_connections";
2929
private final String address;
3030
private final int maxSocketConnections;
3131
private final int numSocketsConnected;

‎client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,9 @@ public final class RemoteConnectionInfo {
6666
PARSER.declareString(constructorArg(), new ParseField(INITIAL_CONNECT_TIMEOUT));
6767
PARSER.declareBoolean(constructorArg(), new ParseField(SKIP_UNAVAILABLE));
6868

69-
PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.ADDRESS));
70-
PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_SOCKET_CONNECTIONS));
71-
PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_SOCKETS_CONNECTED));
69+
PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.PROXY_ADDRESS));
70+
PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_PROXY_SOCKET_CONNECTIONS));
71+
PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_PROXY_SOCKETS_CONNECTED));
7272

7373
PARSER.declareStringArray(optionalConstructorArg(), new ParseField(SniffModeInfo.SEEDS));
7474
PARSER.declareInt(optionalConstructorArg(), new ParseField(SniffModeInfo.MAX_CONNECTIONS_PER_CLUSTER));

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,12 @@
2222
import org.elasticsearch.common.ParseField;
2323
import org.elasticsearch.common.Strings;
2424
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
25+
import org.elasticsearch.common.xcontent.ObjectParser;
2526
import org.elasticsearch.common.xcontent.XContentBuilder;
2627
import org.elasticsearch.common.xcontent.XContentParser;
2728

2829
import java.io.IOException;
30+
import java.util.Locale;
2931
import java.util.Objects;
3032

3133
public class Classification implements DataFrameAnalysis {
@@ -49,6 +51,7 @@ public static Builder builder(String dependentVariable) {
4951
static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
5052
static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
5153
static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
54+
static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
5255
static final ParseField NUM_TOP_CLASSES = new ParseField("num_top_classes");
5356
static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
5457

@@ -67,7 +70,8 @@ public static Builder builder(String dependentVariable) {
6770
(String) a[7],
6871
(Double) a[8],
6972
(Integer) a[9],
70-
(Long) a[10]));
73+
(Long) a[10],
74+
(ClassAssignmentObjective) a[11]));
7175

7276
static {
7377
PARSER.declareString(ConstructingObjectParser.constructorArg(), DEPENDENT_VARIABLE);
@@ -81,6 +85,12 @@ public static Builder builder(String dependentVariable) {
8185
PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), TRAINING_PERCENT);
8286
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_CLASSES);
8387
PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), RANDOMIZE_SEED);
88+
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
89+
if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
90+
return ClassAssignmentObjective.fromString(p.text());
91+
}
92+
throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
93+
}, CLASS_ASSIGNMENT_OBJECTIVE, ObjectParser.ValueType.STRING);
8494
}
8595

8696
private final String dependentVariable;
@@ -92,13 +102,15 @@ public static Builder builder(String dependentVariable) {
92102
private final Integer numTopFeatureImportanceValues;
93103
private final String predictionFieldName;
94104
private final Double trainingPercent;
105+
private final ClassAssignmentObjective classAssignmentObjective;
95106
private final Integer numTopClasses;
96107
private final Long randomizeSeed;
97108

98109
private Classification(String dependentVariable, @Nullable Double lambda, @Nullable Double gamma, @Nullable Double eta,
99110
@Nullable Integer maxTrees, @Nullable Double featureBagFraction,
100111
@Nullable Integer numTopFeatureImportanceValues, @Nullable String predictionFieldName,
101-
@Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed) {
112+
@Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed,
113+
@Nullable ClassAssignmentObjective classAssignmentObjective) {
102114
this.dependentVariable = Objects.requireNonNull(dependentVariable);
103115
this.lambda = lambda;
104116
this.gamma = gamma;
@@ -108,6 +120,7 @@ private Classification(String dependentVariable, @Nullable Double lambda, @Nulla
108120
this.numTopFeatureImportanceValues = numTopFeatureImportanceValues;
109121
this.predictionFieldName = predictionFieldName;
110122
this.trainingPercent = trainingPercent;
123+
this.classAssignmentObjective = classAssignmentObjective;
111124
this.numTopClasses = numTopClasses;
112125
this.randomizeSeed = randomizeSeed;
113126
}
@@ -157,6 +170,10 @@ public Long getRandomizeSeed() {
157170
return randomizeSeed;
158171
}
159172

173+
public ClassAssignmentObjective getClassAssignmentObjective() {
174+
return classAssignmentObjective;
175+
}
176+
160177
public Integer getNumTopClasses() {
161178
return numTopClasses;
162179
}
@@ -192,6 +209,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
192209
if (randomizeSeed != null) {
193210
builder.field(RANDOMIZE_SEED.getPreferredName(), randomizeSeed);
194211
}
212+
if (classAssignmentObjective != null) {
213+
builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
214+
}
195215
if (numTopClasses != null) {
196216
builder.field(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
197217
}
@@ -202,7 +222,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
202222
@Override
203223
public int hashCode() {
204224
return Objects.hash(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues,
205-
predictionFieldName, trainingPercent, randomizeSeed, numTopClasses);
225+
predictionFieldName, trainingPercent, randomizeSeed, numTopClasses, classAssignmentObjective);
206226
}
207227

208228
@Override
@@ -220,14 +240,28 @@ public boolean equals(Object o) {
220240
&& Objects.equals(predictionFieldName, that.predictionFieldName)
221241
&& Objects.equals(trainingPercent, that.trainingPercent)
222242
&& Objects.equals(randomizeSeed, that.randomizeSeed)
223-
&& Objects.equals(numTopClasses, that.numTopClasses);
243+
&& Objects.equals(numTopClasses, that.numTopClasses)
244+
&& Objects.equals(classAssignmentObjective, that.classAssignmentObjective);
224245
}
225246

226247
@Override
227248
public String toString() {
228249
return Strings.toString(this);
229250
}
230251

252+
public enum ClassAssignmentObjective {
253+
MAXIMIZE_ACCURACY, MAXIMIZE_MINIMUM_RECALL;
254+
255+
public static ClassAssignmentObjective fromString(String value) {
256+
return ClassAssignmentObjective.valueOf(value.toUpperCase(Locale.ROOT));
257+
}
258+
259+
@Override
260+
public String toString() {
261+
return name().toLowerCase(Locale.ROOT);
262+
}
263+
}
264+
231265
public static class Builder {
232266
private String dependentVariable;
233267
private Double lambda;
@@ -240,6 +274,7 @@ public static class Builder {
240274
private Double trainingPercent;
241275
private Integer numTopClasses;
242276
private Long randomizeSeed;
277+
private ClassAssignmentObjective classAssignmentObjective;
243278

244279
private Builder(String dependentVariable) {
245280
this.dependentVariable = Objects.requireNonNull(dependentVariable);
@@ -295,9 +330,15 @@ public Builder setNumTopClasses(Integer numTopClasses) {
295330
return this;
296331
}
297332

333+
public Builder setClassAssignmentObjective(ClassAssignmentObjective classAssignmentObjective) {
334+
this.classAssignmentObjective = classAssignmentObjective;
335+
return this;
336+
}
337+
298338
public Classification build() {
299339
return new Classification(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction,
300-
numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed);
340+
numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed,
341+
classAssignmentObjective);
301342
}
302343
}
303344
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,8 @@ public class TrainedModelDefinition implements ToXContentObject {
4343
true,
4444
TrainedModelDefinition.Builder::new);
4545
static {
46-
PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setTrainedModel,
46+
PARSER.declareNamedObject(TrainedModelDefinition.Builder::setTrainedModel,
4747
(p, c, n) -> p.namedObject(TrainedModel.class, n, null),
48-
(modelDocBuilder) -> { /* Noop does not matter client side*/ },
4948
TRAINED_MODEL);
5049
PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setPreProcessors,
5150
(p, c, n) -> p.namedObject(PreProcessor.class, n, null),
@@ -124,11 +123,6 @@ public Builder setTrainedModel(TrainedModel trainedModel) {
124123
return this;
125124
}
126125

127-
private Builder setTrainedModel(List<TrainedModel> trainedModel) {
128-
assert trainedModel.size() == 1;
129-
return setTrainedModel(trainedModel.get(0));
130-
}
131-
132126
public TrainedModelDefinition build() {
133127
return new TrainedModelDefinition(this.trainedModel, this.preProcessors);
134128
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,8 @@ public class Ensemble implements TrainedModel {
5656
p.namedObject(TrainedModel.class, n, null),
5757
(ensembleBuilder) -> { /* Noop does not matter client side */ },
5858
TRAINED_MODELS);
59-
PARSER.declareNamedObjects(Ensemble.Builder::setOutputAggregatorFromParser,
59+
PARSER.declareNamedObject(Ensemble.Builder::setOutputAggregator,
6060
(p, c, n) -> p.namedObject(OutputAggregator.class, n, null),
61-
(ensembleBuilder) -> { /* Noop does not matter client side */ },
6261
AGGREGATE_OUTPUT);
6362
PARSER.declareString(Ensemble.Builder::setTargetType, TARGET_TYPE);
6463
PARSER.declareStringArray(Ensemble.Builder::setClassificationLabels, CLASSIFICATION_LABELS);
@@ -194,9 +193,6 @@ public Builder setClassificationWeights(List<Double> classificationWeights) {
194193
return this;
195194
}
196195

197-
private void setOutputAggregatorFromParser(List<OutputAggregator> outputAggregators) {
198-
this.setOutputAggregator(outputAggregators.get(0));
199-
}
200196

201197
private void setTargetType(String targetType) {
202198
this.targetType = TargetType.fromString(targetType);

‎client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1336,6 +1336,8 @@ public void testPutDataFrameAnalyticsConfig_GivenClassification() throws Excepti
13361336
.setPredictionFieldName("my_dependent_variable_prediction")
13371337
.setTrainingPercent(80.0)
13381338
.setRandomizeSeed(42L)
1339+
.setClassAssignmentObjective(
1340+
org.elasticsearch.client.ml.dataframe.Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY)
13391341
.setNumTopClasses(1)
13401342
.setLambda(1.0)
13411343
.setGamma(1.0)

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@
139139
import org.elasticsearch.client.ml.datafeed.DatafeedStats;
140140
import org.elasticsearch.client.ml.datafeed.DatafeedUpdate;
141141
import org.elasticsearch.client.ml.datafeed.DelayedDataCheckConfig;
142+
import org.elasticsearch.client.ml.dataframe.Classification;
142143
import org.elasticsearch.client.ml.dataframe.DataFrameAnalysis;
143144
import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
144145
import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsDest;
@@ -2969,7 +2970,7 @@ public void testPutDataFrameAnalytics() throws Exception {
29692970
// end::put-data-frame-analytics-outlier-detection-customized
29702971

29712972
// tag::put-data-frame-analytics-classification
2972-
DataFrameAnalysis classification = org.elasticsearch.client.ml.dataframe.Classification.builder("my_dependent_variable") // <1>
2973+
DataFrameAnalysis classification = Classification.builder("my_dependent_variable") // <1>
29732974
.setLambda(1.0) // <2>
29742975
.setGamma(5.5) // <3>
29752976
.setEta(5.5) // <4>
@@ -2979,7 +2980,8 @@ public void testPutDataFrameAnalytics() throws Exception {
29792980
.setPredictionFieldName("my_prediction_field_name") // <8>
29802981
.setTrainingPercent(50.0) // <9>
29812982
.setRandomizeSeed(1234L) // <10>
2982-
.setNumTopClasses(1) // <11>
2983+
.setClassAssignmentObjective(Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY) // <11>
2984+
.setNumTopClasses(1) // <12>
29832985
.build();
29842986
// end::put-data-frame-analytics-classification
29852987

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public static Classification randomClassification() {
3636
.setPredictionFieldName(randomBoolean() ? null : randomAlphaOfLength(10))
3737
.setTrainingPercent(randomBoolean() ? null : randomDoubleBetween(1.0, 100.0, true))
3838
.setRandomizeSeed(randomBoolean() ? null : randomLong())
39+
.setClassAssignmentObjective(randomBoolean() ? null : randomFrom(Classification.ClassAssignmentObjective.values()))
3940
.setNumTopClasses(randomBoolean() ? null : randomIntBetween(0, 10))
4041
.build();
4142
}

‎distribution/docker/docker-compose.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Only used for testing the docker images
2-
version: '3.7'
2+
version: '3.4'
33
services:
44
elasticsearch-default-1:
55
image: elasticsearch:test

‎docs/Versions.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ ifeval::["{release-state}"=="unreleased"]
2929
:rest-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}-SNAPSHOT
3030
:rest-client-sniffer-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}-SNAPSHOT
3131
:rest-high-level-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}-SNAPSHOT
32-
:mapper-extras-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras/{version}-SNAPSHOT
32+
:mapper-extras-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras-client/{version}-SNAPSHOT
3333
:painless-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}-SNAPSHOT
3434
:parent-join-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}-SNAPSHOT
3535
:percolator-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}-SNAPSHOT
@@ -44,7 +44,7 @@ ifeval::["{release-state}"!="unreleased"]
4444
:rest-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client/{version}
4545
:rest-client-sniffer-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-client-sniffer/{version}
4646
:rest-high-level-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/client/elasticsearch-rest-high-level-client/{version}
47-
:mapper-extras-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras/{version}
47+
:mapper-extras-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/mapper-extras-client/{version}
4848
:painless-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}
4949
:parent-join-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}
5050
:percolator-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}

‎docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,8 @@ include-tagged::{doc-tests-file}[{api}-classification]
121121
<8> The name of the prediction field in the results object.
122122
<9> The percentage of training-eligible rows to be used in training. Defaults to 100%.
123123
<10> The seed to be used by the random generator that picks which rows are used in training.
124-
<11> The number of top classes to be reported in the results. Defaults to 2.
124+
<11> The optimization objective to target when assigning class labels. Defaults to maximize_minimum_recall.
125+
<12> The number of top classes to be reported in the results. Defaults to 2.
125126

126127
===== Regression
127128

‎docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@
44
<titleabbrev>Flatten graph</titleabbrev>
55
++++
66

7-
experimental[This functionality is marked as experimental in Lucene]
8-
97
The `flatten_graph` token filter accepts an arbitrary graph token
108
stream, such as that produced by
119
<<analysis-synonym-graph-tokenfilter>>, and flattens it into a single

0 commit comments

Comments
 (0)