Skip to content

Commit 2fbb88c

Browse files
ywelschywelsch
committed
Merge remote-tracking branch 'elastic/master' into use-plugins-for-node-tool
2 parents ce68d75 + d9560e5 commit 2fbb88c

File tree

205 files changed

+4209
-1292
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

205 files changed

+4209
-1292
lines changed

‎.ci/bwcVersions

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ BWC_VERSION:
1111
- "7.4.0"
1212
- "7.4.1"
1313
- "7.4.2"
14-
- "7.4.3"
1514
- "7.5.0"
15+
- "7.5.1"
1616
- "7.6.0"
1717
- "8.0.0"

‎CONTRIBUTING.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,10 @@ We support development in the Eclipse and IntelliJ IDEs.
115115
For Eclipse, the minimum version that we support is [4.13][eclipse].
116116
For IntelliJ, the minimum version that we support is [IntelliJ 2017.2][intellij].
117117

118+
[Docker](https://docs.docker.com/install/) is required for building some Elasticsearch artifacts and executing certain test suites. You can run Elasticsearch without building all the artifacts with:
119+
120+
./gradlew :run
121+
118122
### Configuring IDEs And Running Tests
119123

120124
Eclipse users can automatically configure their IDE: `./gradlew eclipse`
@@ -153,6 +157,10 @@ For IntelliJ, go to
153157
For Eclipse, go to `Preferences->Java->Installed JREs` and add `-ea` to
154158
`VM Arguments`.
155159

160+
Some tests related to locale testing also require the flag
161+
`-Djava.locale.providers` to be set. Set the VM options/VM arguments for
162+
IntelliJ or Eclipse like describe above to use
163+
`-Djava.locale.providers=SPI,COMPAT`.
156164

157165
### Java Language Formatting Guidelines
158166

‎TESTING.asciidoc

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,10 @@ If you want to just run the precommit checks:
263263

264264
Some of these checks will require `docker-compose` installed for bringing up
265265
test fixtures. If it's not present those checks will be skipped automatically.
266+
The host running Docker (or VM if you're using Docker Desktop) needs 4GB of
267+
memory or some of the containers will fail to start. You can tell that you
268+
are short of memory if containers are exiting quickly after starting with
269+
code 137 (128 + 9, where 9 means SIGKILL).
266270

267271
== Testing the REST layer
268272

‎client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,6 @@
3333
import org.elasticsearch.client.indices.GetIndexRequest;
3434
import org.elasticsearch.client.ml.CloseJobRequest;
3535
import org.elasticsearch.client.ml.CloseJobResponse;
36-
import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
37-
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
38-
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse;
3936
import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
4037
import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
4138
import org.elasticsearch.client.ml.DeleteCalendarRequest;
@@ -48,8 +45,11 @@
4845
import org.elasticsearch.client.ml.DeleteJobRequest;
4946
import org.elasticsearch.client.ml.DeleteJobResponse;
5047
import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
48+
import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
5149
import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
5250
import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
51+
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
52+
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse;
5353
import org.elasticsearch.client.ml.FindFileStructureRequest;
5454
import org.elasticsearch.client.ml.FindFileStructureResponse;
5555
import org.elasticsearch.client.ml.FlushJobRequest;
@@ -135,8 +135,6 @@
135135
import org.elasticsearch.client.ml.dataframe.evaluation.classification.AccuracyMetric;
136136
import org.elasticsearch.client.ml.dataframe.evaluation.classification.Classification;
137137
import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric;
138-
import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric.ActualClass;
139-
import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric.PredictedClass;
140138
import org.elasticsearch.client.ml.dataframe.evaluation.regression.MeanSquaredErrorMetric;
141139
import org.elasticsearch.client.ml.dataframe.evaluation.regression.RSquaredMetric;
142140
import org.elasticsearch.client.ml.dataframe.evaluation.regression.Regression;
@@ -1822,9 +1820,12 @@ public void testEvaluateDataFrame_Classification() throws IOException {
18221820
accuracyResult.getActualClasses(),
18231821
equalTo(
18241822
List.of(
1825-
new AccuracyMetric.ActualClass("cat", 5, 0.6), // 3 out of 5 examples labeled as "cat" were classified correctly
1826-
new AccuracyMetric.ActualClass("dog", 4, 0.75), // 3 out of 4 examples labeled as "dog" were classified correctly
1827-
new AccuracyMetric.ActualClass("ant", 1, 0.0)))); // no examples labeled as "ant" were classified correctly
1823+
// 3 out of 5 examples labeled as "cat" were classified correctly
1824+
new AccuracyMetric.ActualClass("cat", 5, 0.6),
1825+
// 3 out of 4 examples labeled as "dog" were classified correctly
1826+
new AccuracyMetric.ActualClass("dog", 4, 0.75),
1827+
// no examples labeled as "ant" were classified correctly
1828+
new AccuracyMetric.ActualClass("ant", 1, 0.0))));
18281829
assertThat(accuracyResult.getOverallAccuracy(), equalTo(0.6)); // 6 out of 10 examples were classified correctly
18291830
}
18301831
{ // No size provided for MulticlassConfusionMatrixMetric, default used instead
@@ -1846,20 +1847,29 @@ public void testEvaluateDataFrame_Classification() throws IOException {
18461847
mcmResult.getConfusionMatrix(),
18471848
equalTo(
18481849
List.of(
1849-
new ActualClass(
1850+
new MulticlassConfusionMatrixMetric.ActualClass(
18501851
"ant",
18511852
1L,
1852-
List.of(new PredictedClass("ant", 0L), new PredictedClass("cat", 1L), new PredictedClass("dog", 0L)),
1853+
List.of(
1854+
new MulticlassConfusionMatrixMetric.PredictedClass("ant", 0L),
1855+
new MulticlassConfusionMatrixMetric.PredictedClass("cat", 1L),
1856+
new MulticlassConfusionMatrixMetric.PredictedClass("dog", 0L)),
18531857
0L),
1854-
new ActualClass(
1858+
new MulticlassConfusionMatrixMetric.ActualClass(
18551859
"cat",
18561860
5L,
1857-
List.of(new PredictedClass("ant", 0L), new PredictedClass("cat", 3L), new PredictedClass("dog", 1L)),
1861+
List.of(
1862+
new MulticlassConfusionMatrixMetric.PredictedClass("ant", 0L),
1863+
new MulticlassConfusionMatrixMetric.PredictedClass("cat", 3L),
1864+
new MulticlassConfusionMatrixMetric.PredictedClass("dog", 1L)),
18581865
1L),
1859-
new ActualClass(
1866+
new MulticlassConfusionMatrixMetric.ActualClass(
18601867
"dog",
18611868
4L,
1862-
List.of(new PredictedClass("ant", 0L), new PredictedClass("cat", 1L), new PredictedClass("dog", 3L)),
1869+
List.of(
1870+
new MulticlassConfusionMatrixMetric.PredictedClass("ant", 0L),
1871+
new MulticlassConfusionMatrixMetric.PredictedClass("cat", 1L),
1872+
new MulticlassConfusionMatrixMetric.PredictedClass("dog", 3L)),
18631873
0L))));
18641874
assertThat(mcmResult.getOtherActualClassCount(), equalTo(0L));
18651875
}
@@ -1882,8 +1892,20 @@ public void testEvaluateDataFrame_Classification() throws IOException {
18821892
mcmResult.getConfusionMatrix(),
18831893
equalTo(
18841894
List.of(
1885-
new ActualClass("cat", 5L, List.of(new PredictedClass("cat", 3L), new PredictedClass("dog", 1L)), 1L),
1886-
new ActualClass("dog", 4L, List.of(new PredictedClass("cat", 1L), new PredictedClass("dog", 3L)), 0L)
1895+
new MulticlassConfusionMatrixMetric.ActualClass(
1896+
"cat",
1897+
5L,
1898+
List.of(
1899+
new MulticlassConfusionMatrixMetric.PredictedClass("cat", 3L),
1900+
new MulticlassConfusionMatrixMetric.PredictedClass("dog", 1L)),
1901+
1L),
1902+
new MulticlassConfusionMatrixMetric.ActualClass(
1903+
"dog",
1904+
4L,
1905+
List.of(
1906+
new MulticlassConfusionMatrixMetric.PredictedClass("cat", 1L),
1907+
new MulticlassConfusionMatrixMetric.PredictedClass("dog", 3L)),
1908+
0L)
18871909
)));
18881910
assertThat(mcmResult.getOtherActualClassCount(), equalTo(1L));
18891911
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -765,6 +765,7 @@ public void testApiNamingConventions() throws Exception {
765765
"cluster.remote_info",
766766
"create",
767767
"get_script_context",
768+
"get_script_languages",
768769
"get_source",
769770
"indices.exists_type",
770771
"indices.get_upgrade",

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@
8383
import org.elasticsearch.script.Script;
8484
import org.elasticsearch.script.ScriptType;
8585
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
86-
import org.elasticsearch.search.sort.SortOrder;
8786
import org.elasticsearch.tasks.TaskId;
8887

8988
import java.util.Collections;
@@ -833,10 +832,6 @@ public void testReindex() throws Exception {
833832
// tag::reindex-request-pipeline
834833
request.setDestPipeline("my_pipeline"); // <1>
835834
// end::reindex-request-pipeline
836-
// tag::reindex-request-sort
837-
request.addSortField("field1", SortOrder.DESC); // <1>
838-
request.addSortField("field2", SortOrder.ASC); // <2>
839-
// end::reindex-request-sort
840835
// tag::reindex-request-script
841836
request.setScript(
842837
new Script(

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,36 +20,56 @@
2020

2121
import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
2222
import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
23+
import org.elasticsearch.client.ml.dataframe.evaluation.classification.AccuracyMetricResultTests;
24+
import org.elasticsearch.client.ml.dataframe.evaluation.classification.Classification;
25+
import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetricResultTests;
2326
import org.elasticsearch.client.ml.dataframe.evaluation.regression.MeanSquaredErrorMetricResultTests;
27+
import org.elasticsearch.client.ml.dataframe.evaluation.regression.RSquaredMetricResultTests;
28+
import org.elasticsearch.client.ml.dataframe.evaluation.regression.Regression;
29+
import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetricResultTests;
30+
import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
31+
import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetricResultTests;
32+
import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetricResultTests;
33+
import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetricResultTests;
2434
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
2535
import org.elasticsearch.common.xcontent.XContentParser;
2636
import org.elasticsearch.test.AbstractXContentTestCase;
2737

2838
import java.io.IOException;
29-
import java.util.ArrayList;
39+
import java.util.Arrays;
3040
import java.util.List;
3141
import java.util.function.Predicate;
3242

3343
public class EvaluateDataFrameResponseTests extends AbstractXContentTestCase<EvaluateDataFrameResponse> {
3444

3545
public static EvaluateDataFrameResponse randomResponse() {
36-
List<EvaluationMetric.Result> metrics = new ArrayList<>();
37-
if (randomBoolean()) {
38-
metrics.add(AucRocMetricResultTests.randomResult());
46+
String evaluationName = randomFrom(BinarySoftClassification.NAME, Classification.NAME, Regression.NAME);
47+
List<EvaluationMetric.Result> metrics;
48+
switch (evaluationName) {
49+
case BinarySoftClassification.NAME:
50+
metrics = randomSubsetOf(
51+
Arrays.asList(
52+
AucRocMetricResultTests.randomResult(),
53+
PrecisionMetricResultTests.randomResult(),
54+
RecallMetricResultTests.randomResult(),
55+
ConfusionMatrixMetricResultTests.randomResult()));
56+
break;
57+
case Regression.NAME:
58+
metrics = randomSubsetOf(
59+
Arrays.asList(
60+
MeanSquaredErrorMetricResultTests.randomResult(),
61+
RSquaredMetricResultTests.randomResult()));
62+
break;
63+
case Classification.NAME:
64+
metrics = randomSubsetOf(
65+
Arrays.asList(
66+
AccuracyMetricResultTests.randomResult(),
67+
MulticlassConfusionMatrixMetricResultTests.randomResult()));
68+
break;
69+
default:
70+
throw new AssertionError("Please add missing \"case\" variant to the \"switch\" statement");
3971
}
40-
if (randomBoolean()) {
41-
metrics.add(PrecisionMetricResultTests.randomResult());
42-
}
43-
if (randomBoolean()) {
44-
metrics.add(RecallMetricResultTests.randomResult());
45-
}
46-
if (randomBoolean()) {
47-
metrics.add(ConfusionMatrixMetricResultTests.randomResult());
48-
}
49-
if (randomBoolean()) {
50-
metrics.add(MeanSquaredErrorMetricResultTests.randomResult());
51-
}
52-
return new EvaluateDataFrameResponse(randomAlphaOfLength(5), metrics);
72+
return new EvaluateDataFrameResponse(evaluationName, metrics);
5373
}
5474

5575
@Override

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetricResultTests.java

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,14 @@
3131
import java.util.stream.Collectors;
3232
import java.util.stream.Stream;
3333

34-
public class AccuracyMetricResultTests extends AbstractXContentTestCase<AccuracyMetric.Result> {
34+
public class AccuracyMetricResultTests extends AbstractXContentTestCase<Result> {
3535

3636
@Override
3737
protected NamedXContentRegistry xContentRegistry() {
3838
return new NamedXContentRegistry(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
3939
}
4040

41-
@Override
42-
protected AccuracyMetric.Result createTestInstance() {
41+
public static Result randomResult() {
4342
int numClasses = randomIntBetween(2, 100);
4443
List<String> classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList());
4544
List<ActualClass> actualClasses = new ArrayList<>(numClasses);
@@ -52,8 +51,13 @@ protected AccuracyMetric.Result createTestInstance() {
5251
}
5352

5453
@Override
55-
protected AccuracyMetric.Result doParseInstance(XContentParser parser) throws IOException {
56-
return AccuracyMetric.Result.fromXContent(parser);
54+
protected Result createTestInstance() {
55+
return randomResult();
56+
}
57+
58+
@Override
59+
protected Result doParseInstance(XContentParser parser) throws IOException {
60+
return Result.fromXContent(parser);
5761
}
5862

5963
@Override

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/ClassificationTests.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ protected NamedXContentRegistry xContentRegistry() {
3838

3939
static Classification createRandom() {
4040
List<EvaluationMetric> metrics =
41-
randomSubsetOf(Arrays.asList(AccuracyMetricTests.createRandom(), MulticlassConfusionMatrixMetricTests.createRandom()));
41+
randomSubsetOf(
42+
Arrays.asList(
43+
AccuracyMetricTests.createRandom(),
44+
MulticlassConfusionMatrixMetricTests.createRandom()));
4245
return new Classification(randomAlphaOfLength(10), randomAlphaOfLength(10), metrics.isEmpty() ? null : metrics);
4346
}
4447

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixMetricResultTests.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,7 @@ protected NamedXContentRegistry xContentRegistry() {
4040
return new NamedXContentRegistry(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
4141
}
4242

43-
@Override
44-
protected Result createTestInstance() {
43+
public static Result randomResult() {
4544
int numClasses = randomIntBetween(2, 100);
4645
List<String> classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList());
4746
List<ActualClass> actualClasses = new ArrayList<>(numClasses);
@@ -60,6 +59,11 @@ protected Result createTestInstance() {
6059
return new Result(actualClasses, randomBoolean() ? randomNonNegativeLong() : null);
6160
}
6261

62+
@Override
63+
protected Result createTestInstance() {
64+
return randomResult();
65+
}
66+
6367
@Override
6468
protected Result doParseInstance(XContentParser parser) throws IOException {
6569
return Result.fromXContent(parser);

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricAucRocPointTests.java renamed to ‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AucRocMetricAucRocPointTests.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,8 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
package org.elasticsearch.client.ml;
19+
package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
2020

21-
import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
2221
import org.elasticsearch.common.xcontent.XContentParser;
2322
import org.elasticsearch.test.AbstractXContentTestCase;
2423

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricResultTests.java renamed to ‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AucRocMetricResultTests.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,8 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
package org.elasticsearch.client.ml;
19+
package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
2020

21-
import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
2221
import org.elasticsearch.common.xcontent.XContentParser;
2322
import org.elasticsearch.test.AbstractXContentTestCase;
2423

@@ -27,11 +26,11 @@
2726
import java.util.stream.Collectors;
2827
import java.util.stream.Stream;
2928

30-
import static org.elasticsearch.client.ml.AucRocMetricAucRocPointTests.randomPoint;
29+
import static org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetricAucRocPointTests.randomPoint;
3130

3231
public class AucRocMetricResultTests extends AbstractXContentTestCase<AucRocMetric.Result> {
3332

34-
static AucRocMetric.Result randomResult() {
33+
public static AucRocMetric.Result randomResult() {
3534
return new AucRocMetric.Result(
3635
randomDouble(),
3736
Stream

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricConfusionMatrixTests.java renamed to ‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/ConfusionMatrixMetricConfusionMatrixTests.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,8 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
package org.elasticsearch.client.ml;
19+
package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
2020

21-
import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
2221
import org.elasticsearch.common.xcontent.XContentParser;
2322
import org.elasticsearch.test.AbstractXContentTestCase;
2423

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricResultTests.java renamed to ‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/ConfusionMatrixMetricResultTests.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,8 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
package org.elasticsearch.client.ml;
19+
package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
2020

21-
import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
2221
import org.elasticsearch.common.xcontent.XContentParser;
2322
import org.elasticsearch.test.AbstractXContentTestCase;
2423

@@ -27,11 +26,11 @@
2726
import java.util.stream.Collectors;
2827
import java.util.stream.Stream;
2928

30-
import static org.elasticsearch.client.ml.ConfusionMatrixMetricConfusionMatrixTests.randomConfusionMatrix;
29+
import static org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetricConfusionMatrixTests.randomConfusionMatrix;
3130

3231
public class ConfusionMatrixMetricResultTests extends AbstractXContentTestCase<ConfusionMatrixMetric.Result> {
3332

34-
static ConfusionMatrixMetric.Result randomResult() {
33+
public static ConfusionMatrixMetric.Result randomResult() {
3534
return new ConfusionMatrixMetric.Result(
3635
Stream
3736
.generate(() -> randomConfusionMatrix())

0 commit comments

Comments
 (0)