Skip to content

Commit fcd0a30

Browse files
gwbrowngwbrown
committed
Merge branch 'master' into slm-retention
2 parents f42a478 + 5493613 commit fcd0a30

File tree

176 files changed

+4151
-1182
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

176 files changed

+4151
-1182
lines changed

‎build.gradle

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,8 +176,8 @@ task verifyVersions {
176176
* after the backport of the backcompat code is complete.
177177
*/
178178

179-
boolean bwc_tests_enabled = true
180-
final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
179+
boolean bwc_tests_enabled = false
180+
final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/45798" /* place a PR link here when committing bwc changes */
181181
if (bwc_tests_enabled == false) {
182182
if (bwc_tests_disabled_issue.isEmpty()) {
183183
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,12 @@ public void apply(Project project) {
9696
TaskProvider<Copy> copyUpgradeTask = configureCopyUpgradeTask(project, upgradeVersion, upgradeDir);
9797
TaskProvider<Copy> copyPluginsTask = configureCopyPluginsTask(project, pluginsDir);
9898

99-
Map<String, TaskProvider<?>> batsTests = new HashMap<>();
99+
TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
100100
for (ElasticsearchDistribution distribution : distributions) {
101-
configureDistroTest(project, distribution);
101+
TaskProvider<?> destructiveTask = configureDistroTest(project, distribution);
102+
destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
102103
}
104+
Map<String, TaskProvider<?>> batsTests = new HashMap<>();
103105
batsTests.put("bats oss", configureBatsTest(project, "oss", distributionsDir, copyDistributionsTask));
104106
batsTests.put("bats default", configureBatsTest(project, "default", distributionsDir, copyDistributionsTask));
105107
configureBatsTest(project, "plugins",distributionsDir, copyDistributionsTask, copyPluginsTask).configure(t ->
@@ -127,7 +129,6 @@ public void apply(Project project) {
127129
}
128130
}
129131

130-
131132
batsTests.forEach((desc, task) -> {
132133
configureVMWrapperTask(vmProject, desc, task.getName(), vmDependencies).configure(t -> {
133134
t.setProgressHandler(new BatsProgressLogger(project.getLogger()));

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
107107
"is a pre-release version of Elasticsearch",
108108
"max virtual memory areas vm.max_map_count"
109109
);
110+
private static final String HOSTNAME_OVERRIDE = "LinuxDarwinHostname";
111+
private static final String COMPUTERNAME_OVERRIDE = "WindowsComputername";
110112

111113
private final String path;
112114
private final String name;
@@ -643,6 +645,10 @@ private Map<String, String> getESEnvironment() {
643645
// Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR
644646
defaultEnv.put("TMP", tmpDir.toString());
645647

648+
// Override the system hostname variables for testing
649+
defaultEnv.put("HOSTNAME", HOSTNAME_OVERRIDE);
650+
defaultEnv.put("COMPUTERNAME", COMPUTERNAME_OVERRIDE);
651+
646652
Set<String> commonKeys = new HashSet<>(environment.keySet());
647653
commonKeys.retainAll(defaultEnv.keySet());
648654
if (commonKeys.isEmpty() == false) {

‎client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -554,7 +554,8 @@ private static Request prepareReindexRequest(ReindexRequest reindexRequest, bool
554554
.withRefresh(reindexRequest.isRefresh())
555555
.withTimeout(reindexRequest.getTimeout())
556556
.withWaitForActiveShards(reindexRequest.getWaitForActiveShards())
557-
.withRequestsPerSecond(reindexRequest.getRequestsPerSecond());
557+
.withRequestsPerSecond(reindexRequest.getRequestsPerSecond())
558+
.withSlices(reindexRequest.getSlices());
558559

559560
if (reindexRequest.getScrollTime() != null) {
560561
params.putParam("scroll", reindexRequest.getScrollTime());
@@ -897,6 +898,10 @@ Params withRouting(String routing) {
897898
return putParam("routing", routing);
898899
}
899900

901+
Params withSlices(int slices) {
902+
return putParam("slices", String.valueOf(slices));
903+
}
904+
900905
Params withStoredFields(String[] storedFields) {
901906
if (storedFields != null && storedFields.length > 0) {
902907
return putParam("stored_fields", String.join(",", storedFields));

‎client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
package org.elasticsearch.client;
2121

2222
import org.elasticsearch.action.ActionListener;
23+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest;
24+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
2325
import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
2426
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
2527
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
@@ -170,6 +172,35 @@ public void verifyRepositoryAsync(VerifyRepositoryRequest verifyRepositoryReques
170172
VerifyRepositoryResponse::fromXContent, listener, emptySet());
171173
}
172174

175+
/**
176+
* Cleans up a snapshot repository.
177+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
178+
* API on elastic.co</a>
179+
* @param cleanupRepositoryRequest the request
180+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
181+
* @return the response
182+
* @throws IOException in case there is a problem sending the request or parsing back the response
183+
*/
184+
public CleanupRepositoryResponse cleanupRepository(CleanupRepositoryRequest cleanupRepositoryRequest, RequestOptions options)
185+
throws IOException {
186+
return restHighLevelClient.performRequestAndParseEntity(cleanupRepositoryRequest, SnapshotRequestConverters::cleanupRepository,
187+
options, CleanupRepositoryResponse::fromXContent, emptySet());
188+
}
189+
190+
/**
191+
* Asynchronously cleans up a snapshot repository.
192+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
193+
* API on elastic.co</a>
194+
* @param cleanupRepositoryRequest the request
195+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
196+
* @param listener the listener to be notified upon request completion
197+
*/
198+
public void cleanupRepositoryAsync(CleanupRepositoryRequest cleanupRepositoryRequest, RequestOptions options,
199+
ActionListener<CleanupRepositoryResponse> listener) {
200+
restHighLevelClient.performRequestAsyncAndParseEntity(cleanupRepositoryRequest, SnapshotRequestConverters::cleanupRepository,
201+
options, CleanupRepositoryResponse::fromXContent, listener, emptySet());
202+
}
203+
173204
/**
174205
* Creates a snapshot.
175206
* <p>

‎client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.http.client.methods.HttpGet;
2424
import org.apache.http.client.methods.HttpPost;
2525
import org.apache.http.client.methods.HttpPut;
26+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest;
2627
import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
2728
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
2829
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
@@ -94,6 +95,20 @@ static Request verifyRepository(VerifyRepositoryRequest verifyRepositoryRequest)
9495
return request;
9596
}
9697

98+
static Request cleanupRepository(CleanupRepositoryRequest cleanupRepositoryRequest) {
99+
String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_snapshot")
100+
.addPathPart(cleanupRepositoryRequest.name())
101+
.addPathPartAsIs("_cleanup")
102+
.build();
103+
Request request = new Request(HttpPost.METHOD_NAME, endpoint);
104+
105+
RequestConverters.Params parameters = new RequestConverters.Params();
106+
parameters.withMasterTimeout(cleanupRepositoryRequest.masterNodeTimeout());
107+
parameters.withTimeout(cleanupRepositoryRequest.timeout());
108+
request.addParameters(parameters.asMap());
109+
return request;
110+
}
111+
97112
static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throws IOException {
98113
String endpoint = new RequestConverters.EndpointBuilder().addPathPart("_snapshot")
99114
.addPathPart(createSnapshotRequest.repository())

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateMemoryUsageResponse.java

Lines changed: 22 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,8 @@
3535

3636
public class EstimateMemoryUsageResponse implements ToXContentObject {
3737

38-
public static final ParseField EXPECTED_MEMORY_USAGE_WITH_ONE_PARTITION =
39-
new ParseField("expected_memory_usage_with_one_partition");
40-
public static final ParseField EXPECTED_MEMORY_USAGE_WITH_MAX_PARTITIONS =
41-
new ParseField("expected_memory_usage_with_max_partitions");
38+
public static final ParseField EXPECTED_MEMORY_WITHOUT_DISK = new ParseField("expected_memory_without_disk");
39+
public static final ParseField EXPECTED_MEMORY_WITH_DISK = new ParseField("expected_memory_with_disk");
4240

4341
static final ConstructingObjectParser<EstimateMemoryUsageResponse, Void> PARSER =
4442
new ConstructingObjectParser<>(
@@ -49,47 +47,44 @@ public class EstimateMemoryUsageResponse implements ToXContentObject {
4947
static {
5048
PARSER.declareField(
5149
optionalConstructorArg(),
52-
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), EXPECTED_MEMORY_USAGE_WITH_ONE_PARTITION.getPreferredName()),
53-
EXPECTED_MEMORY_USAGE_WITH_ONE_PARTITION,
50+
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), EXPECTED_MEMORY_WITHOUT_DISK.getPreferredName()),
51+
EXPECTED_MEMORY_WITHOUT_DISK,
5452
ObjectParser.ValueType.VALUE);
5553
PARSER.declareField(
5654
optionalConstructorArg(),
57-
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), EXPECTED_MEMORY_USAGE_WITH_MAX_PARTITIONS.getPreferredName()),
58-
EXPECTED_MEMORY_USAGE_WITH_MAX_PARTITIONS,
55+
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), EXPECTED_MEMORY_WITH_DISK.getPreferredName()),
56+
EXPECTED_MEMORY_WITH_DISK,
5957
ObjectParser.ValueType.VALUE);
6058
}
6159

6260
public static EstimateMemoryUsageResponse fromXContent(XContentParser parser) {
6361
return PARSER.apply(parser, null);
6462
}
6563

66-
private final ByteSizeValue expectedMemoryUsageWithOnePartition;
67-
private final ByteSizeValue expectedMemoryUsageWithMaxPartitions;
64+
private final ByteSizeValue expectedMemoryWithoutDisk;
65+
private final ByteSizeValue expectedMemoryWithDisk;
6866

69-
public EstimateMemoryUsageResponse(@Nullable ByteSizeValue expectedMemoryUsageWithOnePartition,
70-
@Nullable ByteSizeValue expectedMemoryUsageWithMaxPartitions) {
71-
this.expectedMemoryUsageWithOnePartition = expectedMemoryUsageWithOnePartition;
72-
this.expectedMemoryUsageWithMaxPartitions = expectedMemoryUsageWithMaxPartitions;
67+
public EstimateMemoryUsageResponse(@Nullable ByteSizeValue expectedMemoryWithoutDisk, @Nullable ByteSizeValue expectedMemoryWithDisk) {
68+
this.expectedMemoryWithoutDisk = expectedMemoryWithoutDisk;
69+
this.expectedMemoryWithDisk = expectedMemoryWithDisk;
7370
}
7471

75-
public ByteSizeValue getExpectedMemoryUsageWithOnePartition() {
76-
return expectedMemoryUsageWithOnePartition;
72+
public ByteSizeValue getExpectedMemoryWithoutDisk() {
73+
return expectedMemoryWithoutDisk;
7774
}
7875

79-
public ByteSizeValue getExpectedMemoryUsageWithMaxPartitions() {
80-
return expectedMemoryUsageWithMaxPartitions;
76+
public ByteSizeValue getExpectedMemoryWithDisk() {
77+
return expectedMemoryWithDisk;
8178
}
8279

8380
@Override
8481
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
8582
builder.startObject();
86-
if (expectedMemoryUsageWithOnePartition != null) {
87-
builder.field(
88-
EXPECTED_MEMORY_USAGE_WITH_ONE_PARTITION.getPreferredName(), expectedMemoryUsageWithOnePartition.getStringRep());
83+
if (expectedMemoryWithoutDisk != null) {
84+
builder.field(EXPECTED_MEMORY_WITHOUT_DISK.getPreferredName(), expectedMemoryWithoutDisk.getStringRep());
8985
}
90-
if (expectedMemoryUsageWithMaxPartitions != null) {
91-
builder.field(
92-
EXPECTED_MEMORY_USAGE_WITH_MAX_PARTITIONS.getPreferredName(), expectedMemoryUsageWithMaxPartitions.getStringRep());
86+
if (expectedMemoryWithDisk != null) {
87+
builder.field(EXPECTED_MEMORY_WITH_DISK.getPreferredName(), expectedMemoryWithDisk.getStringRep());
9388
}
9489
builder.endObject();
9590
return builder;
@@ -105,12 +100,12 @@ public boolean equals(Object other) {
105100
}
106101

107102
EstimateMemoryUsageResponse that = (EstimateMemoryUsageResponse) other;
108-
return Objects.equals(expectedMemoryUsageWithOnePartition, that.expectedMemoryUsageWithOnePartition)
109-
&& Objects.equals(expectedMemoryUsageWithMaxPartitions, that.expectedMemoryUsageWithMaxPartitions);
103+
return Objects.equals(expectedMemoryWithoutDisk, that.expectedMemoryWithoutDisk)
104+
&& Objects.equals(expectedMemoryWithDisk, that.expectedMemoryWithDisk);
110105
}
111106

112107
@Override
113108
public int hashCode() {
114-
return Objects.hash(expectedMemoryUsageWithOnePartition, expectedMemoryUsageWithMaxPartitions);
109+
return Objects.hash(expectedMemoryWithoutDisk, expectedMemoryWithDisk);
115110
}
116111
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ public static Builder builder() {
4848
}
4949

5050
private static final ParseField ID = new ParseField("id");
51+
private static final ParseField DESCRIPTION = new ParseField("description");
5152
private static final ParseField SOURCE = new ParseField("source");
5253
private static final ParseField DEST = new ParseField("dest");
5354
private static final ParseField ANALYSIS = new ParseField("analysis");
@@ -60,6 +61,7 @@ public static Builder builder() {
6061

6162
static {
6263
PARSER.declareString(Builder::setId, ID);
64+
PARSER.declareString(Builder::setDescription, DESCRIPTION);
6365
PARSER.declareObject(Builder::setSource, (p, c) -> DataFrameAnalyticsSource.fromXContent(p), SOURCE);
6466
PARSER.declareObject(Builder::setDest, (p, c) -> DataFrameAnalyticsDest.fromXContent(p), DEST);
6567
PARSER.declareObject(Builder::setAnalysis, (p, c) -> parseAnalysis(p), ANALYSIS);
@@ -95,6 +97,7 @@ private static DataFrameAnalysis parseAnalysis(XContentParser parser) throws IOE
9597
}
9698

9799
private final String id;
100+
private final String description;
98101
private final DataFrameAnalyticsSource source;
99102
private final DataFrameAnalyticsDest dest;
100103
private final DataFrameAnalysis analysis;
@@ -103,10 +106,12 @@ private static DataFrameAnalysis parseAnalysis(XContentParser parser) throws IOE
103106
private final Instant createTime;
104107
private final Version version;
105108

106-
private DataFrameAnalyticsConfig(@Nullable String id, @Nullable DataFrameAnalyticsSource source, @Nullable DataFrameAnalyticsDest dest,
107-
@Nullable DataFrameAnalysis analysis, @Nullable FetchSourceContext analyzedFields,
108-
@Nullable ByteSizeValue modelMemoryLimit, @Nullable Instant createTime, @Nullable Version version) {
109+
private DataFrameAnalyticsConfig(@Nullable String id, @Nullable String description, @Nullable DataFrameAnalyticsSource source,
110+
@Nullable DataFrameAnalyticsDest dest, @Nullable DataFrameAnalysis analysis,
111+
@Nullable FetchSourceContext analyzedFields, @Nullable ByteSizeValue modelMemoryLimit,
112+
@Nullable Instant createTime, @Nullable Version version) {
109113
this.id = id;
114+
this.description = description;
110115
this.source = source;
111116
this.dest = dest;
112117
this.analysis = analysis;
@@ -120,6 +125,10 @@ public String getId() {
120125
return id;
121126
}
122127

128+
public String getDescription() {
129+
return description;
130+
}
131+
123132
public DataFrameAnalyticsSource getSource() {
124133
return source;
125134
}
@@ -154,6 +163,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
154163
if (id != null) {
155164
builder.field(ID.getPreferredName(), id);
156165
}
166+
if (description != null) {
167+
builder.field(DESCRIPTION.getPreferredName(), description);
168+
}
157169
if (source != null) {
158170
builder.field(SOURCE.getPreferredName(), source);
159171
}
@@ -189,6 +201,7 @@ public boolean equals(Object o) {
189201

190202
DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o;
191203
return Objects.equals(id, other.id)
204+
&& Objects.equals(description, other.description)
192205
&& Objects.equals(source, other.source)
193206
&& Objects.equals(dest, other.dest)
194207
&& Objects.equals(analysis, other.analysis)
@@ -200,7 +213,7 @@ public boolean equals(Object o) {
200213

201214
@Override
202215
public int hashCode() {
203-
return Objects.hash(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
216+
return Objects.hash(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
204217
}
205218

206219
@Override
@@ -211,6 +224,7 @@ public String toString() {
211224
public static class Builder {
212225

213226
private String id;
227+
private String description;
214228
private DataFrameAnalyticsSource source;
215229
private DataFrameAnalyticsDest dest;
216230
private DataFrameAnalysis analysis;
@@ -226,6 +240,11 @@ public Builder setId(String id) {
226240
return this;
227241
}
228242

243+
public Builder setDescription(String description) {
244+
this.description = description;
245+
return this;
246+
}
247+
229248
public Builder setSource(DataFrameAnalyticsSource source) {
230249
this.source = Objects.requireNonNull(source);
231250
return this;
@@ -262,7 +281,8 @@ public Builder setVersion(Version version) {
262281
}
263282

264283
public DataFrameAnalyticsConfig build() {
265-
return new DataFrameAnalyticsConfig(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
284+
return new DataFrameAnalyticsConfig(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime,
285+
version);
266286
}
267287
}
268288
}

0 commit comments

Comments
 (0)