Skip to content

Commit 935f0df

Browse files
henningandersenhenningandersen
committed
Merge remote-tracking branch 'origin/master' into fix_search_allow_partial_results_false
2 parents a790db8 + c6a25a8 commit 935f0df

File tree

43 files changed

+1338
-72
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1338
-72
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
package org.elasticsearch.client;
2121

2222
import org.elasticsearch.action.ActionListener;
23+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest;
24+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
2325
import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
2426
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
2527
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
@@ -170,6 +172,35 @@ public void verifyRepositoryAsync(VerifyRepositoryRequest verifyRepositoryReques
170172
VerifyRepositoryResponse::fromXContent, listener, emptySet());
171173
}
172174

175+
/**
176+
* Cleans up a snapshot repository.
177+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
178+
* API on elastic.co</a>
179+
* @param cleanupRepositoryRequest the request
180+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
181+
* @return the response
182+
* @throws IOException in case there is a problem sending the request or parsing back the response
183+
*/
184+
public CleanupRepositoryResponse cleanupRepository(CleanupRepositoryRequest cleanupRepositoryRequest, RequestOptions options)
185+
throws IOException {
186+
return restHighLevelClient.performRequestAndParseEntity(cleanupRepositoryRequest, SnapshotRequestConverters::cleanupRepository,
187+
options, CleanupRepositoryResponse::fromXContent, emptySet());
188+
}
189+
190+
/**
191+
* Asynchronously cleans up a snapshot repository.
192+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
193+
* API on elastic.co</a>
194+
* @param cleanupRepositoryRequest the request
195+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
196+
* @param listener the listener to be notified upon request completion
197+
*/
198+
public void cleanupRepositoryAsync(CleanupRepositoryRequest cleanupRepositoryRequest, RequestOptions options,
199+
ActionListener<CleanupRepositoryResponse> listener) {
200+
restHighLevelClient.performRequestAsyncAndParseEntity(cleanupRepositoryRequest, SnapshotRequestConverters::cleanupRepository,
201+
options, CleanupRepositoryResponse::fromXContent, listener, emptySet());
202+
}
203+
173204
/**
174205
* Creates a snapshot.
175206
* <p>

‎client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.http.client.methods.HttpGet;
2424
import org.apache.http.client.methods.HttpPost;
2525
import org.apache.http.client.methods.HttpPut;
26+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest;
2627
import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
2728
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
2829
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
@@ -94,6 +95,20 @@ static Request verifyRepository(VerifyRepositoryRequest verifyRepositoryRequest)
9495
return request;
9596
}
9697

98+
static Request cleanupRepository(CleanupRepositoryRequest cleanupRepositoryRequest) {
99+
String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_snapshot")
100+
.addPathPart(cleanupRepositoryRequest.name())
101+
.addPathPartAsIs("_cleanup")
102+
.build();
103+
Request request = new Request(HttpPost.METHOD_NAME, endpoint);
104+
105+
RequestConverters.Params parameters = new RequestConverters.Params();
106+
parameters.withMasterTimeout(cleanupRepositoryRequest.masterNodeTimeout());
107+
parameters.withTimeout(cleanupRepositoryRequest.timeout());
108+
request.addParameters(parameters.asMap());
109+
return request;
110+
}
111+
97112
static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throws IOException {
98113
String endpoint = new RequestConverters.EndpointBuilder().addPathPart("_snapshot")
99114
.addPathPart(createSnapshotRequest.repository())

‎client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
package org.elasticsearch.client;
2121

2222
import org.elasticsearch.ElasticsearchException;
23+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest;
24+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
2325
import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
2426
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
2527
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
@@ -133,6 +135,17 @@ public void testVerifyRepository() throws IOException {
133135
assertThat(response.getNodes().size(), equalTo(1));
134136
}
135137

138+
public void testCleanupRepository() throws IOException {
139+
AcknowledgedResponse putRepositoryResponse = createTestRepository("test", FsRepository.TYPE, "{\"location\": \".\"}");
140+
assertTrue(putRepositoryResponse.isAcknowledged());
141+
142+
CleanupRepositoryRequest request = new CleanupRepositoryRequest("test");
143+
CleanupRepositoryResponse response = execute(request, highLevelClient().snapshot()::cleanupRepository,
144+
highLevelClient().snapshot()::cleanupRepositoryAsync);
145+
assertThat(response.result().bytes(), equalTo(0L));
146+
assertThat(response.result().blobs(), equalTo(0L));
147+
}
148+
136149
public void testCreateSnapshot() throws IOException {
137150
String repository = "test_repository";
138151
assertTrue(createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}").isAcknowledged());
@@ -317,4 +330,4 @@ private static Map<String, Object> randomUserMetadata() {
317330
}
318331
return metadata;
319332
}
320-
}
333+
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.elasticsearch.client.ml.DeleteJobRequest;
4949
import org.elasticsearch.client.ml.DeleteJobResponse;
5050
import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
51+
import org.elasticsearch.client.ml.EstimateMemoryUsageResponse;
5152
import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
5253
import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
5354
import org.elasticsearch.client.ml.FindFileStructureRequest;
@@ -194,11 +195,13 @@
194195
import java.util.concurrent.TimeUnit;
195196
import java.util.stream.Collectors;
196197

198+
import static org.hamcrest.Matchers.allOf;
197199
import static org.hamcrest.Matchers.closeTo;
198200
import static org.hamcrest.Matchers.containsInAnyOrder;
199201
import static org.hamcrest.Matchers.equalTo;
200202
import static org.hamcrest.Matchers.greaterThan;
201203
import static org.hamcrest.Matchers.hasSize;
204+
import static org.hamcrest.Matchers.lessThan;
202205
import static org.hamcrest.core.Is.is;
203206

204207
public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
@@ -3262,6 +3265,72 @@ public void onFailure(Exception e) {
32623265
}
32633266
}
32643267

3268+
public void testEstimateMemoryUsage() throws Exception {
3269+
createIndex("estimate-test-source-index");
3270+
BulkRequest bulkRequest =
3271+
new BulkRequest("estimate-test-source-index")
3272+
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
3273+
for (int i = 0; i < 10; ++i) {
3274+
bulkRequest.add(new IndexRequest().source(XContentType.JSON, "timestamp", 123456789L, "total", 10L));
3275+
}
3276+
RestHighLevelClient client = highLevelClient();
3277+
client.bulk(bulkRequest, RequestOptions.DEFAULT);
3278+
{
3279+
// tag::estimate-memory-usage-request
3280+
DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder()
3281+
.setSource(DataFrameAnalyticsSource.builder().setIndex("estimate-test-source-index").build())
3282+
.setAnalysis(OutlierDetection.createDefault())
3283+
.build();
3284+
PutDataFrameAnalyticsRequest request = new PutDataFrameAnalyticsRequest(config); // <1>
3285+
// end::estimate-memory-usage-request
3286+
3287+
// tag::estimate-memory-usage-execute
3288+
EstimateMemoryUsageResponse response = client.machineLearning().estimateMemoryUsage(request, RequestOptions.DEFAULT);
3289+
// end::estimate-memory-usage-execute
3290+
3291+
// tag::estimate-memory-usage-response
3292+
ByteSizeValue expectedMemoryWithoutDisk = response.getExpectedMemoryWithoutDisk(); // <1>
3293+
ByteSizeValue expectedMemoryWithDisk = response.getExpectedMemoryWithDisk(); // <2>
3294+
// end::estimate-memory-usage-response
3295+
3296+
// We are pretty liberal here as this test does not aim at verifying concrete numbers but rather end-to-end user workflow.
3297+
ByteSizeValue lowerBound = new ByteSizeValue(1, ByteSizeUnit.KB);
3298+
ByteSizeValue upperBound = new ByteSizeValue(1, ByteSizeUnit.GB);
3299+
assertThat(expectedMemoryWithoutDisk, allOf(greaterThan(lowerBound), lessThan(upperBound)));
3300+
assertThat(expectedMemoryWithDisk, allOf(greaterThan(lowerBound), lessThan(upperBound)));
3301+
}
3302+
{
3303+
DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder()
3304+
.setSource(DataFrameAnalyticsSource.builder().setIndex("estimate-test-source-index").build())
3305+
.setAnalysis(OutlierDetection.createDefault())
3306+
.build();
3307+
PutDataFrameAnalyticsRequest request = new PutDataFrameAnalyticsRequest(config);
3308+
// tag::estimate-memory-usage-execute-listener
3309+
ActionListener<EstimateMemoryUsageResponse> listener = new ActionListener<>() {
3310+
@Override
3311+
public void onResponse(EstimateMemoryUsageResponse response) {
3312+
// <1>
3313+
}
3314+
3315+
@Override
3316+
public void onFailure(Exception e) {
3317+
// <2>
3318+
}
3319+
};
3320+
// end::estimate-memory-usage-execute-listener
3321+
3322+
// Replace the empty listener by a blocking listener in test
3323+
final CountDownLatch latch = new CountDownLatch(1);
3324+
listener = new LatchedActionListener<>(listener, latch);
3325+
3326+
// tag::estimate-memory-usage-execute-async
3327+
client.machineLearning().estimateMemoryUsageAsync(request, RequestOptions.DEFAULT, listener); // <1>
3328+
// end::estimate-memory-usage-execute-async
3329+
3330+
assertTrue(latch.await(30L, TimeUnit.SECONDS));
3331+
}
3332+
}
3333+
32653334
public void testCreateFilter() throws Exception {
32663335
RestHighLevelClient client = highLevelClient();
32673336
{

‎docs/java-rest/high-level/ml/estimate-memory-usage.asciidoc

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
--
2+
:api: estimate-memory-usage
3+
:request: PutDataFrameAnalyticsRequest
4+
:response: EstimateMemoryUsageResponse
5+
--
6+
[id="{upid}-{api}"]
7+
=== Estimate memory usage API
8+
9+
The Estimate memory usage API is used to estimate memory usage of {dfanalytics}.
10+
Estimation results can be used when deciding the appropriate value for `model_memory_limit` setting later on.
11+
12+
The API accepts an +{request}+ object and returns an +{response}+.
13+
14+
[id="{upid}-{api}-request"]
15+
==== Estimate memory usage Request
16+
17+
["source","java",subs="attributes,callouts,macros"]
18+
--------------------------------------------------
19+
include-tagged::{doc-tests-file}[{api}-request]
20+
--------------------------------------------------
21+
<1> Constructing a new request containing a {dataframe-analytics-config} for which memory usage estimation should be performed
22+
23+
include::../execution.asciidoc[]
24+
25+
[id="{upid}-{api}-response"]
26+
==== Response
27+
28+
The returned +{response}+ contains the memory usage estimates.
29+
30+
["source","java",subs="attributes,callouts,macros"]
31+
--------------------------------------------------
32+
include-tagged::{doc-tests-file}[{api}-response]
33+
--------------------------------------------------
34+
<1> Estimated memory usage under the assumption that the whole {dfanalytics} should happen in memory (i.e. without overflowing to disk).
35+
<2> Estimated memory usage under the assumption that overflowing to disk is allowed during {dfanalytics}.

‎docs/java-rest/high-level/supported-apis.asciidoc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,7 @@ The Java High Level REST Client supports the following Machine Learning APIs:
295295
* <<{upid}-start-data-frame-analytics>>
296296
* <<{upid}-stop-data-frame-analytics>>
297297
* <<{upid}-evaluate-data-frame>>
298+
* <<{upid}-estimate-memory-usage>>
298299
* <<{upid}-put-filter>>
299300
* <<{upid}-get-filters>>
300301
* <<{upid}-update-filter>>
@@ -346,6 +347,7 @@ include::ml/delete-data-frame-analytics.asciidoc[]
346347
include::ml/start-data-frame-analytics.asciidoc[]
347348
include::ml/stop-data-frame-analytics.asciidoc[]
348349
include::ml/evaluate-data-frame.asciidoc[]
350+
include::ml/estimate-memory-usage.asciidoc[]
349351
include::ml/put-filter.asciidoc[]
350352
include::ml/get-filters.asciidoc[]
351353
include::ml/update-filter.asciidoc[]

‎docs/reference/ml/df-analytics/apis/estimate-memory-usage-dfanalytics.asciidoc

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,14 @@ Serves as an advice on how to set `model_memory_limit` when creating {dfanalytic
4242
[[ml-estimate-memory-usage-dfanalytics-results]]
4343
==== {api-response-body-title}
4444

45-
`expected_memory_usage_with_one_partition`::
45+
`expected_memory_without_disk`::
4646
(string) Estimated memory usage under the assumption that the whole {dfanalytics} should happen in memory
4747
(i.e. without overflowing to disk).
4848

49-
`expected_memory_usage_with_max_partitions`::
49+
`expected_memory_with_disk`::
5050
(string) Estimated memory usage under the assumption that overflowing to disk is allowed during {dfanalytics}.
51-
`expected_memory_usage_with_max_partitions` is usually smaller than `expected_memory_usage_with_one_partition`
52-
as using disk allows to limit the main memory needed to perform {dfanalytics}.
51+
`expected_memory_with_disk` is usually smaller than `expected_memory_without_disk` as using disk allows to
52+
limit the main memory needed to perform {dfanalytics}.
5353

5454
[[ml-estimate-memory-usage-dfanalytics-example]]
5555
==== {api-examples-title}
@@ -76,8 +76,8 @@ The API returns the following results:
7676
[source,js]
7777
----
7878
{
79-
"expected_memory_usage_with_one_partition": "128MB",
80-
"expected_memory_usage_with_max_partitions": "32MB"
79+
"expected_memory_without_disk": "128MB",
80+
"expected_memory_with_disk": "32MB"
8181
}
8282
----
8383
// TESTRESPONSE

‎docs/reference/modules/snapshots.asciidoc

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,42 @@ POST /_snapshot/my_unverified_backup/_verify
332332

333333
It returns a list of nodes where repository was successfully verified or an error message if verification process failed.
334334

335+
[float]
336+
===== Repository Cleanup
337+
Repositories can over time accumulate data that is not referenced by any existing snapshot. This is a result of the data safety guarantees
338+
the snapshot functionality provides in failure scenarios during snapshot creation and the decentralized nature of the snapshot creation
339+
process. This unreferenced data does in no way negatively impact the performance or safety of a snapshot repository but leads to higher
340+
than necessary storage use. In order to clean up this unreferenced data, users can call the cleanup endpoint for a repository which will
341+
trigger a complete accounting of the repositories contents and subsequent deletion of all unreferenced data that was found.
342+
343+
[source,js]
344+
-----------------------------------
345+
POST /_snapshot/my_repository/_cleanup
346+
-----------------------------------
347+
// CONSOLE
348+
// TEST[continued]
349+
350+
The response to a cleanup request looks as follows:
351+
352+
[source,js]
353+
--------------------------------------------------
354+
{
355+
"results": {
356+
"deleted_bytes": 20,
357+
"deleted_blobs": 5
358+
}
359+
}
360+
--------------------------------------------------
361+
// TESTRESPONSE
362+
363+
Depending on the concrete repository implementation the numbers shown for bytes free as well as the number of blobs removed will either
364+
be an approximation or an exact result. Any non-zero value for the number of blobs removed implies that unreferenced blobs were found and
365+
subsequently cleaned up.
366+
367+
Please note that most of the cleanup operations executed by this endpoint are automatically executed when deleting any snapshot from a
368+
repository. If you regularly delete snapshots, you will in most cases not get any or only minor space savings from using this functionality
369+
and should lower your frequency of invoking it accordingly.
370+
335371
[float]
336372
[[snapshots-take-snapshot]]
337373
=== Snapshot

‎modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.common.blobstore.BlobContainer;
2424
import org.elasticsearch.common.blobstore.BlobMetaData;
2525
import org.elasticsearch.common.blobstore.BlobPath;
26+
import org.elasticsearch.common.blobstore.DeleteResult;
2627
import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
2728

2829
import java.io.BufferedInputStream;
@@ -97,7 +98,7 @@ public void deleteBlob(String blobName) throws IOException {
9798
}
9899

99100
@Override
100-
public void delete() {
101+
public DeleteResult delete() {
101102
throw new UnsupportedOperationException("URL repository is read only");
102103
}
103104

‎plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.elasticsearch.common.blobstore.BlobContainer;
3232
import org.elasticsearch.common.blobstore.BlobMetaData;
3333
import org.elasticsearch.common.blobstore.BlobPath;
34+
import org.elasticsearch.common.blobstore.DeleteResult;
3435
import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
3536
import org.elasticsearch.threadpool.ThreadPool;
3637

@@ -126,9 +127,9 @@ public void deleteBlob(String blobName) throws IOException {
126127
}
127128

128129
@Override
129-
public void delete() throws IOException {
130+
public DeleteResult delete() throws IOException {
130131
try {
131-
blobStore.deleteBlobDirectory(keyPath, threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME));
132+
return blobStore.deleteBlobDirectory(keyPath, threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME));
132133
} catch (URISyntaxException | StorageException e) {
133134
throw new IOException(e);
134135
}

‎plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121

2222
import com.microsoft.azure.storage.LocationMode;
2323
import com.microsoft.azure.storage.StorageException;
24-
2524
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
2625
import org.elasticsearch.common.blobstore.BlobContainer;
2726
import org.elasticsearch.common.blobstore.BlobMetaData;
2827
import org.elasticsearch.common.blobstore.BlobPath;
2928
import org.elasticsearch.common.blobstore.BlobStore;
29+
import org.elasticsearch.common.blobstore.DeleteResult;
3030
import org.elasticsearch.repositories.azure.AzureRepository.Repository;
3131
import org.elasticsearch.threadpool.ThreadPool;
3232

@@ -92,8 +92,9 @@ public void deleteBlob(String blob) throws URISyntaxException, StorageException
9292
service.deleteBlob(clientName, container, blob);
9393
}
9494

95-
public void deleteBlobDirectory(String path, Executor executor) throws URISyntaxException, StorageException, IOException {
96-
service.deleteBlobDirectory(clientName, container, path, executor);
95+
public DeleteResult deleteBlobDirectory(String path, Executor executor)
96+
throws URISyntaxException, StorageException, IOException {
97+
return service.deleteBlobDirectory(clientName, container, path, executor);
9798
}
9899

99100
public InputStream getInputStream(String blob) throws URISyntaxException, StorageException, IOException {

0 commit comments

Comments
 (0)