Skip to content

Commit 34daff9

Browse files
jasontedorjasontedor
committed
Merge remote-tracking branch 'elastic/master' into build-flavor-type-leniency
* elastic/master: SQL: Fix deserialisation issue of TimeProcessor (elastic#40776) Improve GCS docs for using keystore (elastic#40605) Add Restore Operation to SnapshotResiliencyTests (elastic#40634) Small refactorings to analysis components (elastic#40745) SQL: Fix display size for DATE/DATETIME (elastic#40669) add HLRC protocol tests for transform state and stats (elastic#40766) Inline TransportReplAction#registerRequestHandlers (elastic#40762) remove experimental label from search_as_you_type documentation (elastic#40744) Remove some abstractions from `TransportReplicationAction` (elastic#40706) Upgrade to latest build scan plugin (elastic#40702) Use default memory lock setting in testing (elastic#40730) Add Bulk Delete Api to BlobStore (elastic#40322) Remove yaml skips older than 7.0 (elastic#40183) Docs: Move id in the java-api (elastic#40748)
2 parents 82f2f2d + cfea348 commit 34daff9

File tree

187 files changed

+817
-1057
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

187 files changed

+817
-1057
lines changed

‎build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import org.gradle.util.DistributionLocator
2929
import org.gradle.plugins.ide.eclipse.model.SourceFolder
3030

3131
plugins {
32-
id 'com.gradle.build-scan' version '2.0.2'
32+
id 'com.gradle.build-scan' version '2.2.1'
3333
id 'base'
3434
}
3535
if (Boolean.valueOf(project.findProperty('org.elasticsearch.acceptScanTOS') ?: "false")) {

‎client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@ public abstract class IndexerJobStats {
4747
private final long indexFailures;
4848
private final long searchFailures;
4949

50-
public IndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations,
50+
public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocuments, long numInvocations,
5151
long indexTime, long searchTime, long indexTotal, long searchTotal, long indexFailures, long searchFailures) {
5252
this.numPages = numPages;
5353
this.numInputDocuments = numInputDocuments;
54-
this.numOuputDocuments = numOuputDocuments;
54+
this.numOuputDocuments = numOutputDocuments;
5555
this.numInvocations = numInvocations;
5656
this.indexTime = indexTime;
5757
this.indexTotal = indexTotal;

‎client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameIndexerTransformStats.java

+12-12
Original file line numberDiff line numberDiff line change
@@ -29,25 +29,25 @@
2929

3030
public class DataFrameIndexerTransformStats extends IndexerJobStats {
3131

32-
public static final ConstructingObjectParser<DataFrameIndexerTransformStats, Void> PARSER = new ConstructingObjectParser<>(
32+
public static final ConstructingObjectParser<DataFrameIndexerTransformStats, Void> LENIENT_PARSER = new ConstructingObjectParser<>(
3333
NAME, true, args -> new DataFrameIndexerTransformStats((long) args[0], (long) args[1], (long) args[2],
3434
(long) args[3], (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9]));
3535

3636
static {
37-
PARSER.declareLong(constructorArg(), NUM_PAGES);
38-
PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
39-
PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS);
40-
PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
41-
PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
42-
PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
43-
PARSER.declareLong(constructorArg(), INDEX_TOTAL);
44-
PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
45-
PARSER.declareLong(constructorArg(), INDEX_FAILURES);
46-
PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
37+
LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES);
38+
LENIENT_PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
39+
LENIENT_PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS);
40+
LENIENT_PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
41+
LENIENT_PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
42+
LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
43+
LENIENT_PARSER.declareLong(constructorArg(), INDEX_TOTAL);
44+
LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
45+
LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES);
46+
LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
4747
}
4848

4949
public static DataFrameIndexerTransformStats fromXContent(XContentParser parser) throws IOException {
50-
return PARSER.parse(parser, null);
50+
return LENIENT_PARSER.parse(parser, null);
5151
}
5252

5353
public DataFrameIndexerTransformStats(long numPages, long numInputDocuments, long numOuputDocuments,

‎docs/java-api/index.asciidoc

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
[[java-api]]
21
= Java API
32

43
include::../Versions.asciidoc[]
54

5+
[[java-api]]
66
[preface]
77
== Preface
88

‎docs/plugins/repository-gcs.asciidoc

+3-2
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,9 @@ A JSON service account file looks like this:
8888
----
8989
// NOTCONSOLE
9090

91-
To provide this file to the plugin, it must be stored in the {ref}/secure-settings.html[Elasticsearch keystore]. You must add a setting name of the form `gcs.client.NAME.credentials_file`, where `NAME`
92-
is the name of the client configuration for the repository. The implicit client
91+
To provide this file to the plugin, it must be stored in the {ref}/secure-settings.html[Elasticsearch keystore]. You must
92+
add a `file` setting with the name `gcs.client.NAME.credentials_file` using the `add-file` subcommand.
93+
`NAME` is the name of the client configuration for the repository. The implicit client
9394
name is `default`, but a different client name can be specified in the
9495
repository settings with the `client` key.
9596

‎docs/reference/mapping/types/search-as-you-type.asciidoc

-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
[[search-as-you-type]]
22
=== Search as you type datatype
33

4-
experimental[]
5-
64
The `search_as_you_type` field type is a text-like field that is optimized to
75
provide out-of-the-box support for queries that serve an as-you-type completion
86
use case. It creates a series of subfields that are analyzed to index terms

‎docs/reference/sql/functions/date-time.asciidoc

+4
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,10 @@ relative date/time filtering:
187187
include-tagged::{sql-specs}/docs/docs.csv-spec[filterNow]
188188
--------------------------------------------------
189189

190+
[IMPORTANT]
191+
Currently, Using a _precision_ greater than 3 doesn't make any difference to the output of the
192+
function as the maximum number of second fractional digits returned is 3 (milliseconds).
193+
190194
[[sql-functions-datetime-day]]
191195
==== `DAY_OF_MONTH/DOM/DAY`
192196

‎docs/reference/sql/language/data-types.asciidoc

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ s|SQL precision
2626
| <<keyword, `keyword`>> | keyword | VARCHAR | 32,766
2727
| <<text, `text`>> | text | VARCHAR | 2,147,483,647
2828
| <<binary, `binary`>> | binary | VARBINARY | 2,147,483,647
29-
| <<date, `date`>> | datetime | TIMESTAMP | 24
29+
| <<date, `date`>> | datetime | TIMESTAMP | 29
3030
| <<ip, `ip`>> | ip | VARCHAR | 39
3131

3232
4+h| Complex types
@@ -66,7 +66,7 @@ s|SQL type
6666
s|SQL precision
6767

6868

69-
| date | 24
69+
| date | 29
7070
| time | 18
7171
| interval_year | 7
7272
| interval_month | 7

‎plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ public void setup() {
6363
Supplier<QueryShardContext> queryShardContext = () -> {
6464
return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null);
6565
};
66-
parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), indexService.getIndexAnalyzers(),
67-
indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext);
66+
parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), indexService.xContentRegistry(),
67+
indexService.similarityService(), mapperRegistry, queryShardContext);
6868
}
6969

7070
@Override

‎plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java

+52
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
2424
import com.amazonaws.services.s3.model.AmazonS3Exception;
2525
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
26+
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
2627
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
2728
import com.amazonaws.services.s3.model.ObjectListing;
2829
import com.amazonaws.services.s3.model.ObjectMetadata;
@@ -56,6 +57,12 @@
5657

5758
class S3BlobContainer extends AbstractBlobContainer {
5859

60+
/**
61+
* Maximum number of deletes in a {@link DeleteObjectsRequest}.
62+
* @see <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html">S3 Documentation</a>.
63+
*/
64+
private static final int MAX_BULK_DELETES = 1000;
65+
5966
private final S3BlobStore blobStore;
6067
private final String keyPath;
6168

@@ -118,6 +125,51 @@ public void deleteBlob(String blobName) throws IOException {
118125
deleteBlobIgnoringIfNotExists(blobName);
119126
}
120127

128+
@Override
129+
public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) throws IOException {
130+
if (blobNames.isEmpty()) {
131+
return;
132+
}
133+
try (AmazonS3Reference clientReference = blobStore.clientReference()) {
134+
// S3 API only allows 1k blobs per delete so we split up the given blobs into requests of max. 1k deletes
135+
final List<DeleteObjectsRequest> deleteRequests = new ArrayList<>();
136+
final List<String> partition = new ArrayList<>();
137+
for (String blob : blobNames) {
138+
partition.add(buildKey(blob));
139+
if (partition.size() == MAX_BULK_DELETES ) {
140+
deleteRequests.add(bulkDelete(blobStore.bucket(), partition));
141+
partition.clear();
142+
}
143+
}
144+
if (partition.isEmpty() == false) {
145+
deleteRequests.add(bulkDelete(blobStore.bucket(), partition));
146+
}
147+
SocketAccess.doPrivilegedVoid(() -> {
148+
AmazonClientException aex = null;
149+
for (DeleteObjectsRequest deleteRequest : deleteRequests) {
150+
try {
151+
clientReference.client().deleteObjects(deleteRequest);
152+
} catch (AmazonClientException e) {
153+
if (aex == null) {
154+
aex = e;
155+
} else {
156+
aex.addSuppressed(e);
157+
}
158+
}
159+
}
160+
if (aex != null) {
161+
throw aex;
162+
}
163+
});
164+
} catch (final AmazonClientException e) {
165+
throw new IOException("Exception when deleting blobs [" + blobNames + "]", e);
166+
}
167+
}
168+
169+
private static DeleteObjectsRequest bulkDelete(String bucket, List<String> blobs) {
170+
return new DeleteObjectsRequest(bucket).withKeys(blobs.toArray(Strings.EMPTY_ARRAY)).withQuiet(true);
171+
}
172+
121173
@Override
122174
public void deleteBlobIgnoringIfNotExists(String blobName) throws IOException {
123175
try (AmazonS3Reference clientReference = blobStore.clientReference()) {

‎plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> bucke
324324
// Delete Multiple Objects
325325
//
326326
// https://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
327-
handlers.insert(nonAuthPath(HttpPost.METHOD_NAME, "/"), (request) -> {
327+
final RequestHandler bulkDeleteHandler = request -> {
328328
final List<String> deletes = new ArrayList<>();
329329
final List<String> errors = new ArrayList<>();
330330

@@ -344,7 +344,6 @@ private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> bucke
344344
if (closingOffset != -1) {
345345
offset = offset + startMarker.length();
346346
final String objectName = requestBody.substring(offset, closingOffset);
347-
348347
boolean found = false;
349348
for (Bucket bucket : buckets.values()) {
350349
if (bucket.objects.containsKey(objectName)) {
@@ -369,7 +368,9 @@ private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> bucke
369368
}
370369
}
371370
return newInternalError(request.getId(), "Something is wrong with this POST multiple deletes request");
372-
});
371+
};
372+
handlers.insert(nonAuthPath(HttpPost.METHOD_NAME, "/"), bulkDeleteHandler);
373+
handlers.insert(nonAuthPath(HttpPost.METHOD_NAME, "/{bucket}"), bulkDeleteHandler);
373374

374375
// non-authorized requests
375376

‎plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java

+1-5
Original file line numberDiff line numberDiff line change
@@ -158,11 +158,7 @@ public DeleteObjectsResult deleteObjects(DeleteObjectsRequest request) throws Sd
158158

159159
final List<DeleteObjectsResult.DeletedObject> deletions = new ArrayList<>();
160160
for (DeleteObjectsRequest.KeyVersion key : request.getKeys()) {
161-
if (blobs.remove(key.getKey()) == null) {
162-
AmazonS3Exception exception = new AmazonS3Exception("[" + key + "] does not exist.");
163-
exception.setStatusCode(404);
164-
throw exception;
165-
} else {
161+
if (blobs.remove(key.getKey()) != null) {
166162
DeleteObjectsResult.DeletedObject deletion = new DeleteObjectsResult.DeletedObject();
167163
deletion.setKey(key.getKey());
168164
deletions.add(deletion);

‎rest-api-spec/src/main/resources/rest-api-spec/test/bulk/10_basic.yml

+2-8
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
---
22
"Array of objects":
33

4-
- skip:
5-
version: " - 6.99.99"
6-
reason: types are required in requests before 7.0.0
4+
75

86
- do:
97
bulk:
@@ -29,9 +27,7 @@
2927
---
3028
"Empty _id":
3129

32-
- skip:
33-
version: " - 6.99.99"
34-
reason: types are required in requests before 7.0.0
30+
3531

3632
- do:
3733
bulk:
@@ -65,9 +61,7 @@
6561
"empty action":
6662

6763
- skip:
68-
version: " - 6.99.99"
6964
features: headers
70-
reason: types are required in requests before 7.0.0
7165

7266
- do:
7367
catch: /Malformed action\/metadata line \[3\], expected FIELD_NAME but found \[END_OBJECT\]/

‎rest-api-spec/src/main/resources/rest-api-spec/test/bulk/20_list_of_strings.yml

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
---
22
"List of strings":
33

4-
- skip:
5-
version: " - 6.99.99"
6-
reason: types are required in requests before 7.0.0
4+
75
- do:
86
bulk:
97
refresh: true

‎rest-api-spec/src/main/resources/rest-api-spec/test/bulk/30_big_string.yml

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
---
22
"One big string":
33

4-
- skip:
5-
version: " - 6.99.99"
6-
reason: types are required in requests before 7.0.0
4+
75
- do:
86
bulk:
97
refresh: true

‎rest-api-spec/src/main/resources/rest-api-spec/test/bulk/40_source.yml

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
---
22
"Source filtering":
33

4-
- skip:
5-
version: " - 6.99.99"
6-
reason: types are required in requests before 7.0.0
4+
75
- do:
86
index:
97
refresh: true

‎rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yml

+3-9
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
---
22
"refresh=true immediately makes changes are visible in search":
33

4-
- skip:
5-
version: " - 6.99.99"
6-
reason: types are required in requests before 7.0.0
4+
75
- do:
86
bulk:
97
refresh: true
@@ -21,9 +19,7 @@
2119
---
2220
"refresh=empty string immediately makes changes are visible in search":
2321

24-
- skip:
25-
version: " - 6.99.99"
26-
reason: types are required in requests before 7.0.0
22+
2723
- do:
2824
bulk:
2925
refresh: ""
@@ -42,9 +38,7 @@
4238
---
4339
"refresh=wait_for waits until changes are visible in search":
4440

45-
- skip:
46-
version: " - 6.99.99"
47-
reason: types are required in requests before 7.0.0
41+
4842
- do:
4943
bulk:
5044
refresh: wait_for

‎rest-api-spec/src/main/resources/rest-api-spec/test/bulk/70_mix_typeless_typeful.yml

-4
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
11
---
22
"bulk without types on an index that has types":
33

4-
- skip:
5-
version: " - 6.99.99"
6-
reason: Typeless APIs were introduced in 7.0.0
7-
84
- do:
95
indices.create: # not using include_type_name: false on purpose
106
include_type_name: true

‎rest-api-spec/src/main/resources/rest-api-spec/test/bulk/81_cas_with_types.yml

-4
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
11
---
22
"Compare And Swap Sequence Numbers":
33

4-
- skip:
5-
version: " - 6.6.99"
6-
reason: cas operations with sequence numbers was added in 6.7
7-
84
- do:
95
index:
106
index: test_1

‎rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml

-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
---
22
"Test cat thread_pool output":
3-
- skip:
4-
version: " - 6.99.99"
5-
reason: this API was changed in a backwards-incompatible fashion in 7.0.0 so we need to skip in a mixed cluster
63

74
- do:
85
cat.thread_pool: {}

‎rest-api-spec/src/main/resources/rest-api-spec/test/cluster.health/10_basic.yml

-3
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,6 @@
9494

9595
---
9696
"cluster health basic test, one index with wait for no initializing shards":
97-
- skip:
98-
version: " - 6.1.99"
99-
reason: "wait_for_no_initializing_shards is introduced in 6.2.0"
10097

10198
- do:
10299
indices.create:

‎rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/10_basic.yml

-3
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,6 @@
77

88
---
99
"get cluster state returns cluster_uuid at the top level":
10-
- skip:
11-
version: " - 6.3.99"
12-
reason: "cluster state including cluster_uuid at the top level is new in v6.4.0 and higher"
1310

1411
- do:
1512
cluster.state:

0 commit comments

Comments
 (0)