From ee50924bd00397e4e01f9fc499e61d2893b01a18 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 3 Aug 2020 16:15:35 +0100 Subject: [PATCH 01/60] Override auto-create index behaviour via a template flag --- .../elasticsearch/client/IndicesClientIT.java | 8 +- ...ComposableIndexTemplatesResponseTests.java | 2 +- .../template/ComposableTemplateIT.java | 2 +- .../action/bulk/TransportBulkAction.java | 121 +++++++++--------- .../action/support/AutoCreateIndex.java | 22 +++- .../metadata/ComposableIndexTemplate.java | 33 ++++- .../MetadataIndexTemplateService.java | 3 +- .../indices/create/AutoCreateActionTests.java | 6 +- .../MetadataRolloverServiceTests.java | 12 +- .../SimulateIndexTemplateRequestTests.java | 2 +- .../post/SimulateTemplateRequestTests.java | 2 +- ...utComposableIndexTemplateRequestTests.java | 4 +- ...ActionIndicesThatCannotBeCreatedTests.java | 15 +-- .../bulk/TransportBulkActionIngestTests.java | 7 +- .../action/bulk/TransportBulkActionTests.java | 5 - .../bulk/TransportBulkActionTookTests.java | 5 - .../action/support/AutoCreateIndexTests.java | 53 ++++++++ .../ComposableIndexTemplateTests.java | 24 ++-- .../MetadataCreateDataStreamServiceTests.java | 7 +- .../MetadataIndexTemplateServiceTests.java | 74 +++++------ .../metadata/ToAndFromJsonMetadataTests.java | 3 +- .../datastreams/DataStreamIT.java | 12 +- .../datastreams/DataStreamsStatsTests.java | 3 +- .../ml/integration/MlNativeIntegTestCase.java | 3 +- 24 files changed, 253 insertions(+), 175 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index f13d16c9c6893..57530be4cd4e5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -1579,7 +1579,7 @@ public void testDataStreams() throws Exception { CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"@timestamp\":{\"type\":\"date\"}}}"); Template template = new Template(null, mappings, null); ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(Collections.singletonList(dataStreamName), template, - Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate()); + Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate(), null); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name("ds-template").create(true).indexTemplate(indexTemplate); AcknowledgedResponse response = execute(putComposableIndexTemplateRequest, @@ -1655,7 +1655,7 @@ public void testIndexTemplates() throws Exception { Template template = new Template(settings, mappings, Map.of("alias", alias)); List pattern = List.of("pattern"); ComposableIndexTemplate indexTemplate = - new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null); + new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate); @@ -1702,7 +1702,7 @@ public void testSimulateIndexTemplate() throws Exception { Template template = new Template(settings, mappings, Map.of("alias", alias)); List pattern = List.of("pattern"); ComposableIndexTemplate indexTemplate = - new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null); + new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate); @@ -1713,7 +1713,7 @@ public void testSimulateIndexTemplate() throws Exception { SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest("pattern"); AliasMetadata simulationAlias = AliasMetadata.builder("simulation-alias").writeIndex(true).build(); ComposableIndexTemplate simulationTemplate = new ComposableIndexTemplate(pattern, new Template(null, null, - Map.of("simulation-alias", simulationAlias)), Collections.emptyList(), 2L, 1L, new HashMap<>(), null); + Map.of("simulation-alias", simulationAlias)), Collections.emptyList(), 2L, 1L, new HashMap<>(), null, null); PutComposableIndexTemplateRequest newIndexTemplateReq = new PutComposableIndexTemplateRequest().name("used-for-simulation").create(true).indexTemplate(indexTemplate); newIndexTemplateReq.indexTemplate(simulationTemplate); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java index 23759e7b09a58..f244f16b6a598 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java @@ -88,6 +88,6 @@ private static ComposableIndexTemplate randomIndexTemplate() { if (randomBoolean()) { dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate(); } - return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate); + return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate, null); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index a84accde6d169..1fbd740f27c60 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -102,7 +102,7 @@ public void testUsageOfDataStreamFails() throws IOException { Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser)); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null, - null, new ComposableIndexTemplate.DataStreamTemplate()); + null, new ComposableIndexTemplate.DataStreamTemplate(), null); Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet()); Exception actualException = (Exception) e.getCause(); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index c6c7fbf395247..4e96f3f6beb6d 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -216,73 +216,70 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, ActionListe return; } - if (needToCheck()) { - // Attempt to create all the indices that we're going to need during the bulk before we start. - // Step 1: collect all the indices in the request - final Map indices = bulkRequest.requests.stream() - // delete requests should not attempt to create the index (if the index does not - // exists), unless an external versioning is used - .filter(request -> request.opType() != DocWriteRequest.OpType.DELETE - || request.versionType() == VersionType.EXTERNAL - || request.versionType() == VersionType.EXTERNAL_GTE) - .collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2)); - /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create - * that we'll use when we try to run the requests. */ - final Map indicesThatCannotBeCreated = new HashMap<>(); - Set autoCreateIndices = new HashSet<>(); - ClusterState state = clusterService.state(); - for (Map.Entry indexAndFlag : indices.entrySet()) { - boolean shouldAutoCreate; - final String index = indexAndFlag.getKey(); - try { - shouldAutoCreate = shouldAutoCreate(index, state); - } catch (IndexNotFoundException e) { - shouldAutoCreate = false; - indicesThatCannotBeCreated.put(index, e); - } - // We should only auto create if we are not requiring it to be an alias - if (shouldAutoCreate && (indexAndFlag.getValue() == false)) { - autoCreateIndices.add(index); - } + // Attempt to create all the indices that we're going to need during the bulk before we start. + // Step 1: collect all the indices in the request + final Map indices = bulkRequest.requests.stream() + // delete requests should not attempt to create the index (if the index does not + // exists), unless an external versioning is used + .filter(request -> request.opType() != DocWriteRequest.OpType.DELETE + || request.versionType() == VersionType.EXTERNAL + || request.versionType() == VersionType.EXTERNAL_GTE) + .collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2)); + /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create + * that we'll use when we try to run the requests. */ + final Map indicesThatCannotBeCreated = new HashMap<>(); + Set autoCreateIndices = new HashSet<>(); + ClusterState state = clusterService.state(); + for (Map.Entry indexAndFlag : indices.entrySet()) { + boolean shouldAutoCreate; + final String index = indexAndFlag.getKey(); + try { + shouldAutoCreate = shouldAutoCreate(index, state); + } catch (IndexNotFoundException e) { + shouldAutoCreate = false; + indicesThatCannotBeCreated.put(index, e); } - // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back. - if (autoCreateIndices.isEmpty()) { - executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated); - } else { - final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size()); - for (String index : autoCreateIndices) { - createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener<>() { - @Override - public void onResponse(CreateIndexResponse result) { - if (counter.decrementAndGet() == 0) { - threadPool.executor(ThreadPool.Names.WRITE).execute( - () -> executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated)); - } + // We should only auto create if we are not requiring it to be an alias + if (shouldAutoCreate && (indexAndFlag.getValue() == false)) { + autoCreateIndices.add(index); + } + } + + // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back. + if (autoCreateIndices.isEmpty()) { + executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated); + } else { + final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size()); + for (String index : autoCreateIndices) { + createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener<>() { + @Override + public void onResponse(CreateIndexResponse result) { + if (counter.decrementAndGet() == 0) { + threadPool.executor(ThreadPool.Names.WRITE).execute( + () -> executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated)); } + } - @Override - public void onFailure(Exception e) { - if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) { - // fail all requests involving this index, if create didn't work - for (int i = 0; i < bulkRequest.requests.size(); i++) { - DocWriteRequest request = bulkRequest.requests.get(i); - if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) { - bulkRequest.requests.set(i, null); - } + @Override + public void onFailure(Exception e) { + if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) { + // fail all requests involving this index, if create didn't work + for (int i = 0; i < bulkRequest.requests.size(); i++) { + DocWriteRequest request = bulkRequest.requests.get(i); + if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) { + bulkRequest.requests.set(i, null); } } - if (counter.decrementAndGet() == 0) { - executeBulk(task, bulkRequest, startTime, ActionListener.wrap(listener::onResponse, inner -> { - inner.addSuppressed(e); - listener.onFailure(inner); - }), responses, indicesThatCannotBeCreated); - } } - }); - } + if (counter.decrementAndGet() == 0) { + executeBulk(task, bulkRequest, startTime, ActionListener.wrap(listener::onResponse, inner -> { + inner.addSuppressed(e); + listener.onFailure(inner); + }), responses, indicesThatCannotBeCreated); + } + } + }); } - } else { - executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap()); } } @@ -336,10 +333,6 @@ static void prohibitCustomRoutingOnDataStream(DocWriteRequest writeRequest, M } } - boolean needToCheck() { - return autoCreateIndex.needToCheck(); - } - boolean shouldAutoCreate(String index, ClusterState state) { return autoCreateIndex.shouldAutoCreate(index, state); } diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index 02d536592c27d..a2b0b0cc59a9a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -20,7 +20,10 @@ package org.elasticsearch.action.support; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -52,13 +55,6 @@ public AutoCreateIndex(Settings settings, ClusterSettings clusterSettings, Index clusterSettings.addSettingsUpdateConsumer(AUTO_CREATE_INDEX_SETTING, this::setAutoCreate); } - /** - * Do we really need to check if an index should be auto created? - */ - public boolean needToCheck() { - return this.autoCreate.autoCreateIndex; - } - /** * Should the index be auto created? * @throws IndexNotFoundException if the index doesn't exist and shouldn't be auto created @@ -67,6 +63,13 @@ public boolean shouldAutoCreate(String index, ClusterState state) { if (resolver.hasIndexAbstraction(index, state)) { return false; } + + // Templates can override the AUTO_CREATE_INDEX_SETTING setting + final ComposableIndexTemplate template = findTemplate(index, state.metadata()); + if (template != null && template.getAllowAutoCreate()) { + return true; + } + // One volatile read, so that all checks are done against the same instance: final AutoCreate autoCreate = this.autoCreate; if (autoCreate.autoCreateIndex == false) { @@ -100,6 +103,11 @@ void setAutoCreate(AutoCreate autoCreate) { this.autoCreate = autoCreate; } + private ComposableIndexTemplate findTemplate(String name, Metadata metadata) { + final String templateName = MetadataIndexTemplateService.findV2Template(metadata, name, false); + return metadata.templatesV2().get(templateName); + } + static class AutoCreate { private final boolean autoCreateIndex; private final List> expressions; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index a8af8d10faa7a..3e6451b10c526 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -55,6 +55,7 @@ public class ComposableIndexTemplate extends AbstractDiffable PARSER = new ConstructingObjectParser<>("index_template", @@ -65,7 +66,8 @@ public class ComposableIndexTemplate extends AbstractDiffable) a[5], - (DataStreamTemplate) a[6])); + (DataStreamTemplate) a[6], + (Boolean) a[7])); static { PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDEX_PATTERNS); @@ -75,6 +77,7 @@ public class ComposableIndexTemplate extends AbstractDiffable p.map(), METADATA); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DataStreamTemplate.PARSER, DATA_STREAM); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ALLOW_AUTO_CREATE); } private final List indexPatterns; @@ -90,6 +93,8 @@ public class ComposableIndexTemplate extends AbstractDiffable metadata; @Nullable private final DataStreamTemplate dataStreamTemplate; + @Nullable + private final Boolean allowAutoCreate; static Diff readITV2DiffFrom(StreamInput in) throws IOException { return AbstractDiffable.readDiffFrom(ComposableIndexTemplate::new, in); @@ -101,12 +106,12 @@ public static ComposableIndexTemplate parse(XContentParser parser) throws IOExce public ComposableIndexTemplate(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, @Nullable Long priority, @Nullable Long version, @Nullable Map metadata) { - this(indexPatterns, template, componentTemplates, priority, version, metadata, null); + this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null); } public ComposableIndexTemplate(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, @Nullable Long priority, @Nullable Long version, @Nullable Map metadata, - @Nullable DataStreamTemplate dataStreamTemplate) { + @Nullable DataStreamTemplate dataStreamTemplate, @Nullable Boolean allowAutoCreate) { this.indexPatterns = indexPatterns; this.template = template; this.componentTemplates = componentTemplates; @@ -114,6 +119,7 @@ public ComposableIndexTemplate(List indexPatterns, @Nullable Template te this.version = version; this.metadata = metadata; this.dataStreamTemplate = dataStreamTemplate; + this.allowAutoCreate = allowAutoCreate; } public ComposableIndexTemplate(StreamInput in) throws IOException { @@ -132,6 +138,11 @@ public ComposableIndexTemplate(StreamInput in) throws IOException { } else { this.dataStreamTemplate = null; } + if (in.getVersion().onOrAfter(Version.V_7_10_0)) { + this.allowAutoCreate = in.readOptionalBoolean(); + } else { + this.allowAutoCreate = null; + } } public List indexPatterns() { @@ -173,6 +184,11 @@ public DataStreamTemplate getDataStreamTemplate() { return dataStreamTemplate; } + @Nullable + public Boolean getAllowAutoCreate() { + return this.allowAutoCreate; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(this.indexPatterns); @@ -189,6 +205,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_9_0)) { out.writeOptionalWriteable(dataStreamTemplate); } + if (out.getVersion().onOrAfter(Version.V_7_10_0)) { + out.writeOptionalBoolean(allowAutoCreate); + } } @Override @@ -213,6 +232,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (this.dataStreamTemplate != null) { builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate); } + if (this.allowAutoCreate != null) { + builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate); + } builder.endObject(); return builder; } @@ -220,7 +242,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public int hashCode() { return Objects.hash(this.indexPatterns, this.template, this.componentTemplates, this.priority, this.version, - this.metadata, this.dataStreamTemplate); + this.metadata, this.dataStreamTemplate, this.allowAutoCreate); } @Override @@ -238,7 +260,8 @@ public boolean equals(Object obj) { Objects.equals(this.priority, other.priority) && Objects.equals(this.version, other.version) && Objects.equals(this.metadata, other.metadata) && - Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate); + Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate) && + Objects.equals(this.allowAutoCreate, other.allowAutoCreate); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 8131b0e53e5c9..66a21beb3c1b5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -501,7 +501,8 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo final Template finalTemplate = new Template(finalSettings, stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases()); finalIndexTemplate = new ComposableIndexTemplate(template.indexPatterns(), finalTemplate, template.composedOf(), - template.priority(), template.version(), template.metadata(), template.getDataStreamTemplate()); + template.priority(), template.version(), template.metadata(), template.getDataStreamTemplate(), + template.getAllowAutoCreate()); } if (finalIndexTemplate.equals(existing)) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java index 07d48b8b077bd..922ccc5df27c2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java @@ -37,9 +37,9 @@ public void testResolveAutoCreateDataStreams() { { Metadata.Builder mdBuilder = new Metadata.Builder(); DataStreamTemplate dataStreamTemplate = new DataStreamTemplate(); - mdBuilder.put("1", new ComposableIndexTemplate(List.of("legacy-logs-*"), null, null, 10L, null, null, null)); - mdBuilder.put("2", new ComposableIndexTemplate(List.of("logs-*"), null, null, 20L, null, null, dataStreamTemplate)); - mdBuilder.put("3", new ComposableIndexTemplate(List.of("logs-foobar"), null, null, 30L, null, null, dataStreamTemplate)); + mdBuilder.put("1", new ComposableIndexTemplate(List.of("legacy-logs-*"), null, null, 10L, null, null, null, null)); + mdBuilder.put("2", new ComposableIndexTemplate(List.of("logs-*"), null, null, 20L, null, null, dataStreamTemplate, null)); + mdBuilder.put("3", new ComposableIndexTemplate(List.of("logs-foobar"), null, null, 30L, null, null, dataStreamTemplate, null)); metadata = mdBuilder.build(); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index e902f4735a433..f65a00fc45c7c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -372,7 +372,7 @@ public void testRejectDuplicateAliasV2() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("foo-*", "bar-*"), new Template(null, null, aliases), - null, null, null, null, null); + null, null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("test-template", template).build(); @@ -389,7 +389,7 @@ public void testRejectDuplicateAliasV2UsingComponentTemplates() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null); final ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("foo-*", "bar-*"), null, - Collections.singletonList("ct"), null, null, null, null); + Collections.singletonList("ct"), null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("ct", ct) @@ -425,7 +425,7 @@ public void testHiddenAffectsResolvedV2Templates() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("*"), new Template(null, null, aliases), - null, null, null, null, null); + null, null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("test-template", template).build(); @@ -446,7 +446,7 @@ public void testHiddenAffectsResolvedV2ComponentTemplates() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null); final ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("*"), null, - Collections.singletonList("ct"), null, null, null, null); + Collections.singletonList("ct"), null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("ct", ct) @@ -538,7 +538,7 @@ public void testRolloverClusterState() throws Exception { public void testRolloverClusterStateForDataStream() throws Exception { final DataStream dataStream = DataStreamTestHelper.randomInstance(); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStream.getName() + "*"), null, null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); Metadata.Builder builder = Metadata.builder(); builder.put("template", template); for (Index index : dataStream.getIndices()) { @@ -634,7 +634,7 @@ public void testValidation() throws Exception { sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName(); defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStream.getName() + "*"), null, null, null, null, - null, new ComposableIndexTemplate.DataStreamTemplate()); + null, new ComposableIndexTemplate.DataStreamTemplate(), null); builder.put("template", template); for (Index index : dataStream.getIndices()) { builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java index 09c0e6dcac1f1..11db2728b3388 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -63,7 +63,7 @@ public void testIndexNameCannotBeNullOrEmpty() { public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); - ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null); + ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null, null); PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java index ab8b49e03a4e4..b7530c29d2163 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java @@ -64,7 +64,7 @@ public void testIndexNameCannotBeNullOrEmpty() { public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); - ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null); + ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null, null); PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java index 47e25a9cd53f8..e48f6694bc959 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java @@ -58,7 +58,7 @@ protected PutComposableIndexTemplateAction.Request mutateInstance(PutComposableI public void testPutGlobalTemplatesCannotHaveHiddenIndexSetting() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); - ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null); + ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null, null); PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); @@ -84,7 +84,7 @@ public void testPutIndexTemplateV2RequestMustContainTemplate() { public void testValidationOfPriority() { PutComposableIndexTemplateAction.Request req = new PutComposableIndexTemplateAction.Request("test"); - req.indexTemplate(new ComposableIndexTemplate(Arrays.asList("foo", "bar"), null, null, -5L, null, null, null)); + req.indexTemplate(new ComposableIndexTemplate(Arrays.asList("foo", "bar"), null, null, -5L, null, null, null, null)); ActionRequestValidationException validationException = req.validate(); assertThat(validationException, is(notNullValue())); List validationErrors = validationException.validationErrors(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index e182833c4902f..b22edd0fa679b 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -64,11 +64,9 @@ public void testNonExceptional() { bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); bulkRequest.add(new DeleteRequest(randomAlphaOfLength(5))); bulkRequest.add(new UpdateRequest(randomAlphaOfLength(5), randomAlphaOfLength(5))); - // Test emulating auto_create_index=false - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, null); - // Test emulating auto_create_index=true + // Test emulating that index can be auto-created indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true); - // Test emulating all indices already created + // Test emulating that index cannot be auto-created indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false); // Test emulating auto_create_index=true with some indices already created. indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean()); @@ -112,15 +110,19 @@ private void indicesThatCannotBeCreatedTestCase(Set expected, ClusterState state = mock(ClusterState.class); when(state.getMetadata()).thenReturn(Metadata.EMPTY_METADATA); when(clusterService.state()).thenReturn(state); + DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); when(state.getNodes()).thenReturn(discoveryNodes); when(discoveryNodes.getMinNodeVersion()).thenReturn(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT)); + DiscoveryNode localNode = mock(DiscoveryNode.class); when(clusterService.localNode()).thenReturn(localNode); when(localNode.isIngestNode()).thenReturn(randomBoolean()); + final ThreadPool threadPool = mock(ThreadPool.class); final ExecutorService direct = EsExecutors.newDirectExecutorService(); when(threadPool.executor(anyString())).thenReturn(direct); + TransportBulkAction action = new TransportBulkAction(threadPool, mock(TransportService.class), clusterService, null, null, mock(ActionFilters.class), null, null, new IndexingPressure(Settings.EMPTY)) { @@ -130,11 +132,6 @@ void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, Action assertEquals(expected, indicesThatCannotBeCreated.keySet()); } - @Override - boolean needToCheck() { - return null != shouldAutoCreate; // Use "null" to mean "no indices can be created so don't bother checking" - } - @Override boolean shouldAutoCreate(String index, ClusterState state) { return shouldAutoCreate.apply(index); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 8e7af9de16adb..a3634c8458f59 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -147,10 +147,7 @@ SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTING ), new IndexingPressure(SETTINGS) ); } - @Override - protected boolean needToCheck() { - return needToCheck; - } + @Override void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos, final ActionListener listener, final AtomicArray responses, Map indicesThatCannotBeCreated) { @@ -582,7 +579,7 @@ public void testFindDefaultPipelineFromV2TemplateMatch() { ComposableIndexTemplate t1 = new ComposableIndexTemplate(Collections.singletonList("missing_*"), new Template(Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "pipeline2").build(), null, null), - null, null, null, null, null); + null, null, null, null, null, null); ClusterState state = clusterService.state(); Metadata metadata = Metadata.builder() diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 2c6b3eb520ba7..8d39bf48a7b9f 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -80,11 +80,6 @@ null, new ActionFilters(Collections.emptySet()), new Resolver(), new IndexingPressure(Settings.EMPTY)); } - @Override - protected boolean needToCheck() { - return true; - } - @Override void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener listener) { indexCreated = true; diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 44007488433db..a93c6a56fe1ac 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -238,11 +238,6 @@ static class TestTransportBulkAction extends TransportBulkAction { relativeTimeProvider); } - @Override - boolean needToCheck() { - return randomBoolean(); - } - @Override boolean shouldAutoCreate(String index, ClusterState state) { return randomBoolean(); diff --git a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java index ddb78cd46b644..c7563983e9d13 100644 --- a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; @@ -191,6 +192,58 @@ public void testUpdate() { assertThat(autoCreateIndex.getAutoCreate().getExpressions().get(0).v1(), equalTo("logs-*")); } + /** + * Check that if a template matches the index to be created, but that template does not have the allow_auto_create + * setting, then it does not override the global setting. + */ + public void testTemplateSettingsDoesNotOverride() { + String randomIndex = randomAlphaOfLengthBetween(2, 10); + final ComposableIndexTemplate template = new ComposableIndexTemplate( + List.of(randomIndex.charAt(0) + "*"), + null, + List.of(), + null, + null, + Map.of(), + null, + false + ); + + final Metadata metadata = Metadata.builder().indexTemplates(Map.of("test_template", template)).build(); + final ClusterState clusterState = ClusterState.builder(buildClusterState()).metadata(metadata).build(); + + Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build(); + AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings); + IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> + autoCreateIndex.shouldAutoCreate(randomIndex, clusterState)); + assertEquals("no such index [" + randomIndex + "] and [action.auto_create_index] is [false]", e.getMessage()); + } + + /** + * Check that if a template matches the index to be created, and that template has the allow_auto_create + * setting, then it overrides the global setting. + */ + public void testTemplateSettingsOverride() { + String randomIndex = randomAlphaOfLengthBetween(2, 10); + final ComposableIndexTemplate template = new ComposableIndexTemplate( + List.of(randomIndex.charAt(0) + "*"), + null, + List.of(), + null, + null, + Map.of(), + null, + true + ); + + final Metadata metadata = Metadata.builder().indexTemplates(Map.of("test_template", template)).build(); + final ClusterState clusterState = ClusterState.builder(buildClusterState()).metadata(metadata).build(); + + Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build(); + AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings); + assertThat(autoCreateIndex.shouldAutoCreate(randomIndex, clusterState), equalTo(true)); + } + private static ClusterState buildClusterState(String... indices) { Metadata.Builder metadata = Metadata.builder(); for (String index : indices) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index 24efa12b3c966..10d805f780502 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -97,7 +97,9 @@ public static ComposableIndexTemplate randomInstance() { randomBoolean() ? null : randomNonNegativeLong(), randomBoolean() ? null : randomNonNegativeLong(), meta, - dataStreamTemplate); + dataStreamTemplate, + randomBoolean() ? null : randomBoolean() + ); } private static Map randomAliases() { @@ -163,7 +165,7 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori List newIndexPatterns = randomValueOtherThan(orig.indexPatterns(), () -> randomList(1, 4, () -> randomAlphaOfLength(4))); return new ComposableIndexTemplate(newIndexPatterns, orig.template(), orig.composedOf(), - orig.priority(), orig.version(), orig.metadata(), orig.getDataStreamTemplate()); + orig.priority(), orig.version(), orig.metadata(), orig.getDataStreamTemplate(), null); case 1: return new ComposableIndexTemplate(orig.indexPatterns(), randomValueOtherThan(orig.template(), () -> new Template(randomSettings(), @@ -172,7 +174,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 2: List newComposedOf = randomValueOtherThan(orig.composedOf(), () -> randomList(0, 10, () -> randomAlphaOfLength(5))); @@ -182,7 +185,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 3: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -190,7 +194,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori randomValueOtherThan(orig.priority(), ESTestCase::randomNonNegativeLong), orig.version(), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 4: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -198,7 +203,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 5: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -206,7 +212,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), randomValueOtherThan(orig.metadata(), ComposableIndexTemplateTests::randomMeta), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 6: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -214,7 +221,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), orig.metadata(), - randomValueOtherThan(orig.getDataStreamTemplate(), ComposableIndexTemplateTests::randomDataStreamTemplate)); + randomValueOtherThan(orig.getDataStreamTemplate(), ComposableIndexTemplateTests::randomDataStreamTemplate), + orig.getAllowAutoCreate()); default: throw new IllegalStateException("illegal randomization branch"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index d52f36afb1fb1..3e338058732c1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -48,7 +48,7 @@ public void testCreateDataStream() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); @@ -126,7 +126,8 @@ public void testCreateDataStreamNoTemplate() throws Exception { public void testCreateDataStreamNoValidTemplate() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; - ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, null); + ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, null, + null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); @@ -141,7 +142,7 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { public static ClusterState createDataStream(final String dataStreamName) throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 782cbd5d400cf..9749d334894a1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -332,15 +332,15 @@ public void testUpdateComponentTemplateWithIndexHiddenSetting() throws Exception assertNotNull(state.metadata().componentTemplates().get("foo")); ComposableIndexTemplate firstGlobalIndexTemplate = - new ComposableIndexTemplate(List.of("*"), template, List.of("foo"), 1L, null, null, null); + new ComposableIndexTemplate(List.of("*"), template, List.of("foo"), 1L, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate1", firstGlobalIndexTemplate); ComposableIndexTemplate secondGlobalIndexTemplate = - new ComposableIndexTemplate(List.of("*"), template, List.of("foo"), 2L, null, null, null); + new ComposableIndexTemplate(List.of("*"), template, List.of("foo"), 2L, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate2", secondGlobalIndexTemplate); ComposableIndexTemplate fooPatternIndexTemplate = - new ComposableIndexTemplate(List.of("foo-*"), template, List.of("foo"), 3L, null, null, null); + new ComposableIndexTemplate(List.of("foo-*"), template, List.of("foo"), 3L, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "foopatternindextemplate", fooPatternIndexTemplate); // update the component template to set the index.hidden setting @@ -392,7 +392,7 @@ public void testUpdateIndexTemplateV2() throws Exception { List patterns = new ArrayList<>(template.indexPatterns()); patterns.add("new-pattern"); template = new ComposableIndexTemplate(patterns, template.template(), template.composedOf(), template.priority(), - template.version(), template.metadata(), null); + template.version(), template.metadata(), null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template); assertNotNull(state.metadata().templatesV2().get("foo")); @@ -430,7 +430,7 @@ public void testPuttingV2TemplateGeneratesWarning() throws Exception { .build(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -467,7 +467,7 @@ public void onFailure(Exception e) { waitToCreateComponentTemplate.await(10, TimeUnit.SECONDS); ComposableIndexTemplate globalIndexTemplate = new ComposableIndexTemplate(List.of("*"), - null, List.of("ct-with-index-hidden-setting"), null, null, null, null); + null, List.of("ct-with-index-hidden-setting"), null, null, null, null, null); expectThrows(InvalidIndexTemplateException.class, () -> metadataIndexTemplateService.putIndexTemplateV2("testing", true, "template-referencing-ct-with-hidden-index-setting", @@ -492,7 +492,7 @@ public void onFailure(Exception e) { public void testPuttingV1StarTemplateGeneratesWarning() throws Exception { final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); @@ -513,7 +513,7 @@ public void testPuttingV1StarTemplateGeneratesWarning() throws Exception { public void testPuttingV1NonStarTemplateGeneratesError() throws Exception { final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); @@ -547,7 +547,7 @@ public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() .build(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -588,7 +588,7 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() thr .build(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -614,11 +614,11 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() thr public void testPuttingOverlappingV2Template() throws Exception { { ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("egg*", "baz"), - null, null, 1L, null, null, null); + null, null, 1L, null, null, null, null); MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Arrays.asList("abc", "baz*"), - null, null, 1L, null, null, null); + null, null, 1L, null, null, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo2", newTemplate)); assertThat(e.getMessage(), equalTo("index template [foo2] has index patterns [abc, baz*] matching patterns from existing " + @@ -628,11 +628,11 @@ public void testPuttingOverlappingV2Template() throws Exception { { ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("egg*", "baz"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Arrays.asList("abc", "baz*"), - null, null, 0L, null, null, null); + null, null, 0L, null, null, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo2", newTemplate)); assertThat(e.getMessage(), equalTo("index template [foo2] has index patterns [abc, baz*] matching patterns from existing " + @@ -648,9 +648,9 @@ public void testFindV2Templates() throws Exception { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); - ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), null, List.of("ct"), null, 1L, null, null); + ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), null, List.of("ct"), null, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); - ComposableIndexTemplate it2 = new ComposableIndexTemplate(List.of("in*"), null, List.of("ct"), 10L, 2L, null, null); + ComposableIndexTemplate it2 = new ComposableIndexTemplate(List.of("in*"), null, List.of("ct"), 10L, 2L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean()); @@ -665,9 +665,9 @@ public void testFindV2TemplatesForHiddenIndex() throws Exception { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); - ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), null, List.of("ct"), 0L, 1L, null, null); + ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), null, List.of("ct"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); - ComposableIndexTemplate it2 = new ComposableIndexTemplate(List.of("*"), null, List.of("ct"), 10L, 2L, null, null); + ComposableIndexTemplate it2 = new ComposableIndexTemplate(List.of("*"), null, List.of("ct"), 10L, 2L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true); @@ -680,7 +680,7 @@ public void testFindV2InvalidGlobalTemplate() { try { // add an invalid global template that specifies the `index.hidden` setting ComposableIndexTemplate invalidGlobalTemplate = new ComposableIndexTemplate(List.of("*"), - templateWithHiddenSetting, List.of("ct"), 5L, 1L, null, null); + templateWithHiddenSetting, List.of("ct"), 5L, 1L, null, null, null); Metadata invalidGlobalTemplateMetadata = Metadata.builder().putCustom(ComposableIndexTemplateMetadata.TYPE, new ComposableIndexTemplateMetadata(Map.of("invalid_global_template", invalidGlobalTemplate))).build(); @@ -724,7 +724,7 @@ public void testResolveConflictingMappings() throws Exception { " }\n" + " }\n" + " }"), null), - List.of("ct_low", "ct_high"), 0L, 1L, null, null); + List.of("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index", @@ -788,7 +788,7 @@ public void testResolveMappings() throws Exception { " }\n" + " }\n" + " }"), null), - List.of("ct_low", "ct_high"), 0L, 1L, null, null); + List.of("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index", @@ -840,7 +840,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex " }\n" + " }\n" + " }"), null), - List.of("ct1"), 0L, 1L, null, new ComposableIndexTemplate.DataStreamTemplate()); + List.of("ct1"), 0L, 1L, null, new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, true, "logs-data-stream-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "logs-data-stream-template", @@ -879,7 +879,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex " }\n" + " }\n" + " }"), null), - List.of("ct1"), 0L, 1L, null, null); + List.of("ct1"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries", @@ -947,7 +947,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception state = service.addComponentTemplate(state, true, "ct1", ct1); ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("logs*"), null, List.of("ct1"), 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, true, "logs-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "logs-template", @@ -982,7 +982,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception " }\n" + " }"), null); ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("timeseries*"), template, null, 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", @@ -1029,7 +1029,7 @@ public void testResolveSettings() throws Exception { .put("index.blocks.write", false) .put("index.number_of_shards", 3) .build(), null, null), - List.of("ct_low", "ct_high"), 0L, 1L, null, null); + List.of("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), "my-template"); @@ -1057,7 +1057,7 @@ public void testResolveAliases() throws Exception { state = service.addComponentTemplate(state, true, "ct_low", ct2); ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), new Template(null, null, a3), - List.of("ct_low", "ct_high"), 0L, 1L, null, null); + List.of("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(state.metadata(), "my-template"); @@ -1173,7 +1173,7 @@ public void testIndexTemplateFailsToOverrideComponentTemplateMappingField() thr " }\n" + " }\n" + " }"), null), - randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null); + randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null, null); final ClusterState finalState = state; IllegalArgumentException e = expectThrows(IllegalArgumentException.class, @@ -1224,7 +1224,7 @@ public void testUpdateComponentTemplateFailsIfResolvedIndexTemplatesWouldBeInval state = service.addComponentTemplate(state, true, "c2", ct2); ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), new Template(null, null, null), - randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null); + randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null, null); // Great, the templates aren't invalid state = service.addIndexTemplateV2(state, randomBoolean(), "my-template", it); @@ -1296,7 +1296,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { .build(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"), - null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, false, "logs", template); @@ -1318,7 +1318,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Test replacing it with a version without the data stream config IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"), null, null, - 100L, null, null, null); + 100L, null, null, null, null); service.addIndexTemplateV2(stateWithDS, false, "logs", nonDSTemplate); }); @@ -1329,7 +1329,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Test adding a higher priority version that would cause problems e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-my*-*"), null, null, - 105L, null, null, null); + 105L, null, null, null, null); service.addIndexTemplateV2(stateWithDS, false, "logs2", nonDSTemplate); }); @@ -1340,7 +1340,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Change the pattern to one that doesn't match the data stream e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null, - null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); service.addIndexTemplateV2(stateWithDS, false, "logs", newTemplate); }); @@ -1350,12 +1350,12 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Add an additional template that matches our data stream at a lower priority ComposableIndexTemplate mysqlTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-mysql-*"), null, - null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); ClusterState stateWithDSAndTemplate = service.addIndexTemplateV2(stateWithDS, false, "logs-mysql", mysqlTemplate); // We should be able to replace the "logs" template, because we have the "logs-mysql" template that can handle the data stream ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null, null, - 100L, null, null, null); + 100L, null, null, null, null); service.addIndexTemplateV2(stateWithDSAndTemplate, false, "logs", nonDSTemplate); } @@ -1439,9 +1439,9 @@ clusterService, createIndexService, new AliasValidator(), indicesService, @SuppressWarnings("unchecked") public static void assertTemplatesEqual(ComposableIndexTemplate actual, ComposableIndexTemplate expected) { ComposableIndexTemplate actualNoTemplate = new ComposableIndexTemplate(actual.indexPatterns(), null, - actual.composedOf(), actual.priority(), actual.version(), actual.metadata(), actual.getDataStreamTemplate()); + actual.composedOf(), actual.priority(), actual.version(), actual.metadata(), actual.getDataStreamTemplate(), null); ComposableIndexTemplate expectedNoTemplate = new ComposableIndexTemplate(expected.indexPatterns(), null, - expected.composedOf(), expected.priority(), expected.version(), expected.metadata(), expected.getDataStreamTemplate()); + expected.composedOf(), expected.priority(), expected.version(), expected.metadata(), expected.getDataStreamTemplate(), null); assertThat(actualNoTemplate, equalTo(expectedNoTemplate)); Template actualTemplate = actual.template(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index e6af205c3ad6c..eee43ee32c2d5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -79,7 +79,8 @@ public void testSimpleJsonFromAndTo() throws IOException { 5L, 4L, Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken")), - randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate())) + randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate(), + null)) .put(IndexMetadata.builder("test12") .settings(settings(Version.CURRENT) .put("setting1", "value1") diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 69858e81d4463..d8b5bf7596e94 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -279,7 +279,8 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); @@ -362,7 +363,8 @@ public void testTimeStampValidationInvalidFieldMapping() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); @@ -821,7 +823,8 @@ public void testMixedAutoCreate() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet(); @@ -987,7 +990,8 @@ static void putComposableIndexTemplate(String id, @Nullable String mappings, Lis null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java index 2d732b534ef44..5b9edc96bf6e4 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java @@ -218,7 +218,8 @@ private String createDataStream() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ); assertTrue( client().execute( diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 536819a167189..471009d42c88e 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -313,7 +313,8 @@ protected static void createDataStreamAndTemplate(String dataStreamName, String null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()))) + new ComposableIndexTemplate.DataStreamTemplate(), + null))) .actionGet(); client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet(); } From f62c83b2ccd790b4b631ef786c4c6387f227991d Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 4 Aug 2020 20:52:20 +0100 Subject: [PATCH 02/60] WIP - trying to test watcher --- .../core/src/main/resources/ilm-history.json | 1 + .../src/main/resources/logs-template.json | 1 + .../main/resources/logstash-management.json | 1 + .../src/main/resources/metrics-template.json | 1 + .../main/resources/monitoring-alerts-7.json | 1 + .../src/main/resources/monitoring-beats.json | 1 + .../src/main/resources/monitoring-es.json | 1 + .../src/main/resources/monitoring-kibana.json | 1 + .../main/resources/monitoring-logstash.json | 1 + .../results_index_template.json | 1 + .../state_index_template.json | 1 + .../xpack/core/ml/config_index_template.json | 1 + .../core/ml/inference_index_template.json | 1 + .../xpack/core/ml/meta_index_template.json | 1 + .../core/ml/notifications_index_template.json | 1 + .../idp/saml-service-provider-template.json | 1 + .../core/src/main/resources/watches.json | 1 + .../execute_watch/90_auto_create_index.yml | 75 +++++++++++++++++++ 18 files changed, 92 insertions(+) create mode 100644 x-pack/plugin/watcher/qa/rest/src/test/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml diff --git a/x-pack/plugin/core/src/main/resources/ilm-history.json b/x-pack/plugin/core/src/main/resources/ilm-history.json index bb79af74f2034..934163f2cd00d 100644 --- a/x-pack/plugin/core/src/main/resources/ilm-history.json +++ b/x-pack/plugin/core/src/main/resources/ilm-history.json @@ -12,6 +12,7 @@ "index.hidden": true, "index.format": 1 }, + "allow_auto_create": true, "mappings": { "_doc": { "dynamic": false, diff --git a/x-pack/plugin/core/src/main/resources/logs-template.json b/x-pack/plugin/core/src/main/resources/logs-template.json index 0e23872ec0170..9c21a2b6ae504 100644 --- a/x-pack/plugin/core/src/main/resources/logs-template.json +++ b/x-pack/plugin/core/src/main/resources/logs-template.json @@ -6,6 +6,7 @@ "logs-mappings", "logs-settings" ], + "allow_auto_create": true, "_meta": { "description": "default logs template installed by x-pack", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/logstash-management.json b/x-pack/plugin/core/src/main/resources/logstash-management.json index d9528238dc0fb..00b0826a02574 100644 --- a/x-pack/plugin/core/src/main/resources/logstash-management.json +++ b/x-pack/plugin/core/src/main/resources/logstash-management.json @@ -7,6 +7,7 @@ "codec": "best_compression" } }, + "allow_auto_create": true, "mappings" : { "_doc" : { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/metrics-template.json b/x-pack/plugin/core/src/main/resources/metrics-template.json index cccd852196d41..0da4169de79b7 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-template.json +++ b/x-pack/plugin/core/src/main/resources/metrics-template.json @@ -6,6 +6,7 @@ "metrics-mappings", "metrics-settings" ], + "allow_auto_create": true, "_meta": { "description": "default metrics template installed by x-pack", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json b/x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json index 5016eca9e92fc..427254bfb5610 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json @@ -10,6 +10,7 @@ "codec": "best_compression" } }, + "allow_auto_create": true, "mappings": { "_doc": { "dynamic": false, diff --git a/x-pack/plugin/core/src/main/resources/monitoring-beats.json b/x-pack/plugin/core/src/main/resources/monitoring-beats.json index a39507ec8a4d9..5cdf7fef90978 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-beats.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-beats.json @@ -10,6 +10,7 @@ "index.number_of_shards": 1 }, "version": 7000099, + "allow_auto_create": true, "mappings": { "_doc": { "dynamic": false, diff --git a/x-pack/plugin/core/src/main/resources/monitoring-es.json b/x-pack/plugin/core/src/main/resources/monitoring-es.json index 9517c4a49c88b..05168fb51a168 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-es.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-es.json @@ -8,6 +8,7 @@ "index.format": 7, "index.codec": "best_compression" }, + "allow_auto_create": true, "mappings": { "_doc": { "date_detection": false, diff --git a/x-pack/plugin/core/src/main/resources/monitoring-kibana.json b/x-pack/plugin/core/src/main/resources/monitoring-kibana.json index 1f916625c37f3..b30c593963fb6 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-kibana.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-kibana.json @@ -8,6 +8,7 @@ "index.format": 7, "index.codec": "best_compression" }, + "allow_auto_create": true, "mappings": { "_doc": { "dynamic": false, diff --git a/x-pack/plugin/core/src/main/resources/monitoring-logstash.json b/x-pack/plugin/core/src/main/resources/monitoring-logstash.json index a3dc60ae803b9..f24b096b087b6 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-logstash.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-logstash.json @@ -8,6 +8,7 @@ "index.format": 7, "index.codec": "best_compression" }, + "allow_auto_create": true, "mappings": { "_doc": { "dynamic": false, diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json index bccad3e753678..009325d0d435f 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json @@ -4,6 +4,7 @@ "index_patterns" : [ ".ml-anomalies-*" ], + "allow_auto_create": true, "settings" : { "index" : { "translog" : { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json index 64635b658e3b2..ce42c65cd4c29 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json @@ -12,6 +12,7 @@ "index.lifecycle.name": "${xpack.ml.index.lifecycle.name}", "index.lifecycle.rollover_alias": "${xpack.ml.index.lifecycle.rollover_alias}" }, + "allow_auto_create": true, "mappings" : { "_doc": { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json index 8c6c352ab245a..7756ffdd45b9f 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json @@ -4,6 +4,7 @@ "index_patterns" : [ ".ml-config" ], + "allow_auto_create": true, "settings" : { "index" : { "max_result_window" : "${xpack.ml.config.max_result_window}", diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json index a77a0119e953b..2b680b0515ac3 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json @@ -10,6 +10,7 @@ "auto_expand_replicas" : "0-1" } }, + "allow_auto_create": true, "mappings" : { "_doc": { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json index 19df45c52509c..042d7dea80ae4 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json @@ -10,6 +10,7 @@ "auto_expand_replicas" : "0-1" } }, + "allow_auto_create": true, "mappings" : { "_doc": { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json index e8ac3f1fcce6f..f5f9a5990306f 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json @@ -11,6 +11,7 @@ "hidden": true } }, + "allow_auto_create": true, "mappings" : { "_doc": { "_meta" : { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json index 60f964fe13483..35ea3ed218ce4 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json @@ -14,6 +14,7 @@ "index.refresh_interval": "1s", "index.format": 1 }, + "allow_auto_create": true, "mappings": { "_doc": { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/watches.json b/x-pack/plugin/core/src/main/resources/watches.json index 5181dfdc864f8..748347185d039 100644 --- a/x-pack/plugin/core/src/main/resources/watches.json +++ b/x-pack/plugin/core/src/main/resources/watches.json @@ -8,6 +8,7 @@ "index.format": 6, "index.priority": 800 }, + "allow_auto_create": true, "mappings": { "_doc": { "dynamic" : "strict", diff --git a/x-pack/plugin/watcher/qa/rest/src/test/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml b/x-pack/plugin/watcher/qa/rest/src/test/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml new file mode 100644 index 0000000000000..e9224638612b5 --- /dev/null +++ b/x-pack/plugin/watcher/qa/rest/src/test/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml @@ -0,0 +1,75 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + - do: + cluster.put_settings: + body: + transient: + action.auto_create_index: false + +--- +teardown: + - do: + watcher.delete_watch: + id: "test_watch" + ignore: 404 + - do: + cluster.put_settings: + body: + transient: + action.auto_create_index: null + +--- +"Test execute watch api with configured trigger data timestamps": + - do: + watcher.put_watch: + id: "test_watch" + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "simple": { + "foo": "bar" + } + }, + "condition": { + "always": {} + }, + "actions": { + "indexme" : { + "index" : { + "index" : "my_test_index", + "doc_id": "my-id" + } + } + } + } + - match: { _id: "test_watch" } + - match: { created: true } + + - do: + watcher.execute_watch: + id: "test_watch" + body: > + { + "trigger_data" : { + "triggered_time" : "2012-12-12T12:12:12.120Z", + "scheduled_time" : "2000-12-12T12:12:12.120Z" + } + } + + - match: { watch_record.watch_id: "test_watch" } + - match: { watch_record.trigger_event.type: "manual" } + - match: { watch_record.trigger_event.triggered_time: "2012-12-12T12:12:12.120Z" } + - match: { watch_record.trigger_event.manual.schedule.scheduled_time: "2000-12-12T12:12:12.120Z" } + - match: { watch_record.state: "executed" } + - match: { watch_record.status.execution_state: "executed" } + - match: { watch_record.status.state.active: true } + - is_true: watch_record.node + - match: { watch_record.status.actions.indexme.ack.state: "ackable" } + - gt: { watch_record.result.execution_duration: 0 } + From 3bb580ca7a16b15405b93b95bba4116657ed0e33 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 2 Sep 2020 11:10:36 +0100 Subject: [PATCH 03/60] Fixes --- .../action/support/AutoCreateIndex.java | 5 ++++ .../MetadataIndexTemplateService.java | 1 + .../core/src/main/resources/ilm-history.json | 2 +- .../main/resources/monitoring-alerts-7.json | 1 - .../src/main/resources/monitoring-beats.json | 1 - .../src/main/resources/monitoring-es.json | 1 - .../src/main/resources/monitoring-kibana.json | 1 - .../main/resources/monitoring-logstash.json | 1 - .../results_index_template.json | 1 - .../state_index_template.json | 1 - .../xpack/core/ml/config_index_template.json | 1 - .../core/ml/inference_index_template.json | 1 - .../xpack/core/ml/meta_index_template.json | 1 - .../core/ml/notifications_index_template.json | 1 - .../idp/saml-service-provider-template.json | 1 - .../core/src/main/resources/slm-history.json | 1 + .../src/main/resources/triggered-watches.json | 1 + .../main/resources/watch-history-no-ilm.json | 1 + .../src/main/resources/watch-history.json | 1 + .../core/src/main/resources/watches.json | 2 +- .../exporter/local/LocalExporter.java | 2 +- .../execute_watch/90_auto_create_index.yml | 25 +------------------ 22 files changed, 14 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index a2b0b0cc59a9a..5cdff7a16a992 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.support; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -42,6 +44,7 @@ * a write operation is about to happen in a non existing index. */ public final class AutoCreateIndex { + private static final Logger logger = LogManager.getLogger(AutoCreateIndex.class); public static final Setting AUTO_CREATE_INDEX_SETTING = new Setting<>("action.auto_create_index", "true", AutoCreate::new, Property.NodeScope, Setting.Property.Dynamic); @@ -65,7 +68,9 @@ public boolean shouldAutoCreate(String index, ClusterState state) { } // Templates can override the AUTO_CREATE_INDEX_SETTING setting + logger.warn("Looking for template for {}", index); final ComposableIndexTemplate template = findTemplate(index, state.metadata()); + logger.warn("Results: {}", template); if (template != null && template.getAllowAutoCreate()) { return true; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 66a21beb3c1b5..2867aaf5c87e6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -905,6 +905,7 @@ public static String findV2Template(Metadata metadata, String indexName, boolean final Map matchedTemplates = new HashMap<>(); for (Map.Entry entry : metadata.templatesV2().entrySet()) { final String name = entry.getKey(); + logger.warn("XXX: template v2 name = {}", name); final ComposableIndexTemplate template = entry.getValue(); if (isHidden == false) { final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate); diff --git a/x-pack/plugin/core/src/main/resources/ilm-history.json b/x-pack/plugin/core/src/main/resources/ilm-history.json index c544c28256391..6f545b55eff8f 100644 --- a/x-pack/plugin/core/src/main/resources/ilm-history.json +++ b/x-pack/plugin/core/src/main/resources/ilm-history.json @@ -9,7 +9,6 @@ "index.auto_expand_replicas": "0-1", "index.lifecycle.name": "ilm-history-ilm-policy" }, - "allow_auto_create": true, "mappings": { "dynamic": false, "properties": { @@ -78,6 +77,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for ILM history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json b/x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json index 427254bfb5610..5016eca9e92fc 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json @@ -10,7 +10,6 @@ "codec": "best_compression" } }, - "allow_auto_create": true, "mappings": { "_doc": { "dynamic": false, diff --git a/x-pack/plugin/core/src/main/resources/monitoring-beats.json b/x-pack/plugin/core/src/main/resources/monitoring-beats.json index 5cdf7fef90978..a39507ec8a4d9 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-beats.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-beats.json @@ -10,7 +10,6 @@ "index.number_of_shards": 1 }, "version": 7000099, - "allow_auto_create": true, "mappings": { "_doc": { "dynamic": false, diff --git a/x-pack/plugin/core/src/main/resources/monitoring-es.json b/x-pack/plugin/core/src/main/resources/monitoring-es.json index 05168fb51a168..9517c4a49c88b 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-es.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-es.json @@ -8,7 +8,6 @@ "index.format": 7, "index.codec": "best_compression" }, - "allow_auto_create": true, "mappings": { "_doc": { "date_detection": false, diff --git a/x-pack/plugin/core/src/main/resources/monitoring-kibana.json b/x-pack/plugin/core/src/main/resources/monitoring-kibana.json index b30c593963fb6..1f916625c37f3 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-kibana.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-kibana.json @@ -8,7 +8,6 @@ "index.format": 7, "index.codec": "best_compression" }, - "allow_auto_create": true, "mappings": { "_doc": { "dynamic": false, diff --git a/x-pack/plugin/core/src/main/resources/monitoring-logstash.json b/x-pack/plugin/core/src/main/resources/monitoring-logstash.json index f24b096b087b6..a3dc60ae803b9 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-logstash.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-logstash.json @@ -8,7 +8,6 @@ "index.format": 7, "index.codec": "best_compression" }, - "allow_auto_create": true, "mappings": { "_doc": { "dynamic": false, diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json index 009325d0d435f..bccad3e753678 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json @@ -4,7 +4,6 @@ "index_patterns" : [ ".ml-anomalies-*" ], - "allow_auto_create": true, "settings" : { "index" : { "translog" : { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json index ce42c65cd4c29..64635b658e3b2 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json @@ -12,7 +12,6 @@ "index.lifecycle.name": "${xpack.ml.index.lifecycle.name}", "index.lifecycle.rollover_alias": "${xpack.ml.index.lifecycle.rollover_alias}" }, - "allow_auto_create": true, "mappings" : { "_doc": { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json index 7756ffdd45b9f..8c6c352ab245a 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json @@ -4,7 +4,6 @@ "index_patterns" : [ ".ml-config" ], - "allow_auto_create": true, "settings" : { "index" : { "max_result_window" : "${xpack.ml.config.max_result_window}", diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json index 99dcb17b99baa..00f5eb2a90fe2 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json @@ -10,7 +10,6 @@ "auto_expand_replicas" : "0-1" } }, - "allow_auto_create": true, "mappings" : { "_doc": { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json index 042d7dea80ae4..19df45c52509c 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json @@ -10,7 +10,6 @@ "auto_expand_replicas" : "0-1" } }, - "allow_auto_create": true, "mappings" : { "_doc": { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json index f5f9a5990306f..e8ac3f1fcce6f 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json @@ -11,7 +11,6 @@ "hidden": true } }, - "allow_auto_create": true, "mappings" : { "_doc": { "_meta" : { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json index 35ea3ed218ce4..60f964fe13483 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json @@ -14,7 +14,6 @@ "index.refresh_interval": "1s", "index.format": 1 }, - "allow_auto_create": true, "mappings": { "_doc": { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/slm-history.json b/x-pack/plugin/core/src/main/resources/slm-history.json index b9320c170ecd4..efc61471d795f 100644 --- a/x-pack/plugin/core/src/main/resources/slm-history.json +++ b/x-pack/plugin/core/src/main/resources/slm-history.json @@ -57,6 +57,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for SLM history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/triggered-watches.json b/x-pack/plugin/core/src/main/resources/triggered-watches.json index 1cb43c51c6ef5..e11f7d9853186 100644 --- a/x-pack/plugin/core/src/main/resources/triggered-watches.json +++ b/x-pack/plugin/core/src/main/resources/triggered-watches.json @@ -37,6 +37,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for triggered watches indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json index 2f4433536d4ae..d9de434e08275 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json +++ b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json @@ -613,6 +613,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for watcher history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/watch-history.json b/x-pack/plugin/core/src/main/resources/watch-history.json index c5b380165fd78..7ee688da832d0 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history.json +++ b/x-pack/plugin/core/src/main/resources/watch-history.json @@ -564,6 +564,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for watcher history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/watches.json b/x-pack/plugin/core/src/main/resources/watches.json index 1d8c22d4da64d..4c1108c9f38ad 100644 --- a/x-pack/plugin/core/src/main/resources/watches.json +++ b/x-pack/plugin/core/src/main/resources/watches.json @@ -9,7 +9,6 @@ "index.format": 6, "index.priority": 800 }, - "allow_auto_create": true, "mappings": { "dynamic": "strict", "properties": { @@ -60,6 +59,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for watches indices", "managed": true diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 131bda16cd59f..90e7796385761 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -430,7 +430,7 @@ private boolean hasTemplate(final ClusterState clusterState, final String templa // FIXME this should use the IndexTemplateMetadataUpgrader private void putTemplate(String template, String source, ActionListener listener) { - logger.debug("installing template [{}]", template); + logger.warn("installing template [{}]", template); PutIndexTemplateRequest request = new PutIndexTemplateRequest(template).source(source, XContentType.JSON); assert !Thread.currentThread().isInterrupted() : "current thread has been interrupted before putting index template!!!"; diff --git a/x-pack/plugin/watcher/qa/rest/src/test/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml b/x-pack/plugin/watcher/qa/rest/src/test/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml index e9224638612b5..b9b9a39dab776 100644 --- a/x-pack/plugin/watcher/qa/rest/src/test/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml +++ b/x-pack/plugin/watcher/qa/rest/src/test/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml @@ -22,7 +22,7 @@ teardown: action.auto_create_index: null --- -"Test execute watch api with configured trigger data timestamps": +"Ensure watch is created even when auto_create_index is disabled": - do: watcher.put_watch: id: "test_watch" @@ -50,26 +50,3 @@ teardown: } - match: { _id: "test_watch" } - match: { created: true } - - - do: - watcher.execute_watch: - id: "test_watch" - body: > - { - "trigger_data" : { - "triggered_time" : "2012-12-12T12:12:12.120Z", - "scheduled_time" : "2000-12-12T12:12:12.120Z" - } - } - - - match: { watch_record.watch_id: "test_watch" } - - match: { watch_record.trigger_event.type: "manual" } - - match: { watch_record.trigger_event.triggered_time: "2012-12-12T12:12:12.120Z" } - - match: { watch_record.trigger_event.manual.schedule.scheduled_time: "2000-12-12T12:12:12.120Z" } - - match: { watch_record.state: "executed" } - - match: { watch_record.status.execution_state: "executed" } - - match: { watch_record.status.state.active: true } - - is_true: watch_record.node - - match: { watch_record.status.actions.indexme.ack.state: "ackable" } - - gt: { watch_record.result.execution_duration: 0 } - From 42cfc007eb7e2486a4ef11aa3fcf34264cbd20a7 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 2 Sep 2020 15:35:48 +0100 Subject: [PATCH 04/60] Remove logging --- .../org/elasticsearch/action/support/AutoCreateIndex.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index 5cdff7a16a992..4d40f55fe3ec9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -19,8 +19,6 @@ package org.elasticsearch.action.support; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -44,8 +42,6 @@ * a write operation is about to happen in a non existing index. */ public final class AutoCreateIndex { - private static final Logger logger = LogManager.getLogger(AutoCreateIndex.class); - public static final Setting AUTO_CREATE_INDEX_SETTING = new Setting<>("action.auto_create_index", "true", AutoCreate::new, Property.NodeScope, Setting.Property.Dynamic); @@ -68,9 +64,7 @@ public boolean shouldAutoCreate(String index, ClusterState state) { } // Templates can override the AUTO_CREATE_INDEX_SETTING setting - logger.warn("Looking for template for {}", index); final ComposableIndexTemplate template = findTemplate(index, state.metadata()); - logger.warn("Results: {}", template); if (template != null && template.getAllowAutoCreate()) { return true; } From 961cfcf792de36cc688c87271cc4c2dcba2e3c65 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 9 Sep 2020 12:17:39 +0100 Subject: [PATCH 05/60] Use AutoCreateIndex in AutoCreateAction --- .../indices/create/AutoCreateAction.java | 15 +++- .../action/bulk/TransportBulkAction.java | 69 +++++++--------- ...ActionIndicesThatCannotBeCreatedTests.java | 8 +- .../bulk/TransportBulkActionIngestTests.java | 7 +- .../action/bulk/TransportBulkActionTests.java | 20 ++--- .../bulk/TransportBulkActionTookTests.java | 11 --- .../snapshots/SnapshotResiliencyTests.java | 82 +++++++++---------- 7 files changed, 96 insertions(+), 116 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index 24934c30aede1..8bcdd93459cfa 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; +import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; @@ -65,16 +66,19 @@ public static final class TransportAction extends TransportMasterNodeAction v1 || v2)); - /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create - * that we'll use when we try to run the requests. */ + /* Step 2: filter the list of indices to find those that don't currently exist. */ final Map indicesThatCannotBeCreated = new HashMap<>(); Set autoCreateIndices = new HashSet<>(); ClusterState state = clusterService.state(); for (Map.Entry indexAndFlag : indices.entrySet()) { - boolean shouldAutoCreate; final String index = indexAndFlag.getKey(); - try { - shouldAutoCreate = shouldAutoCreate(index, state); - } catch (IndexNotFoundException e) { - shouldAutoCreate = false; - indicesThatCannotBeCreated.put(index, e); - } + boolean shouldAutoCreate = indexNameExpressionResolver.hasIndexAbstraction(index, state) == false; // We should only auto create if we are not requiring it to be an alias if (shouldAutoCreate && (indexAndFlag.getValue() == false)) { autoCreateIndices.add(index); @@ -262,7 +251,11 @@ public void onResponse(CreateIndexResponse result) { @Override public void onFailure(Exception e) { - if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) { + final Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof IndexNotFoundException) { + indicesThatCannotBeCreated.put(index, (IndexNotFoundException) e); + } + else if ((cause instanceof ResourceAlreadyExistsException) == false) { // fail all requests involving this index, if create didn't work for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest request = bulkRequest.requests.get(i); @@ -333,10 +326,6 @@ static void prohibitCustomRoutingOnDataStream(DocWriteRequest writeRequest, M } } - boolean shouldAutoCreate(String index, ClusterState state) { - return autoCreateIndex.shouldAutoCreate(index, state); - } - void createIndex(String index, TimeValue timeout, Version minNodeVersion, diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index b22edd0fa679b..2c8d0ce53a896 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -124,7 +125,7 @@ private void indicesThatCannotBeCreatedTestCase(Set expected, when(threadPool.executor(anyString())).thenReturn(direct); TransportBulkAction action = new TransportBulkAction(threadPool, mock(TransportService.class), clusterService, - null, null, mock(ActionFilters.class), null, null, + null, null, mock(ActionFilters.class), new IndexNameExpressionResolver(), new IndexingPressure(Settings.EMPTY)) { @Override void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener listener, @@ -132,11 +133,6 @@ void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, Action assertEquals(expected, indicesThatCannotBeCreated.keySet()); } - @Override - boolean shouldAutoCreate(String index, ClusterState state) { - return shouldAutoCreate.apply(index); - } - @Override void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener listener) { // If we try to create an index just immediately assume it worked diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index a3634c8458f59..7b577b8991d53 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -140,11 +140,8 @@ class TestTransportBulkAction extends TransportBulkAction { TestTransportBulkAction() { super(threadPool, transportService, clusterService, ingestService, - null, new ActionFilters(Collections.emptySet()), null, - new AutoCreateIndex( - SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), - new IndexNameExpressionResolver() - ), new IndexingPressure(SETTINGS) + null, new ActionFilters(Collections.emptySet()), new IndexNameExpressionResolver(), + new IndexingPressure(SETTINGS) ); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 8d39bf48a7b9f..d3fd0d247f7c3 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -19,6 +19,15 @@ package org.elasticsearch.action.bulk; +import static org.elasticsearch.action.bulk.TransportBulkAction.prohibitCustomRoutingOnDataStream; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.createDataStream; +import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +import java.util.Collections; +import java.util.concurrent.TimeUnit; + import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; @@ -28,7 +37,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterState; @@ -51,15 +59,6 @@ import org.junit.After; import org.junit.Before; -import java.util.Collections; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.action.bulk.TransportBulkAction.prohibitCustomRoutingOnDataStream; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.createDataStream; -import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - public class TransportBulkActionTests extends ESTestCase { /** Services needed by bulk action */ @@ -76,7 +75,6 @@ class TestTransportBulkAction extends TransportBulkAction { TestTransportBulkAction() { super(TransportBulkActionTests.this.threadPool, transportService, clusterService, null, null, new ActionFilters(Collections.emptySet()), new Resolver(), - new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver()), new IndexingPressure(Settings.EMPTY)); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index a93c6a56fe1ac..42371158b52d6 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -124,7 +123,6 @@ void doExecute(ActionType action, Request request, ActionListener Date: Thu, 10 Sep 2020 15:42:01 +0100 Subject: [PATCH 06/60] Imports --- .../action/bulk/TransportBulkAction.java | 36 ++++----- .../action/bulk/TransportBulkActionTests.java | 18 ++--- .../snapshots/SnapshotResiliencyTests.java | 78 +++++++++---------- 3 files changed, 66 insertions(+), 66 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 4375cdb1002a1..53ccbf0bd5ee3 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -19,24 +19,6 @@ package org.elasticsearch.action.bulk; -import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY; -import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; -import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicIntegerArray; -import java.util.function.LongSupplier; -import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SparseFixedBitSet; @@ -90,6 +72,24 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.function.LongSupplier; +import java.util.stream.Collectors; + +import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + /** * Groups bulk request items by shard, optionally creating non-existent indices and * delegates to {@link TransportShardBulkAction} for shard-level bulk execution diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index d3fd0d247f7c3..f1521b375a1ca 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -19,15 +19,6 @@ package org.elasticsearch.action.bulk; -import static org.elasticsearch.action.bulk.TransportBulkAction.prohibitCustomRoutingOnDataStream; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.createDataStream; -import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -import java.util.Collections; -import java.util.concurrent.TimeUnit; - import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; @@ -59,6 +50,15 @@ import org.junit.After; import org.junit.Before; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.action.bulk.TransportBulkAction.prohibitCustomRoutingOnDataStream; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.createDataStream; +import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + public class TransportBulkActionTests extends ESTestCase { /** Services needed by bulk action */ diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index da094fafccd19..0bfab4444cdfa 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -19,45 +19,6 @@ package org.elasticsearch.snapshots; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; -import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; -import static org.elasticsearch.env.Environment.PATH_HOME_SETTING; -import static org.elasticsearch.monitor.StatusInfo.Status.HEALTHY; -import static org.elasticsearch.node.Node.NODE_NAME_SETTING; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.either; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.iterableWithSize; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.Mockito.mock; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; @@ -231,6 +192,45 @@ import org.junit.After; import org.junit.Before; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; +import static org.elasticsearch.env.Environment.PATH_HOME_SETTING; +import static org.elasticsearch.monitor.StatusInfo.Status.HEALTHY; +import static org.elasticsearch.node.Node.NODE_NAME_SETTING; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + public class SnapshotResiliencyTests extends ESTestCase { private DeterministicTaskQueue deterministicTaskQueue; From b00297e5e09ab23a05a2058cabe5c1e86d3864a0 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 14 Sep 2020 11:57:15 +0100 Subject: [PATCH 07/60] Fixes --- .../indices/create/AutoCreateAction.java | 2 +- .../action/bulk/TransportBulkAction.java | 3 +- .../MetadataIndexTemplateService.java | 1 - ...ActionIndicesThatCannotBeCreatedTests.java | 57 ++++++++++++------- .../bulk/TransportBulkActionIngestTests.java | 3 +- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index 8bcdd93459cfa..0c0a0854267fb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -137,7 +137,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { } else { String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); - // This will throw an exception if creating this index is prohibited + // This will throw an exception if the index does not exist and creating it is prohibited final boolean shouldAutoCreate = autoCreateIndex.shouldAutoCreate(indexName, currentState); if (shouldAutoCreate == false) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 53ccbf0bd5ee3..3c088b64acf68 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -221,7 +221,8 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, ActionListe || request.versionType() == VersionType.EXTERNAL || request.versionType() == VersionType.EXTERNAL_GTE) .collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2)); - /* Step 2: filter the list of indices to find those that don't currently exist. */ + + // Step 2: filter the list of indices to find those that don't currently exist. final Map indicesThatCannotBeCreated = new HashMap<>(); Set autoCreateIndices = new HashSet<>(); ClusterState state = clusterService.state(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 2867aaf5c87e6..66a21beb3c1b5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -905,7 +905,6 @@ public static String findV2Template(Metadata metadata, String indexName, boolean final Map matchedTemplates = new HashMap<>(); for (Map.Entry entry : metadata.templatesV2().entrySet()) { final String name = entry.getKey(); - logger.warn("XXX: template v2 name = {}", name); final ComposableIndexTemplate template = entry.getValue(); if (isHidden == false) { final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index 2c8d0ce53a896..ede8622be515f 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -37,28 +37,28 @@ import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.VersionType; import org.elasticsearch.index.IndexingPressure; +import org.elasticsearch.index.VersionType; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.Arrays; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; import java.util.function.Function; import static java.util.Collections.emptySet; -import static java.util.Collections.singleton; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class TransportBulkActionIndicesThatCannotBeCreatedTests extends ESTestCase { + private static final Consumer noop = index -> {}; + public void testNonExceptional() { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); @@ -66,11 +66,11 @@ public void testNonExceptional() { bulkRequest.add(new DeleteRequest(randomAlphaOfLength(5))); bulkRequest.add(new UpdateRequest(randomAlphaOfLength(5), randomAlphaOfLength(5))); // Test emulating that index can be auto-created - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true); + indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true, noop); // Test emulating that index cannot be auto-created - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false); + indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false, noop); // Test emulating auto_create_index=true with some indices already created. - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean()); + indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean(), noop); } public void testAllFail() { @@ -79,7 +79,7 @@ public void testAllFail() { bulkRequest.add(new IndexRequest("can't")); bulkRequest.add(new DeleteRequest("do").version(0).versionType(VersionType.EXTERNAL)); bulkRequest.add(new UpdateRequest("nothin", randomAlphaOfLength(5))); - indicesThatCannotBeCreatedTestCase(new HashSet<>(Arrays.asList("no", "can't", "do", "nothin")), bulkRequest, index -> { + indicesThatCannotBeCreatedTestCase(Set.of("no", "can't", "do", "nothin"), bulkRequest, index -> true, index -> { throw new IndexNotFoundException("Can't make it because I say so"); }); } @@ -89,24 +89,16 @@ public void testSomeFail() { bulkRequest.add(new IndexRequest("ok")); bulkRequest.add(new IndexRequest("bad")); // Emulate auto_create_index=-bad,+* - indicesThatCannotBeCreatedTestCase(singleton("bad"), bulkRequest, index -> { - if (index.equals("bad")) { - throw new IndexNotFoundException("Can't make it because I say so"); - } - return true; - }); - // Emulate auto_create_index=false but the "ok" index already exists - indicesThatCannotBeCreatedTestCase(singleton("bad"), bulkRequest, index -> { + indicesThatCannotBeCreatedTestCase(Set.of("bad"), bulkRequest, index -> true, index -> { if (index.equals("bad")) { throw new IndexNotFoundException("Can't make it because I say so"); } - return false; }); } private void indicesThatCannotBeCreatedTestCase(Set expected, - BulkRequest bulkRequest, Function shouldAutoCreate) { + BulkRequest bulkRequest, Function shouldAutoCreate, Consumer simulateAutoCreate) { ClusterService clusterService = mock(ClusterService.class); ClusterState state = mock(ClusterState.class); when(state.getMetadata()).thenReturn(Metadata.EMPTY_METADATA); @@ -124,8 +116,15 @@ private void indicesThatCannotBeCreatedTestCase(Set expected, final ExecutorService direct = EsExecutors.newDirectExecutorService(); when(threadPool.executor(anyString())).thenReturn(direct); + final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver() { + @Override + public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) { + return shouldAutoCreate.apply(indexAbstraction) == false; + } + }; + TransportBulkAction action = new TransportBulkAction(threadPool, mock(TransportService.class), clusterService, - null, null, mock(ActionFilters.class), new IndexNameExpressionResolver(), + null, null, mock(ActionFilters.class), indexNameExpressionResolver, new IndexingPressure(Settings.EMPTY)) { @Override void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener listener, @@ -135,10 +134,24 @@ void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, Action @Override void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener listener) { - // If we try to create an index just immediately assume it worked - listener.onResponse(new CreateIndexResponse(true, true, index) {}); + try { + simulateAutoCreate.accept(index); + // If we try to create an index just immediately assume it worked + listener.onResponse(new CreateIndexResponse(true, true, index) { + }); + } catch (Exception e) { + listener.onFailure(e); + } } }; - action.doExecute(null, bulkRequest, null); + action.doExecute(null, bulkRequest, new ActionListener<>() { + @Override + public void onResponse(BulkResponse bulkItemResponses) {} + + @Override + public void onFailure(Exception e) { + throw new AssertionError(e); + } + }); } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 7b577b8991d53..88ca80b18b1c4 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -34,10 +34,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -45,7 +45,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; From ad9e699cb813c432dd4265569eb8605fae779aca Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 14 Sep 2020 13:42:23 +0100 Subject: [PATCH 08/60] Fix compile error --- .../xpack/cluster/routing/allocation/DataTierIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java index d20c7169d3cca..2e56c8800eb98 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java @@ -160,7 +160,7 @@ public void testTemplateOverridesDefaults() { Template t = new Template(Settings.builder() .put(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE, DataTier.DATA_WARM) .build(), null, null); - ComposableIndexTemplate ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null); + ComposableIndexTemplate ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null, null); client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); @@ -177,7 +177,7 @@ public void testTemplateOverridesDefaults() { t = new Template(Settings.builder() .putNull(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE) .build(), null, null); - ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null); + ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null, null); client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); From c42e3cebae0183aee5c03ea4e80423ce2d961d5b Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 14 Sep 2020 16:39:32 +0100 Subject: [PATCH 09/60] Tweak ComposableIndexTemplate serialisation under code changes land in 7.10 --- .../cluster/metadata/ComposableIndexTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 3e6451b10c526..f5934c2ddd13f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -138,7 +138,7 @@ public ComposableIndexTemplate(StreamInput in) throws IOException { } else { this.dataStreamTemplate = null; } - if (in.getVersion().onOrAfter(Version.V_7_10_0)) { + if (in.getVersion().after(Version.V_7_10_0)) { this.allowAutoCreate = in.readOptionalBoolean(); } else { this.allowAutoCreate = null; @@ -205,7 +205,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_9_0)) { out.writeOptionalWriteable(dataStreamTemplate); } - if (out.getVersion().onOrAfter(Version.V_7_10_0)) { + if (out.getVersion().after(Version.V_7_10_0)) { out.writeOptionalBoolean(allowAutoCreate); } } From 6a689b26b0a7f7dbb3769457f6158055d993077c Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 16 Sep 2020 11:23:00 +0100 Subject: [PATCH 10/60] Fix auto create index bug --- .../action/admin/indices/create/AutoCreateAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index 0c0a0854267fb..c818510d1d01e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -136,6 +136,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { return clusterState; } else { String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); + indexNameRef.set(indexName); // This will throw an exception if the index does not exist and creating it is prohibited final boolean shouldAutoCreate = autoCreateIndex.shouldAutoCreate(indexName, currentState); @@ -145,7 +146,6 @@ public ClusterState execute(ClusterState currentState) throws Exception { return currentState; } - indexNameRef.set(indexName); CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest(request.cause(), indexName, request.index()) .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()); From 2d54d2c6c79b007a813d367c901fb49dae35921e Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 16 Sep 2020 13:54:14 +0100 Subject: [PATCH 11/60] Test fix --- .../action/bulk/BulkProcessorClusterSettingsIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java index c9494aeb89f94..741817dfd50a3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java @@ -49,7 +49,8 @@ public void testBulkProcessorAutoCreateRestrictions() { assertTrue("Missing index should have been flagged", responses[1].isFailed()); assertThat( responses[1].getFailureMessage(), - equalTo("[wontwork] org.elasticsearch.index.IndexNotFoundException: no such index [wontwork]")); + equalTo("[wontwork] org.elasticsearch.index.IndexNotFoundException: no such index [wontwork]" + + " and [action.auto_create_index] is [false]")); assertFalse("Operation on existing index should succeed", responses[2].isFailed()); } } From 85abcbc721a3cf86a8d8b325e5b6038ed89dd98f Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 16 Sep 2020 14:47:01 +0100 Subject: [PATCH 12/60] Handle null Boolean --- .../java/org/elasticsearch/action/support/AutoCreateIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index 4d40f55fe3ec9..a276061f64678 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -65,7 +65,7 @@ public boolean shouldAutoCreate(String index, ClusterState state) { // Templates can override the AUTO_CREATE_INDEX_SETTING setting final ComposableIndexTemplate template = findTemplate(index, state.metadata()); - if (template != null && template.getAllowAutoCreate()) { + if (template != null && template.getAllowAutoCreate() != null && template.getAllowAutoCreate()) { return true; } From a3e3691da0d2ac82251d99884035afcd5d3624b3 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 18 Sep 2020 12:08:29 +0100 Subject: [PATCH 13/60] Imports --- .../snapshots/SnapshotResiliencyTests.java | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 0bfab4444cdfa..ec31d7a210b0f 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -65,6 +65,7 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction; import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction; import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.TransportBulkAction; @@ -81,6 +82,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.PlainActionFuture; @@ -150,7 +152,6 @@ import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; @@ -192,24 +193,6 @@ import org.junit.After; import org.junit.Before; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; -import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; -import static org.elasticsearch.env.Environment.PATH_HOME_SETTING; -import static org.elasticsearch.monitor.StatusInfo.Status.HEALTHY; -import static org.elasticsearch.node.Node.NODE_NAME_SETTING; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.either; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.iterableWithSize; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.Mockito.mock; - import java.io.IOException; import java.nio.file.Path; import java.util.Collection; @@ -231,6 +214,24 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; +import static org.elasticsearch.env.Environment.PATH_HOME_SETTING; +import static org.elasticsearch.monitor.StatusInfo.Status.HEALTHY; +import static org.elasticsearch.node.Node.NODE_NAME_SETTING; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Mockito.mock; + public class SnapshotResiliencyTests extends ESTestCase { private DeterministicTaskQueue deterministicTaskQueue; From 9421140135bd9c79a1be9f0cc740b3ad670669e8 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 18 Sep 2020 16:13:19 +0100 Subject: [PATCH 14/60] Checkstyle --- .../org/elasticsearch/snapshots/SnapshotResiliencyTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index ec31d7a210b0f..d40a3c909ee37 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -65,7 +65,6 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction; import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction; import org.elasticsearch.action.bulk.BulkAction; -import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.TransportBulkAction; @@ -82,7 +81,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.PlainActionFuture; @@ -152,6 +150,7 @@ import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; From ebb42d4a1085279f04e5f82c41db574a56bd871c Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 18 Sep 2020 16:26:59 +0100 Subject: [PATCH 15/60] Tweaks --- .../action/support/AutoCreateIndex.java | 4 +- .../action/support/AutoCreateIndexTests.java | 38 ++++++++++++++++--- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index a276061f64678..3f823bc6fb3ea 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -102,8 +102,8 @@ void setAutoCreate(AutoCreate autoCreate) { this.autoCreate = autoCreate; } - private ComposableIndexTemplate findTemplate(String name, Metadata metadata) { - final String templateName = MetadataIndexTemplateService.findV2Template(metadata, name, false); + private ComposableIndexTemplate findTemplate(String indexName, Metadata metadata) { + final String templateName = MetadataIndexTemplateService.findV2Template(metadata, indexName, false); return metadata.templatesV2().get(templateName); } diff --git a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java index c7563983e9d13..16e08fabb3fd3 100644 --- a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java @@ -193,10 +193,38 @@ public void testUpdate() { } /** - * Check that if a template matches the index to be created, but that template does not have the allow_auto_create - * setting, then it does not override the global setting. + * Check that if a template matches the index to be created, but that template does not have a value + * for the allow_auto_create setting at all, then it does not cause the auto-create logic to trip over + * on a null value. */ - public void testTemplateSettingsDoesNotOverride() { + public void testCanHandleNullAutoCreateSettingInTemplate() { + String randomIndex = randomAlphaOfLengthBetween(2, 10); + final ComposableIndexTemplate template = new ComposableIndexTemplate( + List.of(randomIndex.charAt(0) + "*"), + null, + List.of(), + null, + null, + Map.of(), + null, + null + ); + + final Metadata metadata = Metadata.builder().indexTemplates(Map.of("test_template", template)).build(); + final ClusterState clusterState = ClusterState.builder(buildClusterState()).metadata(metadata).build(); + + Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build(); + AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings); + IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> + autoCreateIndex.shouldAutoCreate(randomIndex, clusterState)); + assertEquals("no such index [" + randomIndex + "] and [action.auto_create_index] is [false]", e.getMessage()); + } + + /** + * Check that if a template matches the index to be created, but that template has the allow_auto_create + * setting turned off, then it does not override the global setting. + */ + public void testDisabledAutoCreateTemplateSettingDoesNotOverride() { String randomIndex = randomAlphaOfLengthBetween(2, 10); final ComposableIndexTemplate template = new ComposableIndexTemplate( List.of(randomIndex.charAt(0) + "*"), @@ -221,9 +249,9 @@ public void testTemplateSettingsDoesNotOverride() { /** * Check that if a template matches the index to be created, and that template has the allow_auto_create - * setting, then it overrides the global setting. + * setting enabled, then it overrides the global setting. */ - public void testTemplateSettingsOverride() { + public void testEnabledAutoCreateTemplateSettingDoesOverride() { String randomIndex = randomAlphaOfLengthBetween(2, 10); final ComposableIndexTemplate template = new ComposableIndexTemplate( List.of(randomIndex.charAt(0) + "*"), From bfe8c8c4666ef56323d504747fd296e796121ccb Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 18 Sep 2020 16:52:14 +0100 Subject: [PATCH 16/60] Post-merge fix --- .../elasticsearch/xpack/datastreams/DataTierDataStreamIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java index 8c2025e6855f4..c5a75522ae589 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java @@ -43,7 +43,8 @@ public void testDefaultDataStreamAllocateToHot() { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ); client().execute( PutComposableIndexTemplateAction.INSTANCE, From 2694fb2776c4f710f9159bf85c5412d6ff535af9 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 18 Sep 2020 17:03:07 +0100 Subject: [PATCH 17/60] Update synthentics template to add allow_auto_create --- x-pack/plugin/core/src/main/resources/synthetics-template.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/core/src/main/resources/synthetics-template.json b/x-pack/plugin/core/src/main/resources/synthetics-template.json index 9bbbe0ef9e3df..1036a29cf5a9d 100644 --- a/x-pack/plugin/core/src/main/resources/synthetics-template.json +++ b/x-pack/plugin/core/src/main/resources/synthetics-template.json @@ -6,6 +6,7 @@ "synthetics-mappings", "synthetics-settings" ], + "allow_auto_create": true, "_meta": { "description": "default synthetics template installed by x-pack", "managed": true From 39abb3816a8884705ae406a2b6be9fc33d9a7217 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 22 Sep 2020 10:23:16 +0100 Subject: [PATCH 18/60] Address review feedback --- .../elasticsearch/cluster/metadata/ComposableIndexTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index f5934c2ddd13f..cb23d51ab6765 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -138,7 +138,7 @@ public ComposableIndexTemplate(StreamInput in) throws IOException { } else { this.dataStreamTemplate = null; } - if (in.getVersion().after(Version.V_7_10_0)) { + if (in.getVersion().after(Version.V_8_0_0)) { this.allowAutoCreate = in.readOptionalBoolean(); } else { this.allowAutoCreate = null; From 4b9034aea89ed39b671e4f59f608001edcb91f31 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 22 Sep 2020 10:45:17 +0100 Subject: [PATCH 19/60] Address review feedback --- .../xpack/monitoring/exporter/local/LocalExporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 90e7796385761..131bda16cd59f 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -430,7 +430,7 @@ private boolean hasTemplate(final ClusterState clusterState, final String templa // FIXME this should use the IndexTemplateMetadataUpgrader private void putTemplate(String template, String source, ActionListener listener) { - logger.warn("installing template [{}]", template); + logger.debug("installing template [{}]", template); PutIndexTemplateRequest request = new PutIndexTemplateRequest(template).source(source, XContentType.JSON); assert !Thread.currentThread().isInterrupted() : "current thread has been interrupted before putting index template!!!"; From 2e9a4c716a8e432701e42a0c8065dccccc2bf030 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 22 Sep 2020 14:48:41 +0100 Subject: [PATCH 20/60] Specify the same version for ser/deser Honestly, what was I thinking. --- .../elasticsearch/cluster/metadata/ComposableIndexTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index cb23d51ab6765..14fbcfb2aceab 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -205,7 +205,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_9_0)) { out.writeOptionalWriteable(dataStreamTemplate); } - if (out.getVersion().after(Version.V_7_10_0)) { + if (out.getVersion().after(Version.V_8_0_0)) { out.writeOptionalBoolean(allowAutoCreate); } } From 1d0770988fe672838603ce7aa7e15263e1dd90e4 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 24 Sep 2020 12:47:55 +0100 Subject: [PATCH 21/60] Fix stupid typo --- .../cluster/metadata/ComposableIndexTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 14fbcfb2aceab..c34b52db935d9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -138,7 +138,7 @@ public ComposableIndexTemplate(StreamInput in) throws IOException { } else { this.dataStreamTemplate = null; } - if (in.getVersion().after(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { this.allowAutoCreate = in.readOptionalBoolean(); } else { this.allowAutoCreate = null; @@ -205,7 +205,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_9_0)) { out.writeOptionalWriteable(dataStreamTemplate); } - if (out.getVersion().after(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { out.writeOptionalBoolean(allowAutoCreate); } } From 93e2440d98bd07edfc7dd22ec8f85ebdb805858f Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 28 Sep 2020 12:08:19 +0100 Subject: [PATCH 22/60] Address review comments --- .../action/support/AutoCreateIndex.java | 9 ++++-- .../metadata/ComposableIndexTemplate.java | 6 ++-- .../action/support/AutoCreateIndexTests.java | 31 +++++++++++++++++-- .../90_auto_create_index.yml | 0 4 files changed, 40 insertions(+), 6 deletions(-) rename x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/{execute_watch => put_watch}/90_auto_create_index.yml (100%) diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index 3f823bc6fb3ea..379afc046edc0 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -65,8 +65,13 @@ public boolean shouldAutoCreate(String index, ClusterState state) { // Templates can override the AUTO_CREATE_INDEX_SETTING setting final ComposableIndexTemplate template = findTemplate(index, state.metadata()); - if (template != null && template.getAllowAutoCreate() != null && template.getAllowAutoCreate()) { - return true; + if (template != null && template.getAllowAutoCreate() != null) { + if (template.getAllowAutoCreate()) { + return true; + } else { + // An explicit false value overrides AUTO_CREATE_INDEX_SETTING + throw new IndexNotFoundException("composable template " + template.indexPatterns() + " forbids index auto creation"); + } } // One volatile read, so that all checks are done against the same instance: diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index c34b52db935d9..28efd3e0593d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -57,6 +57,8 @@ public class ComposableIndexTemplate extends AbstractDiffable PARSER = new ConstructingObjectParser<>("index_template", false, @@ -138,7 +140,7 @@ public ComposableIndexTemplate(StreamInput in) throws IOException { } else { this.dataStreamTemplate = null; } - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(ALLOW_AUTO_CREATE_VERSION)) { this.allowAutoCreate = in.readOptionalBoolean(); } else { this.allowAutoCreate = null; @@ -205,7 +207,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_9_0)) { out.writeOptionalWriteable(dataStreamTemplate); } - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(ALLOW_AUTO_CREATE_VERSION)) { out.writeOptionalBoolean(allowAutoCreate); } } diff --git a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java index 16e08fabb3fd3..13a4b23e59c36 100644 --- a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java @@ -192,6 +192,33 @@ public void testUpdate() { assertThat(autoCreateIndex.getAutoCreate().getExpressions().get(0).v1(), equalTo("logs-*")); } + /** + * Check that if a template matches the index to be created, but that template does not have a value + * for the allow_auto_create setting at all, and the auto_create_index setting matches the index + * to be created, then the null in the template does not override the auto_create_index logic and the + * index can be created. + */ + public void testNullAllowAutoCreateInTemplateDoesNotOverrideMatchingAutoCreateIndexSetting() { + String randomIndex = randomAlphaOfLengthBetween(2, 10); + final ComposableIndexTemplate template = new ComposableIndexTemplate( + List.of(randomIndex.charAt(0) + "*"), + null, + List.of(), + null, + null, + Map.of(), + null, + null + ); + + final Metadata metadata = Metadata.builder().indexTemplates(Map.of("test_template", template)).build(); + final ClusterState clusterState = ClusterState.builder(buildClusterState()).metadata(metadata).build(); + + Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), randomIndex.charAt(0) + "*").build(); + AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings); + assertTrue(autoCreateIndex.shouldAutoCreate(randomIndex, clusterState)); + } + /** * Check that if a template matches the index to be created, but that template does not have a value * for the allow_auto_create setting at all, then it does not cause the auto-create logic to trip over @@ -222,7 +249,7 @@ public void testCanHandleNullAutoCreateSettingInTemplate() { /** * Check that if a template matches the index to be created, but that template has the allow_auto_create - * setting turned off, then it does not override the global setting. + * setting turned off, then it overrides the global setting. */ public void testDisabledAutoCreateTemplateSettingDoesNotOverride() { String randomIndex = randomAlphaOfLengthBetween(2, 10); @@ -244,7 +271,7 @@ public void testDisabledAutoCreateTemplateSettingDoesNotOverride() { AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings); IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> autoCreateIndex.shouldAutoCreate(randomIndex, clusterState)); - assertEquals("no such index [" + randomIndex + "] and [action.auto_create_index] is [false]", e.getMessage()); + assertEquals("no such index [composable template [" + randomIndex.charAt(0) + "*] forbids index auto creation]", e.getMessage()); } /** diff --git a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/90_auto_create_index.yml similarity index 100% rename from x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/execute_watch/90_auto_create_index.yml rename to x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/90_auto_create_index.yml From 4a2e64c85312130a20c2e97cc40e14a58fcde260 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 28 Sep 2020 12:15:25 +0100 Subject: [PATCH 23/60] Increment some template versions due to the new setting --- .../core/slm/history/SnapshotLifecycleTemplateRegistry.java | 3 ++- .../watcher/support/WatcherIndexTemplateRegistryField.java | 3 ++- .../xpack/ilm/history/ILMHistoryTemplateRegistry.java | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index 0628dcc4b8f93..818a30c3acfbc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -37,7 +37,8 @@ public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry { // version 1: initial // version 2: converted to hidden index // version 3: templates moved to composable templates - public static final int INDEX_TEMPLATE_VERSION = 3; + // version 4: add `allow_auto_create` setting + public static final int INDEX_TEMPLATE_VERSION = 4; public static final String SLM_TEMPLATE_VERSION_VARIABLE = "xpack.slm.template.version"; public static final String SLM_TEMPLATE_NAME = ".slm-history"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java index 82994613a8ade..6a072e25df245 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java @@ -17,8 +17,9 @@ public final class WatcherIndexTemplateRegistryField { // version 10: add support for foreach path in actions // version 11: watch history indices are hidden // version 12: templates changed to composable templates + // version 13: add `allow_auto_create` setting // Note: if you change this, also inform the kibana team around the watcher-ui - public static final int INDEX_TEMPLATE_VERSION = 12; + public static final int INDEX_TEMPLATE_VERSION = 13; public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION; public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches"; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index 08494b7fb70e8..c8835acbd0b39 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -29,7 +29,8 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry { // version 1: initial // version 2: convert to hidden index // version 3: templates moved to composable templates - public static final int INDEX_TEMPLATE_VERSION = 3; + // version 4: add `allow_auto_create` setting + public static final int INDEX_TEMPLATE_VERSION = 4; public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version"; public static final String ILM_TEMPLATE_NAME = "ilm-history"; From 82ad6e46e7268ad7b11da38cfd00eded94fc4fde Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 28 Sep 2020 14:02:29 +0100 Subject: [PATCH 24/60] Add Java REST tests for auto-creating indices --- .../elasticsearch/http/AutoCreateIndexIT.java | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java new file mode 100644 index 0000000000000..d75c0aef0f74c --- /dev/null +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java @@ -0,0 +1,125 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.http; + +import org.elasticsearch.action.support.AutoCreateIndex; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.io.InputStreamReader; + +import static org.hamcrest.Matchers.containsString; + +public class AutoCreateIndexIT extends ESRestTestCase { + + /** + * Check that setting {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} to false + * disable the automatic creation on indices. + */ + public void testCannotAutoCreateIndexWhenDisabled() throws IOException { + configureAutoCreateIndex(false); + + // Attempt to add a document to a non-existing index. Auto-creating the index should fail owing to the setting above. + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456"); + indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }"); + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent())), + containsString("no such index [recipe_kr] and [action.auto_create_index] is [false]") + ); + } + + /** + * Check that automatically creating an index is allowed, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} + * is false, when the index name matches a template and that template has allow_auto_create + * set to true. + */ + public void testCanAutoCreateTemplateWhenAllowedByTemplate() throws IOException { + configureAutoCreateIndex(false); + + createTemplateWithAllowAutoCreate(true); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + assertOK(this.indexDocument()); + } + + /** + * Check that automatically creating an index is disallowed when the index name matches a template and that template has + * allow_auto_create explicitly to false, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} + * is set to true. + */ + public void testCannotAutoCreateTemplateWhenDisallowedByTemplate() throws IOException { + configureAutoCreateIndex(true); + + createTemplateWithAllowAutoCreate(false); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent())), + containsString("no such index [composable template [recipe*] forbids index auto creation]") + ); + } + + + private void configureAutoCreateIndex(boolean value) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("transient") + .field(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), value) + .endObject() + .endObject(); + + final Request settingsRequest = new Request("PUT", "_cluster/settings"); + settingsRequest.setJsonEntity(Strings.toString(builder)); + final Response settingsResponse = client().performRequest(settingsRequest); + assertOK(settingsResponse); + } + + private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .array("index_patterns", "recipe*") + .field("allow_auto_create", allowAutoCreate) + .endObject(); + + final Request createTemplateRequest = new Request("PUT", "_index_template/recipe_template"); + createTemplateRequest.setJsonEntity(Strings.toString(builder)); + final Response createTemplateResponse = client().performRequest(createTemplateRequest); + assertOK(createTemplateResponse); + } + + private Response indexDocument() throws IOException { + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456"); + indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }"); + return client().performRequest(indexDocumentRequest); + } +} From 6022b3c35e5996b1cfe6a44db64dde824ea86292 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 28 Sep 2020 14:53:12 +0100 Subject: [PATCH 25/60] Checkstyle --- .../test/java/org/elasticsearch/http/AutoCreateIndexIT.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java index d75c0aef0f74c..18369f5ac368e 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.io.InputStreamReader; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.hamcrest.Matchers.containsString; public class AutoCreateIndexIT extends ESRestTestCase { @@ -49,7 +50,7 @@ public void testCannotAutoCreateIndexWhenDisabled() throws IOException { final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); assertThat( - Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent())), + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), containsString("no such index [recipe_kr] and [action.auto_create_index] is [false]") ); } @@ -84,7 +85,7 @@ public void testCannotAutoCreateTemplateWhenDisallowedByTemplate() throws IOExce final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); assertThat( - Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent())), + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), containsString("no such index [composable template [recipe*] forbids index auto creation]") ); } From 6127caf1c0fc37feaca3f3bc6446df140bd77150 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 12 Oct 2020 14:33:45 +0100 Subject: [PATCH 26/60] Update component template docs --- .../indices/put-component-template.asciidoc | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index c5c48cb521fac..4a0c189a9f424 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -4,10 +4,10 @@ Put component template ++++ -Creates or updates a component template. -Component templates are building blocks for constructing <>. -that specify index <>, <>, -and <>. +Creates or updates a component template. +Component templates are building blocks for constructing <>. +that specify index <>, <>, +and <>. [source,console] -------------------------------------------------- @@ -55,10 +55,10 @@ DELETE _component_template/template_* [[put-component-template-api-desc]] ==== {api-description-title} -An index template can be composed of multiple component templates. +An index template can be composed of multiple component templates. To use a component template, specify it in an index template's `composed_of` list. -Component templates are only applied to new data streams and indices -as part of a matching index template. +Component templates are only applied to new data streams and indices +as part of a matching index template. Settings and mappings specified directly in the index template or the <> request override any settings or mappings specified in a component template. @@ -112,6 +112,16 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings] Version number used to manage component templates externally. This number is not automatically generated or incremented by {es}. +`allow_auto_create`:: +(Optional, boolean) +This setting overrides the value of the +<> cluster setting. If set to +`true` in a template, then indices can be automatically created using that +template even if auto-creation of indices is disabled via +`actions.auto_create_index`. If set to `false`, then indices matching the +template must always be explicitly created, and may never be automatically +created. + `_meta`:: (Optional, object) Optional user metadata about the component template. May have any contents. @@ -157,7 +167,7 @@ To be applied, a component template must be included in an index template's `com [[component-templates-version]] ===== Component template versioning -You can use the `version` parameter to add a version number to a component template. +You can use the `version` parameter to add a version number to a component template. External systems can use these version numbers to simplify template management. The `version` parameter is optional and not automatically generated or used by {es}. @@ -182,7 +192,7 @@ To check the `version`, you can use the < Date: Mon, 12 Oct 2020 15:14:52 +0100 Subject: [PATCH 27/60] Checkstyle --- .../action/bulk/TransportBulkActionIngestTests.java | 1 - .../org/elasticsearch/action/bulk/TransportBulkActionTests.java | 1 - 2 files changed, 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index ecf507740b1af..691c9bfe17eff 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index a363b17f6761b..75bfcad1a9caf 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterState; From c0394e4da0efacbf0d3eaee45de6f47e5ff784f2 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 13 Oct 2020 10:17:22 +0100 Subject: [PATCH 28/60] Docs tweak --- docs/reference/indices/put-component-template.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index 4a0c189a9f424..cddc376394239 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -118,7 +118,7 @@ This setting overrides the value of the <> cluster setting. If set to `true` in a template, then indices can be automatically created using that template even if auto-creation of indices is disabled via -`actions.auto_create_index`. If set to `false`, then indices matching the +`actions.auto_create_index`. If set to `false`, then indices matching the template must always be explicitly created, and may never be automatically created. From 61ab28ecaaed2915c7b1002434575d9410cd3bca Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 19 Oct 2020 11:35:00 +0100 Subject: [PATCH 29/60] Make data stream subject to auto_create_index --- .../indices/create/AutoCreateAction.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index b7eb2c461b7db..00966f007c125 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -116,7 +116,17 @@ protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { @Override public ClusterState execute(ClusterState currentState) throws Exception { - DataStreamTemplate dataStreamTemplate = resolveAutoCreateDataStream(request, currentState.metadata()); + final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); + + // This will throw an exception if the index or data stream does not exist and creating it is prohibited. + final boolean shouldAutoCreate = autoCreateIndex.shouldAutoCreate(indexName, currentState); + + if (shouldAutoCreate == false) { + // The index or data stream already exists. + return currentState; + } + + final DataStreamTemplate dataStreamTemplate = resolveAutoCreateDataStream(request, currentState.metadata()); if (dataStreamTemplate != null) { CreateDataStreamClusterStateUpdateRequest createRequest = new CreateDataStreamClusterStateUpdateRequest( request.index(), request.masterNodeTimeout(), request.timeout()); @@ -124,17 +134,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { indexNameRef.set(clusterState.metadata().dataStreams().get(request.index()).getIndices().get(0).getName()); return clusterState; } else { - String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); indexNameRef.set(indexName); - - // This will throw an exception if the index does not exist and creating it is prohibited - final boolean shouldAutoCreate = autoCreateIndex.shouldAutoCreate(indexName, currentState); - - if (shouldAutoCreate == false) { - // The index already exists. - return currentState; - } - CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest(request.cause(), indexName, request.index()) .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()); From 6c4c81769993bfc544e0909cf6484e98f4f7a740 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 19 Oct 2020 13:44:31 +0100 Subject: [PATCH 30/60] Add tests for auto-creating data streams --- .../elasticsearch/http/AutoCreateIndexIT.java | 4 +- .../datastreams/AutoCreateDataStreamIT.java | 127 ++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java index 18369f5ac368e..bc8c8c8bfb9ee 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/AutoCreateIndexIT.java @@ -60,7 +60,7 @@ public void testCannotAutoCreateIndexWhenDisabled() throws IOException { * is false, when the index name matches a template and that template has allow_auto_create * set to true. */ - public void testCanAutoCreateTemplateWhenAllowedByTemplate() throws IOException { + public void testCanAutoCreateIndexWhenAllowedByTemplate() throws IOException { configureAutoCreateIndex(false); createTemplateWithAllowAutoCreate(true); @@ -75,7 +75,7 @@ public void testCanAutoCreateTemplateWhenAllowedByTemplate() throws IOException * allow_auto_create explicitly to false, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} * is set to true. */ - public void testCannotAutoCreateTemplateWhenDisallowedByTemplate() throws IOException { + public void testCannotAutoCreateIndexWhenDisallowedByTemplate() throws IOException { configureAutoCreateIndex(true); createTemplateWithAllowAutoCreate(false); diff --git a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java new file mode 100644 index 0000000000000..f10731c7f51dd --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.xpack.datastreams; + +import org.elasticsearch.action.support.AutoCreateIndex; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.time.Instant; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.Matchers.containsString; + +public class AutoCreateDataStreamIT extends ESRestTestCase { + + /** + * Check that setting {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} to false + * disable the automatic creation on data streams. + */ + public void testCannotAutoCreateDataStreamWhenDisabled() throws IOException { + configureAutoCreateIndex(false); + + // Attempt to add a document to a non-existing data stream. Auto-creating the data stream should fail owing to the setting above. + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456"); + indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }"); + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), + containsString("no such index [recipe_kr] and [action.auto_create_index] is [false]") + ); + } + + /** + * Check that automatically creating an index is allowed, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} + * is false, when the index name matches a template and that template has allow_auto_create + * set to true. + */ + public void testCanAutoCreateDataStreamWhenAllowedByTemplate() throws IOException { + configureAutoCreateIndex(false); + + createTemplateWithAllowAutoCreate(true); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + assertOK(this.indexDocument()); + } + + /** + * Check that automatically creating a data stream is disallowed when the data stream name matches a template and that template has + * allow_auto_create explicitly to false, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} + * is set to true. + */ + public void testCannotAutoCreateDataStreamWhenDisallowedByTemplate() throws IOException { + configureAutoCreateIndex(true); + + createTemplateWithAllowAutoCreate(false); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), + containsString("no such index [composable template [recipe*] forbids index auto creation]") + ); + } + + private void configureAutoCreateIndex(boolean value) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("transient") + .field(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), value) + .endObject() + .endObject(); + + final Request settingsRequest = new Request("PUT", "_cluster/settings"); + settingsRequest.setJsonEntity(Strings.toString(builder)); + final Response settingsResponse = client().performRequest(settingsRequest); + assertOK(settingsResponse); + } + + private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .array("index_patterns", "recipe*") + .field("allow_auto_create", allowAutoCreate) + .startObject("data_stream").endObject() + .endObject(); + + final Request createTemplateRequest = new Request("PUT", "_index_template/recipe_template"); + createTemplateRequest.setJsonEntity(Strings.toString(builder)); + final Response createTemplateResponse = client().performRequest(createTemplateRequest); + assertOK(createTemplateResponse); + } + + private Response indexDocument() throws IOException { + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc"); + indexDocumentRequest.setJsonEntity("{ \"@timestamp\": \"" + Instant.now() + "\", \"name\": \"Kimchi\" }"); + return client().performRequest(indexDocumentRequest); + } +} From ed692c7d595e6caa7d3e7a807b49825eda4a14bf Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 19 Oct 2020 14:23:34 +0100 Subject: [PATCH 31/60] Formatting --- .../xpack/datastreams/AutoCreateDataStreamIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java index f10731c7f51dd..178d8d959f3e6 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java @@ -110,7 +110,8 @@ private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws I .startObject() .array("index_patterns", "recipe*") .field("allow_auto_create", allowAutoCreate) - .startObject("data_stream").endObject() + .startObject("data_stream") + .endObject() .endObject(); final Request createTemplateRequest = new Request("PUT", "_index_template/recipe_template"); From 4f051af5c0842e944de6e315b12433621e3d143b Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 19 Oct 2020 15:52:27 -0400 Subject: [PATCH 32/60] Fix max/min aggs for unsigned_long (#63904) Max and min aggs were producing wrong results for unsigned_long field if field was indexed. If field is indexed for max/min aggs instead of field data, we use values from indexed Points, values of which are derived using method pointReaderIfPossible. Before UnsignedLongFieldType#pointReaderIfPossible was incorrectly producing values, as it failed to shift them back to original values. This patch fixes method pointReaderIfPossible to produce correct original values. Relates to #60050 --- .../unsignedlong/UnsignedLongFieldMapper.java | 7 +++++-- .../UnsignedLongLeafFieldData.java | 2 +- .../xpack/unsignedlong/UnsignedLongTests.java | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 0ffc98edef592..6fc2d9a418cf5 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -49,6 +49,8 @@ import java.util.function.Function; import java.util.function.Supplier; +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongLeafFieldData.convertUnsignedLongToDouble; + public class UnsignedLongFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "unsigned_long"; @@ -272,7 +274,8 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { @Override public Function pointReaderIfPossible() { if (isSearchable()) { - return (value) -> LongPoint.decodeDimension(value, 0); + // convert from the shifted value back to the original value + return (value) -> convertUnsignedLongToDouble(LongPoint.decodeDimension(value, 0)); } return null; } @@ -525,7 +528,7 @@ private static long parseUnsignedLong(Object value) { } /** - * Convert an unsigned long to the singed long by subtract 2^63 from it + * Convert an unsigned long to the signed long by subtract 2^63 from it * @param value – unsigned long value in the range [0; 2^64-1], values greater than 2^63-1 are negative * @return signed long value in the range [-2^63; 2^63-1] */ diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java index 001506bbb7da5..d1729f937931e 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -112,7 +112,7 @@ public Object nextValue() throws IOException { }; } - private static double convertUnsignedLongToDouble(long value) { + static double convertUnsignedLongToDouble(long value) { if (value < 0L) { return sortableSignedLongToUnsigned(value); // add 2 ^ 63 } else { diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java index c31dfe987dd52..88a5183d9f4c9 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java @@ -21,7 +21,9 @@ import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.range.Range; import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.metrics.Min; import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.aggregations.metrics.Max; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; @@ -36,6 +38,8 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.range; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.elasticsearch.search.aggregations.AggregationBuilders.max; +import static org.elasticsearch.search.aggregations.AggregationBuilders.min; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; @@ -273,6 +277,20 @@ public void testAggs() { double expectedSum = Arrays.stream(values).mapToDouble(Number::doubleValue).sum(); assertEquals(expectedSum, sum.getValue(), 0.001); } + // max agg + { + SearchResponse response = client().prepareSearch("idx").setSize(0).addAggregation(max("ul_max").field("ul_field")).get(); + assertSearchResponse(response); + Max max = response.getAggregations().get("ul_max"); + assertEquals(1.8446744073709551615E19, max.getValue(), 0.001); + } + // min agg + { + SearchResponse response = client().prepareSearch("idx").setSize(0).addAggregation(min("ul_min").field("ul_field")).get(); + assertSearchResponse(response); + Min min = response.getAggregations().get("ul_min"); + assertEquals(0, min.getValue(), 0.001); + } } public void testSortDifferentFormatsShouldFail() { From 44d7d6619a680ab72ebb7493e436e2a9451f2b6b Mon Sep 17 00:00:00 2001 From: William Brafford Date: Mon, 19 Oct 2020 17:18:15 -0400 Subject: [PATCH 33/60] Add APM configuration index to Kibana system indices (#63756) * Add APM index to Kibana system indices, making it accessible through the _kibana endpoint and giving it the same access privileges as the other Kibana system indices. * Parameterize kibana system index tests by index name --- .../kibana/KibanaSystemIndexIT.java | 117 ++++++++++++------ .../elasticsearch/kibana/KibanaPlugin.java | 2 +- .../kibana/KibanaPluginTests.java | 2 +- 3 files changed, 84 insertions(+), 37 deletions(-) diff --git a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java index df6e894db2776..92a4ee7d5b431 100644 --- a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java +++ b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java @@ -19,6 +19,8 @@ package org.elasticsearch.kibana; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -27,6 +29,7 @@ import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; +import java.util.Arrays; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -34,18 +37,35 @@ public class KibanaSystemIndexIT extends ESRestTestCase { + private final String indexName; + + public KibanaSystemIndexIT(@Name("indexName") String indexName) { + this.indexName = indexName; + } + + @ParametersFactory + public static Iterable data() { + return Arrays.asList( + new Object[] { ".kibana" }, + new Object[] { ".kibana-1" }, + new Object[] { ".reporting" }, + new Object[] { ".apm-agent-configuration" } + ); + } + public void testCreateIndex() throws IOException { - Request request = new Request("PUT", "/_kibana/.kibana-1"); + Request request = new Request("PUT", "/_kibana/" + indexName); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } public void testAliases() throws IOException { - Request request = new Request("PUT", "/_kibana/.kibana-1"); + assumeFalse("In this test, .kibana is the alias name", ".kibana".equals(indexName)); + Request request = new Request("PUT", "/_kibana/" + indexName); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("PUT", "/_kibana/.kibana-1/_alias/.kibana"); + request = new Request("PUT", "/_kibana/" + indexName + "/_alias/.kibana"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); @@ -57,22 +77,22 @@ public void testAliases() throws IOException { public void testBulkToKibanaIndex() throws IOException { Request request = new Request("POST", "/_kibana/_bulk"); - request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n"); + request.setJsonEntity("{ \"index\" : { \"_index\" : \"" + indexName + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } public void testRefresh() throws IOException { Request request = new Request("POST", "/_kibana/_bulk"); - request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n"); + request.setJsonEntity("{ \"index\" : { \"_index\" : \"" + indexName + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("GET", "/_kibana/.kibana/_refresh"); + request = new Request("GET", "/_kibana/" + indexName + "/_refresh"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request getRequest = new Request("GET", "/_kibana/.kibana/_doc/1"); + Request getRequest = new Request("GET", "/_kibana/" + indexName + "/_doc/1"); Response getResponse = client().performRequest(getRequest); assertThat(getResponse.getStatusLine().getStatusCode(), is(200)); String responseBody = EntityUtils.toString(getResponse.getEntity()); @@ -82,13 +102,13 @@ public void testRefresh() throws IOException { public void testGetFromKibanaIndex() throws IOException { Request request = new Request("POST", "/_kibana/_bulk"); - request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n"); + request.setJsonEntity("{ \"index\" : { \"_index\" : \"" + indexName + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n"); request.addParameter("refresh", "true"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request getRequest = new Request("GET", "/_kibana/.kibana/_doc/1"); + Request getRequest = new Request("GET", "/_kibana/" + indexName + "/_doc/1"); Response getResponse = client().performRequest(getRequest); assertThat(getResponse.getStatusLine().getStatusCode(), is(200)); String responseBody = EntityUtils.toString(getResponse.getEntity()); @@ -99,8 +119,12 @@ public void testGetFromKibanaIndex() throws IOException { public void testMultiGetFromKibanaIndex() throws IOException { Request request = new Request("POST", "/_kibana/_bulk"); request.setJsonEntity( - "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" - + "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" + + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" ); request.addParameter("refresh", "true"); @@ -109,7 +133,12 @@ public void testMultiGetFromKibanaIndex() throws IOException { Request getRequest = new Request("GET", "/_kibana/_mget"); getRequest.setJsonEntity( - "{ \"docs\" : [ { \"_index\" : \".kibana\", \"_id\" : \"1\" }, " + "{ \"_index\" : \".kibana\", \"_id\" : \"2\" } ] }\n" + "{ \"docs\" : [ { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"1\" }, " + + "{ \"_index\" : \"" + + indexName + + "\", \"_id\" : \"2\" } ] }\n" ); Response getResponse = client().performRequest(getRequest); assertThat(getResponse.getStatusLine().getStatusCode(), is(200)); @@ -123,15 +152,19 @@ public void testMultiGetFromKibanaIndex() throws IOException { public void testSearchFromKibanaIndex() throws IOException { Request request = new Request("POST", "/_kibana/_bulk"); request.setJsonEntity( - "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" - + "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" + + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" ); request.addParameter("refresh", "true"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request searchRequest = new Request("GET", "/_kibana/.kibana/_search"); + Request searchRequest = new Request("GET", "/_kibana/" + indexName + "/_search"); searchRequest.setJsonEntity("{ \"query\" : { \"match_all\" : {} } }\n"); Response getResponse = client().performRequest(searchRequest); assertThat(getResponse.getStatusLine().getStatusCode(), is(200)); @@ -145,15 +178,19 @@ public void testSearchFromKibanaIndex() throws IOException { public void testDeleteFromKibanaIndex() throws IOException { Request request = new Request("POST", "/_kibana/_bulk"); request.setJsonEntity( - "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" - + "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" + + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" ); request.addParameter("refresh", "true"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request deleteRequest = new Request("DELETE", "/_kibana/.kibana/_doc/1"); + Request deleteRequest = new Request("DELETE", "/_kibana/" + indexName + "/_doc/1"); Response deleteResponse = client().performRequest(deleteRequest); assertThat(deleteResponse.getStatusLine().getStatusCode(), is(200)); } @@ -161,63 +198,67 @@ public void testDeleteFromKibanaIndex() throws IOException { public void testDeleteByQueryFromKibanaIndex() throws IOException { Request request = new Request("POST", "/_kibana/_bulk"); request.setJsonEntity( - "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" - + "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" + + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" ); request.addParameter("refresh", "true"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request dbqRequest = new Request("POST", "/_kibana/.kibana/_delete_by_query"); + Request dbqRequest = new Request("POST", "/_kibana/" + indexName + "/_delete_by_query"); dbqRequest.setJsonEntity("{ \"query\" : { \"match_all\" : {} } }\n"); Response dbqResponse = client().performRequest(dbqRequest); assertThat(dbqResponse.getStatusLine().getStatusCode(), is(200)); } public void testUpdateIndexSettings() throws IOException { - Request request = new Request("PUT", "/_kibana/.kibana-1"); + Request request = new Request("PUT", "/_kibana/" + indexName); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("PUT", "/_kibana/.kibana-1/_settings"); + request = new Request("PUT", "/_kibana/" + indexName + "/_settings"); request.setJsonEntity("{ \"index.blocks.read_only\" : false }"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } public void testGetIndex() throws IOException { - Request request = new Request("PUT", "/_kibana/.kibana-1"); + Request request = new Request("PUT", "/_kibana/" + indexName); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("GET", "/_kibana/.kibana-1"); + request = new Request("GET", "/_kibana/" + indexName); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - assertThat(EntityUtils.toString(response.getEntity()), containsString(".kibana-1")); + assertThat(EntityUtils.toString(response.getEntity()), containsString(indexName)); } public void testIndexingAndUpdatingDocs() throws IOException { - Request request = new Request("PUT", "/_kibana/.kibana-1/_doc/1"); + Request request = new Request("PUT", "/_kibana/" + indexName + "/_doc/1"); request.setJsonEntity("{ \"foo\" : \"bar\" }"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(201)); - request = new Request("PUT", "/_kibana/.kibana-1/_create/2"); + request = new Request("PUT", "/_kibana/" + indexName + "/_create/2"); request.setJsonEntity("{ \"foo\" : \"bar\" }"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(201)); - request = new Request("POST", "/_kibana/.kibana-1/_doc"); + request = new Request("POST", "/_kibana/" + indexName + "/_doc"); request.setJsonEntity("{ \"foo\" : \"bar\" }"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(201)); - request = new Request("GET", "/_kibana/.kibana-1/_refresh"); + request = new Request("GET", "/_kibana/" + indexName + "/_refresh"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("POST", "/_kibana/.kibana-1/_update/1"); + request = new Request("POST", "/_kibana/" + indexName + "/_update/1"); request.setJsonEntity("{ \"doc\" : { \"foo\" : \"baz\" } }"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); @@ -226,15 +267,21 @@ public void testIndexingAndUpdatingDocs() throws IOException { public void testScrollingDocs() throws IOException { Request request = new Request("POST", "/_kibana/_bulk"); request.setJsonEntity( - "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" - + "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" - + "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"3\" } }\n{ \"baz\" : \"tag\" }\n" + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" + + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" + + "{ \"index\" : { \"_index\" : \"" + + indexName + + "\", \"_id\" : \"3\" } }\n{ \"baz\" : \"tag\" }\n" ); request.addParameter("refresh", "true"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request searchRequest = new Request("GET", "/_kibana/.kibana/_search"); + Request searchRequest = new Request("GET", "/_kibana/" + indexName + "/_search"); searchRequest.setJsonEntity("{ \"size\" : 1,\n\"query\" : { \"match_all\" : {} } }\n"); searchRequest.addParameter("scroll", "1m"); response = client().performRequest(searchRequest); diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java index 186da3f827fcb..f6a09beab2fc2 100644 --- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java +++ b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java @@ -62,7 +62,7 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin { public static final Setting> KIBANA_INDEX_NAMES_SETTING = Setting.listSetting( "kibana.system_indices", - List.of(".kibana*", ".reporting"), + List.of(".kibana*", ".reporting", ".apm-agent-configuration"), Function.identity(), Property.NodeScope ); diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java index b1551b5a4874e..4e95b9b810317 100644 --- a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java +++ b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java @@ -38,7 +38,7 @@ public void testKibanaIndexNames() { .stream() .map(SystemIndexDescriptor::getIndexPattern) .collect(Collectors.toUnmodifiableList()), - contains(".kibana*", ".reporting") + contains(".kibana*", ".reporting", ".apm-agent-configuration") ); final List names = List.of("." + randomAlphaOfLength(4), "." + randomAlphaOfLength(6)); final List namesFromDescriptors = new KibanaPlugin().getSystemIndexDescriptors( From e145266634a3e9f44d4888165bb5e3dcddca411b Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 20 Oct 2020 10:30:50 +0300 Subject: [PATCH 34/60] [ML] Rename evaluation metric result fields to value (#63809) Renames data frame analytics _evaluate API results as follows: - per class accuracy renamed from `accuracy` to `value` - per class precision renamed from `precision` to `value` - per class recall renamed from `recall` to `value` - auc_roc `score` renamed to `value` for both outlier detection and classification --- .../MlEvaluationNamedXContentProvider.java | 10 +- .../classification/AccuracyMetric.java | 73 +-------- .../classification/AucRocMetric.java | 154 +----------------- .../classification/PerClassSingleValue.java | 81 +++++++++ .../classification/PrecisionMetric.java | 67 +------- .../classification/RecallMetric.java | 67 +------- .../evaluation/common/AucRocPoint.java | 104 ++++++++++++ .../evaluation/common/AucRocResult.java | 109 +++++++++++++ .../outlierdetection/AucRocMetric.java | 3 +- .../client/MachineLearningIT.java | 33 ++-- .../MlClientDocumentationIT.java | 5 +- .../ml/EvaluateDataFrameResponseTests.java | 5 +- .../AccuracyMetricResultTests.java | 5 +- ...sts.java => PerClassSingleValueTests.java} | 17 +- .../PrecisionMetricResultTests.java | 5 +- .../RecallMetricResultTests.java | 5 +- .../evaluation/common/AucRocPointTests.java | 46 ++++++ .../AucRocResultTests.java} | 19 ++- .../apis/evaluate-dfanalytics.asciidoc | 4 +- .../MlEvaluationNamedXContentProvider.java | 9 +- .../evaluation/classification/Accuracy.java | 87 +--------- .../evaluation/classification/AucRoc.java | 1 + .../classification/PerClassSingleValue.java | 85 ++++++++++ .../evaluation/classification/Precision.java | 85 +--------- .../evaluation/classification/Recall.java | 85 +--------- .../AbstractAucRoc.java | 34 ++-- .../evaluation/outlierdetection/AucRoc.java | 2 +- .../classification/AccuracyResultTests.java | 5 +- .../classification/AccuracyTests.java | 11 +- .../classification/AucRocResultTests.java | 4 +- .../PerClassSingleValueTests.java | 30 ++++ .../classification/PrecisionResultTests.java | 5 +- .../classification/RecallResultTests.java | 5 +- .../AbstractAucRocTests.java | 5 +- .../ClassificationEvaluationIT.java | 135 +++++++-------- .../ml/integration/ClassificationIT.java | 15 +- .../OutlierDetectionEvaluationIT.java | 4 +- .../test/ml/evaluate_data_frame.yml | 30 ++-- 38 files changed, 686 insertions(+), 763 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PerClassSingleValue.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocPoint.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocResult.java rename client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/{AucRocMetricAucRocPointTests.java => PerClassSingleValueTests.java} (66%) create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocPointTests.java rename client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/{classification/AucRocMetricResultTests.java => common/AucRocResultTests.java} (70%) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValue.java rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/{classification => common}/AbstractAucRoc.java (92%) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/{classification => common}/AbstractAucRocTests.java (94%) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java index e4534c5603bd8..9b77518474e67 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java @@ -24,6 +24,7 @@ import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric; import org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetric; import org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetric; +import org.elasticsearch.client.ml.dataframe.evaluation.common.AucRocResult; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.ConfusionMatrixMetric; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.OutlierDetection; import org.elasticsearch.client.ml.dataframe.evaluation.regression.HuberMetric; @@ -122,10 +123,9 @@ Evaluation.class, new ParseField(OutlierDetection.NAME), OutlierDetection::fromX // Evaluation metrics results new NamedXContentRegistry.Entry( EvaluationMetric.Result.class, - new ParseField( - registeredMetricName( - OutlierDetection.NAME, org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.AucRocMetric.NAME)), - org.elasticsearch.client.ml.dataframe.evaluation.classification.AucRocMetric.Result::fromXContent), + new ParseField(registeredMetricName( + OutlierDetection.NAME, org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.AucRocMetric.NAME)), + AucRocResult::fromXContent), new NamedXContentRegistry.Entry( EvaluationMetric.Result.class, new ParseField( @@ -145,7 +145,7 @@ Evaluation.class, new ParseField(OutlierDetection.NAME), OutlierDetection::fromX new NamedXContentRegistry.Entry( EvaluationMetric.Result.class, new ParseField(registeredMetricName(Classification.NAME, AucRocMetric.NAME)), - AucRocMetric.Result::fromXContent), + AucRocResult::fromXContent), new NamedXContentRegistry.Entry( EvaluationMetric.Result.class, new ParseField(registeredMetricName(Classification.NAME, AccuracyMetric.NAME)), diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetric.java index 151783499e46b..9f496d4cf1245 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetric.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetric.java @@ -20,11 +20,9 @@ import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -99,10 +97,10 @@ public static class Result implements EvaluationMetric.Result { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("accuracy_result", true, a -> new Result((List) a[0], (double) a[1])); + new ConstructingObjectParser<>("accuracy_result", true, a -> new Result((List) a[0], (double) a[1])); static { - PARSER.declareObjectArray(constructorArg(), PerClassResult.PARSER, CLASSES); + PARSER.declareObjectArray(constructorArg(), PerClassSingleValue.PARSER, CLASSES); PARSER.declareDouble(constructorArg(), OVERALL_ACCURACY); } @@ -111,11 +109,11 @@ public static Result fromXContent(XContentParser parser) { } /** List of per-class results. */ - private final List classes; + private final List classes; /** Fraction of documents for which predicted class equals the actual class. */ private final double overallAccuracy; - public Result(List classes, double overallAccuracy) { + public Result(List classes, double overallAccuracy) { this.classes = Collections.unmodifiableList(Objects.requireNonNull(classes)); this.overallAccuracy = overallAccuracy; } @@ -125,7 +123,7 @@ public String getMetricName() { return NAME; } - public List getClasses() { + public List getClasses() { return classes; } @@ -156,65 +154,4 @@ public int hashCode() { return Objects.hash(classes, overallAccuracy); } } - - public static class PerClassResult implements ToXContentObject { - - private static final ParseField CLASS_NAME = new ParseField("class_name"); - private static final ParseField ACCURACY = new ParseField("accuracy"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("accuracy_per_class_result", true, a -> new PerClassResult((String) a[0], (double) a[1])); - - static { - PARSER.declareString(constructorArg(), CLASS_NAME); - PARSER.declareDouble(constructorArg(), ACCURACY); - } - - /** Name of the class. */ - private final String className; - /** Fraction of documents that are either true positives or true negatives wrt {@code className}. */ - private final double accuracy; - - public PerClassResult(String className, double accuracy) { - this.className = Objects.requireNonNull(className); - this.accuracy = accuracy; - } - - public String getClassName() { - return className; - } - - public double getAccuracy() { - return accuracy; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(CLASS_NAME.getPreferredName(), className); - builder.field(ACCURACY.getPreferredName(), accuracy); - builder.endObject(); - return builder; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PerClassResult that = (PerClassResult) o; - return Objects.equals(this.className, that.className) - && this.accuracy == that.accuracy; - } - - @Override - public int hashCode() { - return Objects.hash(className, accuracy); - } - - @Override - public String toString() { - return Strings.toString(this); - } - } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetric.java index 5e9b28303c977..aec1d8655db7c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetric.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetric.java @@ -19,18 +19,14 @@ package org.elasticsearch.client.ml.dataframe.evaluation.classification; import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric; -import org.elasticsearch.common.Nullable; +import org.elasticsearch.client.ml.dataframe.evaluation.common.AucRocResult; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; -import java.util.Collections; -import java.util.List; import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -43,12 +39,11 @@ */ public class AucRocMetric implements EvaluationMetric { - public static final String NAME = "auc_roc"; + public static final String NAME = AucRocResult.NAME; public static final ParseField CLASS_NAME = new ParseField("class_name"); public static final ParseField INCLUDE_CURVE = new ParseField("include_curve"); - @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, args -> new AucRocMetric((String) args[0], (Boolean) args[1])); @@ -106,149 +101,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(className, includeCurve); } - - public static class Result implements EvaluationMetric.Result { - - public static Result fromXContent(XContentParser parser) { - return PARSER.apply(parser, null); - } - - private static final ParseField SCORE = new ParseField("score"); - private static final ParseField CURVE = new ParseField("curve"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>( - "auc_roc_result", true, args -> new Result((double) args[0], (List) args[1])); - - static { - PARSER.declareDouble(constructorArg(), SCORE); - PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> AucRocPoint.fromXContent(p), CURVE); - } - - private final double score; - private final List curve; - - public Result(double score, @Nullable List curve) { - this.score = score; - this.curve = curve; - } - - @Override - public String getMetricName() { - return NAME; - } - - public double getScore() { - return score; - } - - public List getCurve() { - return curve == null ? null : Collections.unmodifiableList(curve); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(); - builder.field(SCORE.getPreferredName(), score); - if (curve != null && curve.isEmpty() == false) { - builder.field(CURVE.getPreferredName(), curve); - } - builder.endObject(); - return builder; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Result that = (Result) o; - return score == that.score - && Objects.equals(curve, that.curve); - } - - @Override - public int hashCode() { - return Objects.hash(score, curve); - } - - @Override - public String toString() { - return Strings.toString(this); - } - } - - public static final class AucRocPoint implements ToXContentObject { - - public static AucRocPoint fromXContent(XContentParser parser) { - return PARSER.apply(parser, null); - } - - private static final ParseField TPR = new ParseField("tpr"); - private static final ParseField FPR = new ParseField("fpr"); - private static final ParseField THRESHOLD = new ParseField("threshold"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>( - "auc_roc_point", - true, - args -> new AucRocPoint((double) args[0], (double) args[1], (double) args[2])); - - static { - PARSER.declareDouble(constructorArg(), TPR); - PARSER.declareDouble(constructorArg(), FPR); - PARSER.declareDouble(constructorArg(), THRESHOLD); - } - - private final double tpr; - private final double fpr; - private final double threshold; - - public AucRocPoint(double tpr, double fpr, double threshold) { - this.tpr = tpr; - this.fpr = fpr; - this.threshold = threshold; - } - - public double getTruePositiveRate() { - return tpr; - } - - public double getFalsePositiveRate() { - return fpr; - } - - public double getThreshold() { - return threshold; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder - .startObject() - .field(TPR.getPreferredName(), tpr) - .field(FPR.getPreferredName(), fpr) - .field(THRESHOLD.getPreferredName(), threshold) - .endObject(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AucRocPoint that = (AucRocPoint) o; - return tpr == that.tpr && fpr == that.fpr && threshold == that.threshold; - } - - @Override - public int hashCode() { - return Objects.hash(tpr, fpr, threshold); - } - - @Override - public String toString() { - return Strings.toString(this); - } - } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PerClassSingleValue.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PerClassSingleValue.java new file mode 100644 index 0000000000000..2caf09085e751 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PerClassSingleValue.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.dataframe.evaluation.classification; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class PerClassSingleValue implements ToXContentObject { + private static final ParseField CLASS_NAME = new ParseField("class_name"); + private static final ParseField VALUE = new ParseField("value"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("per_class_result", true, a -> new PerClassSingleValue((String) a[0], (double) a[1])); + + static { + PARSER.declareString(constructorArg(), CLASS_NAME); + PARSER.declareDouble(constructorArg(), VALUE); + } + + private final String className; + private final double value; + + public PerClassSingleValue(String className, double value) { + this.className = Objects.requireNonNull(className); + this.value = value; + } + + public String getClassName() { + return className; + } + + public double getValue() { + return value; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CLASS_NAME.getPreferredName(), className); + builder.field(VALUE.getPreferredName(), value); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PerClassSingleValue that = (PerClassSingleValue) o; + return Objects.equals(this.className, that.className) + && this.value == that.value; + } + + @Override + public int hashCode() { + return Objects.hash(className, value); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PrecisionMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PrecisionMetric.java index 8eff7986dcc36..64190d23b45b8 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PrecisionMetric.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PrecisionMetric.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -85,10 +84,10 @@ public static class Result implements EvaluationMetric.Result { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("precision_result", true, a -> new Result((List) a[0], (double) a[1])); + new ConstructingObjectParser<>("precision_result", true, a -> new Result((List) a[0], (double) a[1])); static { - PARSER.declareObjectArray(constructorArg(), PerClassResult.PARSER, CLASSES); + PARSER.declareObjectArray(constructorArg(), PerClassSingleValue.PARSER, CLASSES); PARSER.declareDouble(constructorArg(), AVG_PRECISION); } @@ -97,11 +96,11 @@ public static Result fromXContent(XContentParser parser) { } /** List of per-class results. */ - private final List classes; + private final List classes; /** Average of per-class precisions. */ private final double avgPrecision; - public Result(List classes, double avgPrecision) { + public Result(List classes, double avgPrecision) { this.classes = Collections.unmodifiableList(Objects.requireNonNull(classes)); this.avgPrecision = avgPrecision; } @@ -111,7 +110,7 @@ public String getMetricName() { return NAME; } - public List getClasses() { + public List getClasses() { return classes; } @@ -142,60 +141,4 @@ public int hashCode() { return Objects.hash(classes, avgPrecision); } } - - public static class PerClassResult implements ToXContentObject { - - private static final ParseField CLASS_NAME = new ParseField("class_name"); - private static final ParseField PRECISION = new ParseField("precision"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("precision_per_class_result", true, a -> new PerClassResult((String) a[0], (double) a[1])); - - static { - PARSER.declareString(constructorArg(), CLASS_NAME); - PARSER.declareDouble(constructorArg(), PRECISION); - } - - /** Name of the class. */ - private final String className; - /** Fraction of documents predicted as belonging to the {@code predictedClass} class predicted correctly. */ - private final double precision; - - public PerClassResult(String className, double precision) { - this.className = Objects.requireNonNull(className); - this.precision = precision; - } - - public String getClassName() { - return className; - } - - public double getPrecision() { - return precision; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(CLASS_NAME.getPreferredName(), className); - builder.field(PRECISION.getPreferredName(), precision); - builder.endObject(); - return builder; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PerClassResult that = (PerClassResult) o; - return Objects.equals(this.className, that.className) - && this.precision == that.precision; - } - - @Override - public int hashCode() { - return Objects.hash(className, precision); - } - } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/RecallMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/RecallMetric.java index d46a70da8c3f6..f973eada09599 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/RecallMetric.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/RecallMetric.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -85,10 +84,10 @@ public static class Result implements EvaluationMetric.Result { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("recall_result", true, a -> new Result((List) a[0], (double) a[1])); + new ConstructingObjectParser<>("recall_result", true, a -> new Result((List) a[0], (double) a[1])); static { - PARSER.declareObjectArray(constructorArg(), PerClassResult.PARSER, CLASSES); + PARSER.declareObjectArray(constructorArg(), PerClassSingleValue.PARSER, CLASSES); PARSER.declareDouble(constructorArg(), AVG_RECALL); } @@ -97,11 +96,11 @@ public static Result fromXContent(XContentParser parser) { } /** List of per-class results. */ - private final List classes; + private final List classes; /** Average of per-class recalls. */ private final double avgRecall; - public Result(List classes, double avgRecall) { + public Result(List classes, double avgRecall) { this.classes = Collections.unmodifiableList(Objects.requireNonNull(classes)); this.avgRecall = avgRecall; } @@ -111,7 +110,7 @@ public String getMetricName() { return NAME; } - public List getClasses() { + public List getClasses() { return classes; } @@ -142,60 +141,4 @@ public int hashCode() { return Objects.hash(classes, avgRecall); } } - - public static class PerClassResult implements ToXContentObject { - - private static final ParseField CLASS_NAME = new ParseField("class_name"); - private static final ParseField RECALL = new ParseField("recall"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("recall_per_class_result", true, a -> new PerClassResult((String) a[0], (double) a[1])); - - static { - PARSER.declareString(constructorArg(), CLASS_NAME); - PARSER.declareDouble(constructorArg(), RECALL); - } - - /** Name of the class. */ - private final String className; - /** Fraction of documents actually belonging to the {@code actualClass} class predicted correctly. */ - private final double recall; - - public PerClassResult(String className, double recall) { - this.className = Objects.requireNonNull(className); - this.recall = recall; - } - - public String getClassName() { - return className; - } - - public double getRecall() { - return recall; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(CLASS_NAME.getPreferredName(), className); - builder.field(RECALL.getPreferredName(), recall); - builder.endObject(); - return builder; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PerClassResult that = (PerClassResult) o; - return Objects.equals(this.className, that.className) - && this.recall == that.recall; - } - - @Override - public int hashCode() { - return Objects.hash(className, recall); - } - } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocPoint.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocPoint.java new file mode 100644 index 0000000000000..92abb460134dc --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocPoint.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.dataframe.evaluation.common; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class AucRocPoint implements ToXContentObject { + + public static AucRocPoint fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + private static final ParseField TPR = new ParseField("tpr"); + private static final ParseField FPR = new ParseField("fpr"); + private static final ParseField THRESHOLD = new ParseField("threshold"); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "auc_roc_point", + true, + args -> new AucRocPoint((double) args[0], (double) args[1], (double) args[2])); + + static { + PARSER.declareDouble(constructorArg(), TPR); + PARSER.declareDouble(constructorArg(), FPR); + PARSER.declareDouble(constructorArg(), THRESHOLD); + } + + private final double tpr; + private final double fpr; + private final double threshold; + + public AucRocPoint(double tpr, double fpr, double threshold) { + this.tpr = tpr; + this.fpr = fpr; + this.threshold = threshold; + } + + public double getTruePositiveRate() { + return tpr; + } + + public double getFalsePositiveRate() { + return fpr; + } + + public double getThreshold() { + return threshold; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder + .startObject() + .field(TPR.getPreferredName(), tpr) + .field(FPR.getPreferredName(), fpr) + .field(THRESHOLD.getPreferredName(), threshold) + .endObject(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AucRocPoint that = (AucRocPoint) o; + return tpr == that.tpr && fpr == that.fpr && threshold == that.threshold; + } + + @Override + public int hashCode() { + return Objects.hash(tpr, fpr, threshold); + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocResult.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocResult.java new file mode 100644 index 0000000000000..e61730bd12380 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocResult.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.dataframe.evaluation.common; + +import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class AucRocResult implements EvaluationMetric.Result { + + public static final String NAME = "auc_roc"; + + public static AucRocResult fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + private static final ParseField VALUE = new ParseField("value"); + private static final ParseField CURVE = new ParseField("curve"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + NAME, true, args -> new AucRocResult((double) args[0], (List) args[1])); + + static { + PARSER.declareDouble(constructorArg(), VALUE); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> AucRocPoint.fromXContent(p), CURVE); + } + + private final double value; + private final List curve; + + public AucRocResult(double value, @Nullable List curve) { + this.value = value; + this.curve = curve; + } + + @Override + public String getMetricName() { + return NAME; + } + + public double getValue() { + return value; + } + + public List getCurve() { + return curve == null ? null : Collections.unmodifiableList(curve); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field(VALUE.getPreferredName(), value); + if (curve != null && curve.isEmpty() == false) { + builder.field(CURVE.getPreferredName(), curve); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AucRocResult that = (AucRocResult) o; + return value == that.value + && Objects.equals(curve, that.curve); + } + + @Override + public int hashCode() { + return Objects.hash(value, curve); + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/outlierdetection/AucRocMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/outlierdetection/AucRocMetric.java index 76d8c514daef6..aebb10792f0b4 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/outlierdetection/AucRocMetric.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/outlierdetection/AucRocMetric.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection; import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric; +import org.elasticsearch.client.ml.dataframe.evaluation.common.AucRocResult; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,7 +37,7 @@ */ public class AucRocMetric implements EvaluationMetric { - public static final String NAME = "auc_roc"; + public static final String NAME = AucRocResult.NAME; public static final ParseField INCLUDE_CURVE = new ParseField("include_curve"); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index a065be7eae0e1..0ef00e8bb0e23 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -141,8 +141,11 @@ import org.elasticsearch.client.ml.dataframe.evaluation.classification.AucRocMetric; import org.elasticsearch.client.ml.dataframe.evaluation.classification.Classification; import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric; +import org.elasticsearch.client.ml.dataframe.evaluation.classification.PerClassSingleValue; import org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetric; import org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetric; +import org.elasticsearch.client.ml.dataframe.evaluation.common.AucRocPoint; +import org.elasticsearch.client.ml.dataframe.evaluation.common.AucRocResult; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.ConfusionMatrixMetric; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.OutlierDetection; import org.elasticsearch.client.ml.dataframe.evaluation.regression.HuberMetric; @@ -1789,17 +1792,17 @@ public void testEvaluateDataFrame_OutlierDetection() throws IOException { assertThat(confusionMatrix.getFalseNegatives(), equalTo(3L)); // docs #5, #6 and #7 assertNull(confusionMatrixResult.getScoreByThreshold("0.1")); - AucRocMetric.Result aucRocResult = + AucRocResult aucRocResult = evaluateDataFrameResponse.getMetricByName(org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.AucRocMetric.NAME); assertThat(aucRocResult.getMetricName(), equalTo(AucRocMetric.NAME)); - assertThat(aucRocResult.getScore(), closeTo(0.70025, 1e-9)); + assertThat(aucRocResult.getValue(), closeTo(0.70025, 1e-9)); assertNotNull(aucRocResult.getCurve()); - List curve = aucRocResult.getCurve(); - AucRocMetric.AucRocPoint curvePointAtThreshold0 = curve.stream().filter(p -> p.getThreshold() == 0.0).findFirst().get(); + List curve = aucRocResult.getCurve(); + AucRocPoint curvePointAtThreshold0 = curve.stream().filter(p -> p.getThreshold() == 0.0).findFirst().get(); assertThat(curvePointAtThreshold0.getTruePositiveRate(), equalTo(1.0)); assertThat(curvePointAtThreshold0.getFalsePositiveRate(), equalTo(1.0)); assertThat(curvePointAtThreshold0.getThreshold(), equalTo(0.0)); - AucRocMetric.AucRocPoint curvePointAtThreshold1 = curve.stream().filter(p -> p.getThreshold() == 1.0).findFirst().get(); + AucRocPoint curvePointAtThreshold1 = curve.stream().filter(p -> p.getThreshold() == 1.0).findFirst().get(); assertThat(curvePointAtThreshold1.getTruePositiveRate(), equalTo(0.0)); assertThat(curvePointAtThreshold1.getFalsePositiveRate(), equalTo(0.0)); assertThat(curvePointAtThreshold1.getThreshold(), equalTo(1.0)); @@ -1925,9 +1928,9 @@ public void testEvaluateDataFrame_Classification() throws IOException { assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME)); assertThat(evaluateDataFrameResponse.getMetrics().size(), equalTo(1)); - AucRocMetric.Result aucRocResult = evaluateDataFrameResponse.getMetricByName(AucRocMetric.NAME); + AucRocResult aucRocResult = evaluateDataFrameResponse.getMetricByName(AucRocMetric.NAME); assertThat(aucRocResult.getMetricName(), equalTo(AucRocMetric.NAME)); - assertThat(aucRocResult.getScore(), closeTo(0.6425, 1e-9)); + assertThat(aucRocResult.getValue(), closeTo(0.6425, 1e-9)); assertNotNull(aucRocResult.getCurve()); } { // Accuracy @@ -1947,11 +1950,11 @@ public void testEvaluateDataFrame_Classification() throws IOException { equalTo( List.of( // 9 out of 10 examples were classified correctly - new AccuracyMetric.PerClassResult("ant", 0.9), + new PerClassSingleValue("ant", 0.9), // 6 out of 10 examples were classified correctly - new AccuracyMetric.PerClassResult("cat", 0.6), + new PerClassSingleValue("cat", 0.6), // 8 out of 10 examples were classified correctly - new AccuracyMetric.PerClassResult("dog", 0.8)))); + new PerClassSingleValue("dog", 0.8)))); assertThat(accuracyResult.getOverallAccuracy(), equalTo(0.6)); // 6 out of 10 examples were classified correctly } { // Precision @@ -1971,9 +1974,9 @@ public void testEvaluateDataFrame_Classification() throws IOException { equalTo( List.of( // 3 out of 5 examples labeled as "cat" were classified correctly - new PrecisionMetric.PerClassResult("cat", 0.6), + new PerClassSingleValue("cat", 0.6), // 3 out of 4 examples labeled as "dog" were classified correctly - new PrecisionMetric.PerClassResult("dog", 0.75)))); + new PerClassSingleValue("dog", 0.75)))); assertThat(precisionResult.getAvgPrecision(), equalTo(0.675)); } { // Recall @@ -1993,11 +1996,11 @@ public void testEvaluateDataFrame_Classification() throws IOException { equalTo( List.of( // 3 out of 5 examples labeled as "cat" were classified correctly - new RecallMetric.PerClassResult("cat", 0.6), + new PerClassSingleValue("cat", 0.6), // 3 out of 4 examples labeled as "dog" were classified correctly - new RecallMetric.PerClassResult("dog", 0.75), + new PerClassSingleValue("dog", 0.75), // no examples labeled as "ant" were classified correctly - new RecallMetric.PerClassResult("ant", 0.0)))); + new PerClassSingleValue("ant", 0.0)))); assertThat(recallResult.getAvgRecall(), equalTo(0.45)); } { // No size provided for MulticlassConfusionMatrixMetric, default used instead diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index b9add106b82d7..4e753365c1e4d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -162,6 +162,7 @@ import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric.PredictedClass; import org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetric; import org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetric; +import org.elasticsearch.client.ml.dataframe.evaluation.common.AucRocResult; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.ConfusionMatrixMetric; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.ConfusionMatrixMetric.ConfusionMatrix; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.OutlierDetection; @@ -3529,8 +3530,8 @@ public void testEvaluateDataFrame_Classification() throws Exception { List confusionMatrix = multiclassConfusionMatrix.getConfusionMatrix(); // <8> long otherClassesCount = multiclassConfusionMatrix.getOtherActualClassCount(); // <9> - AucRocMetric.Result aucRocResult = response.getMetricByName(AucRocMetric.NAME); // <10> - double aucRocScore = aucRocResult.getScore(); // <11> + AucRocResult aucRocResult = response.getMetricByName(AucRocMetric.NAME); // <10> + double aucRocScore = aucRocResult.getValue(); // <11> // end::evaluate-data-frame-results-classification assertThat(accuracyResult.getMetricName(), equalTo(AccuracyMetric.NAME)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java index 50fe97b51956d..a3d23f820d259 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.client.ml.dataframe.evaluation.regression.MeanSquaredErrorMetricResultTests; import org.elasticsearch.client.ml.dataframe.evaluation.regression.RSquaredMetricResultTests; import org.elasticsearch.client.ml.dataframe.evaluation.regression.Regression; -import org.elasticsearch.client.ml.dataframe.evaluation.classification.AucRocMetricResultTests; +import org.elasticsearch.client.ml.dataframe.evaluation.common.AucRocResultTests; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.OutlierDetection; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.ConfusionMatrixMetricResultTests; import org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.PrecisionMetricResultTests; @@ -49,7 +49,7 @@ public static EvaluateDataFrameResponse randomResponse() { case OutlierDetection.NAME: metrics = randomSubsetOf( Arrays.asList( - AucRocMetricResultTests.randomResult(), + AucRocResultTests.randomResult(), PrecisionMetricResultTests.randomResult(), RecallMetricResultTests.randomResult(), ConfusionMatrixMetricResultTests.randomResult())); @@ -63,6 +63,7 @@ public static EvaluateDataFrameResponse randomResponse() { case Classification.NAME: metrics = randomSubsetOf( Arrays.asList( + AucRocResultTests.randomResult(), AccuracyMetricResultTests.randomResult(), org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetricResultTests.randomResult(), org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetricResultTests.randomResult(), diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetricResultTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetricResultTests.java index 8758cea86c451..00b254fe3d863 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetricResultTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetricResultTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.client.ml.dataframe.evaluation.classification; import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; -import org.elasticsearch.client.ml.dataframe.evaluation.classification.AccuracyMetric.PerClassResult; import org.elasticsearch.client.ml.dataframe.evaluation.classification.AccuracyMetric.Result; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -41,10 +40,10 @@ protected NamedXContentRegistry xContentRegistry() { public static Result randomResult() { int numClasses = randomIntBetween(2, 100); List classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList()); - List classes = new ArrayList<>(numClasses); + List classes = new ArrayList<>(numClasses); for (int i = 0; i < numClasses; i++) { double accuracy = randomDoubleBetween(0.0, 1.0, true); - classes.add(new PerClassResult(classNames.get(i), accuracy)); + classes.add(new PerClassSingleValue(classNames.get(i), accuracy)); } double overallAccuracy = randomDoubleBetween(0.0, 1.0, true); return new Result(classes, overallAccuracy); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetricAucRocPointTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java similarity index 66% rename from client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetricAucRocPointTests.java rename to client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java index d3242906e34c4..0a3d3db5d68b4 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetricAucRocPointTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.client.ml.dataframe.evaluation.classification; import org.elasticsearch.common.xcontent.XContentParser; @@ -23,20 +24,16 @@ import java.io.IOException; -public class AucRocMetricAucRocPointTests extends AbstractXContentTestCase { - - static AucRocMetric.AucRocPoint randomPoint() { - return new AucRocMetric.AucRocPoint(randomDouble(), randomDouble(), randomDouble()); - } +public class PerClassSingleValueTests extends AbstractXContentTestCase { @Override - protected AucRocMetric.AucRocPoint createTestInstance() { - return randomPoint(); + protected PerClassSingleValue createTestInstance() { + return new PerClassSingleValue(randomAlphaOfLength(10), randomDouble()); } @Override - protected AucRocMetric.AucRocPoint doParseInstance(XContentParser parser) throws IOException { - return AucRocMetric.AucRocPoint.fromXContent(parser); + protected PerClassSingleValue doParseInstance(XContentParser parser) throws IOException { + return PerClassSingleValue.PARSER.apply(parser, null); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PrecisionMetricResultTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PrecisionMetricResultTests.java index ef6e41e78f0e8..50d023bf5e3e7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PrecisionMetricResultTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/PrecisionMetricResultTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.client.ml.dataframe.evaluation.classification; import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; -import org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetric.PerClassResult; import org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetric.Result; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -41,10 +40,10 @@ protected NamedXContentRegistry xContentRegistry() { public static Result randomResult() { int numClasses = randomIntBetween(2, 100); List classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList()); - List classes = new ArrayList<>(numClasses); + List classes = new ArrayList<>(numClasses); for (int i = 0; i < numClasses; i++) { double precision = randomDoubleBetween(0.0, 1.0, true); - classes.add(new PerClassResult(classNames.get(i), precision)); + classes.add(new PerClassSingleValue(classNames.get(i), precision)); } double avgPrecision = randomDoubleBetween(0.0, 1.0, true); return new Result(classes, avgPrecision); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/RecallMetricResultTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/RecallMetricResultTests.java index f8fffb405ea1b..1f001e6014fb9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/RecallMetricResultTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/RecallMetricResultTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.client.ml.dataframe.evaluation.classification; import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; -import org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetric.PerClassResult; import org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetric.Result; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -41,10 +40,10 @@ protected NamedXContentRegistry xContentRegistry() { public static Result randomResult() { int numClasses = randomIntBetween(2, 100); List classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList()); - List classes = new ArrayList<>(numClasses); + List classes = new ArrayList<>(numClasses); for (int i = 0; i < numClasses; i++) { double recall = randomDoubleBetween(0.0, 1.0, true); - classes.add(new PerClassResult(classNames.get(i), recall)); + classes.add(new PerClassSingleValue(classNames.get(i), recall)); } double avgRecall = randomDoubleBetween(0.0, 1.0, true); return new Result(classes, avgRecall); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocPointTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocPointTests.java new file mode 100644 index 0000000000000..7faaceb7036fd --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocPointTests.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.dataframe.evaluation.common; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +public class AucRocPointTests extends AbstractXContentTestCase { + + static AucRocPoint randomPoint() { + return new AucRocPoint(randomDouble(), randomDouble(), randomDouble()); + } + + @Override + protected AucRocPoint createTestInstance() { + return randomPoint(); + } + + @Override + protected AucRocPoint doParseInstance(XContentParser parser) throws IOException { + return AucRocPoint.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetricResultTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocResultTests.java similarity index 70% rename from client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetricResultTests.java rename to client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocResultTests.java index 40ada86f48445..85410b9d7e293 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/AucRocMetricResultTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/evaluation/common/AucRocResultTests.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -16,8 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.client.ml.dataframe.evaluation.classification; +package org.elasticsearch.client.ml.dataframe.evaluation.common; +import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractXContentTestCase; @@ -26,25 +27,25 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -public class AucRocMetricResultTests extends AbstractXContentTestCase { +public class AucRocResultTests extends AbstractXContentTestCase { - public static AucRocMetric.Result randomResult() { - return new AucRocMetric.Result( + public static EvaluationMetric.Result randomResult() { + return new AucRocResult( randomDouble(), Stream - .generate(AucRocMetricAucRocPointTests::randomPoint) + .generate(AucRocPointTests::randomPoint) .limit(randomIntBetween(1, 10)) .collect(Collectors.toList())); } @Override - protected AucRocMetric.Result createTestInstance() { + protected EvaluationMetric.Result createTestInstance() { return randomResult(); } @Override - protected AucRocMetric.Result doParseInstance(XContentParser parser) throws IOException { - return AucRocMetric.Result.fromXContent(parser); + protected EvaluationMetric.Result doParseInstance(XContentParser parser) throws IOException { + return AucRocResult.fromXContent(parser); } @Override diff --git a/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc index f9c643689a28b..4ae5db53de1b9 100644 --- a/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc @@ -258,7 +258,7 @@ The API returns the following results: { "outlier_detection": { "auc_roc": { - "score": 0.92584757746414444 + "value": 0.92584757746414444 }, "confusion_matrix": { "0.25": { @@ -534,7 +534,7 @@ The API returns the following result: { "classification" : { "auc_roc" : { - "score" : 0.8941788639536681 + "value" : 0.8941788639536681 } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java index c7ae0a3848775..d7ad4781e1be7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java @@ -11,10 +11,11 @@ import org.elasticsearch.plugins.spi.NamedXContentProvider; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.AucRoc; -import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision; -import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Classification; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.common.AbstractAucRoc; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.outlierdetection.ConfusionMatrix; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.outlierdetection.OutlierDetection; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.outlierdetection.ScoreByThresholdResult; @@ -179,8 +180,8 @@ public static List getNamedWriteables() { registeredMetricName(OutlierDetection.NAME, ConfusionMatrix.NAME), ConfusionMatrix.Result::new), new NamedWriteableRegistry.Entry(EvaluationMetricResult.class, - registeredMetricName(Classification.NAME, AucRoc.NAME), - AucRoc.Result::new), + AbstractAucRoc.Result.NAME, + AbstractAucRoc.Result::new), new NamedWriteableRegistry.Entry(EvaluationMetricResult.class, registeredMetricName(Classification.NAME, MulticlassConfusionMatrix.NAME), MulticlassConfusionMatrix.Result::new), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java index 96d249326fbe6..347b13dd8fd7a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Accuracy.java @@ -10,11 +10,9 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.Script; @@ -150,14 +148,14 @@ public Optional getResult() { * Time complexity of this method is linear wrt multiclass confusion matrix size, so O(n^2) where n is the matrix dimension. * This method is visible for testing only. */ - static List computePerClassAccuracy(MulticlassConfusionMatrix.Result matrixResult) { + static List computePerClassAccuracy(MulticlassConfusionMatrix.Result matrixResult) { assert matrixResult.getOtherActualClassCount() == 0; // Number of actual classes taken into account int n = matrixResult.getConfusionMatrix().size(); // Total number of documents taken into account long totalDocCount = matrixResult.getConfusionMatrix().stream().mapToLong(MulticlassConfusionMatrix.ActualClass::getActualClassDocCount).sum(); - List classes = new ArrayList<>(n); + List classes = new ArrayList<>(n); for (int i = 0; i < n; ++i) { String className = matrixResult.getConfusionMatrix().get(i).getActualClass(); // Start with the assumption that all the docs were predicted correctly. @@ -172,7 +170,7 @@ static List computePerClassAccuracy(MulticlassConfusionMatrix.Re } // Subtract errors (false negatives) for classes other than explicitly listed in confusion matrix correctDocCount -= matrixResult.getConfusionMatrix().get(i).getOtherPredictedClassDocCount(); - classes.add(new PerClassResult(className, ((double)correctDocCount) / totalDocCount)); + classes.add(new PerClassSingleValue(className, ((double)correctDocCount) / totalDocCount)); } return classes; } @@ -209,10 +207,10 @@ public static class Result implements EvaluationMetricResult { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("accuracy_result", true, a -> new Result((List) a[0], (double) a[1])); + new ConstructingObjectParser<>("accuracy_result", true, a -> new Result((List) a[0], (double) a[1])); static { - PARSER.declareObjectArray(constructorArg(), PerClassResult.PARSER, CLASSES); + PARSER.declareObjectArray(constructorArg(), PerClassSingleValue.PARSER, CLASSES); PARSER.declareDouble(constructorArg(), OVERALL_ACCURACY); } @@ -221,17 +219,17 @@ public static Result fromXContent(XContentParser parser) { } /** List of per-class results. */ - private final List classes; + private final List classes; /** Fraction of documents for which predicted class equals the actual class. */ private final double overallAccuracy; - public Result(List classes, double overallAccuracy) { + public Result(List classes, double overallAccuracy) { this.classes = Collections.unmodifiableList(ExceptionsHelper.requireNonNull(classes, CLASSES)); this.overallAccuracy = overallAccuracy; } public Result(StreamInput in) throws IOException { - this.classes = Collections.unmodifiableList(in.readList(PerClassResult::new)); + this.classes = Collections.unmodifiableList(in.readList(PerClassSingleValue::new)); this.overallAccuracy = in.readDouble(); } @@ -245,7 +243,7 @@ public String getMetricName() { return NAME.getPreferredName(); } - public List getClasses() { + public List getClasses() { return classes; } @@ -282,71 +280,4 @@ public int hashCode() { return Objects.hash(classes, overallAccuracy); } } - - public static class PerClassResult implements ToXContentObject, Writeable { - - private static final ParseField CLASS_NAME = new ParseField("class_name"); - private static final ParseField ACCURACY = new ParseField("accuracy"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("accuracy_per_class_result", true, a -> new PerClassResult((String) a[0], (double) a[1])); - - static { - PARSER.declareString(constructorArg(), CLASS_NAME); - PARSER.declareDouble(constructorArg(), ACCURACY); - } - - /** Name of the class. */ - private final String className; - /** Fraction of documents that are either true positives or true negatives wrt {@code className}. */ - private final double accuracy; - - public PerClassResult(String className, double accuracy) { - this.className = ExceptionsHelper.requireNonNull(className, CLASS_NAME); - this.accuracy = accuracy; - } - - public PerClassResult(StreamInput in) throws IOException { - this.className = in.readString(); - this.accuracy = in.readDouble(); - } - - public String getClassName() { - return className; - } - - public double getAccuracy() { - return accuracy; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(className); - out.writeDouble(accuracy); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(CLASS_NAME.getPreferredName(), className); - builder.field(ACCURACY.getPreferredName(), accuracy); - builder.endObject(); - return builder; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PerClassResult that = (PerClassResult) o; - return Objects.equals(this.className, that.className) - && this.accuracy == that.accuracy; - } - - @Override - public int hashCode() { - return Objects.hash(className, accuracy); - } - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRoc.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRoc.java index 28353691d12ce..b8b53339db3e8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRoc.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRoc.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationFields; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.common.AbstractAucRoc; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValue.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValue.java new file mode 100644 index 0000000000000..3b33ccd5d60c6 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValue.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class PerClassSingleValue implements ToXContentObject, Writeable { + + private static final ParseField CLASS_NAME = new ParseField("class_name"); + private static final ParseField VALUE = new ParseField("value"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("per_class_result", true, a -> new PerClassSingleValue((String) a[0], (double) a[1])); + + static { + PARSER.declareString(constructorArg(), CLASS_NAME); + PARSER.declareDouble(constructorArg(), VALUE); + } + + private final String className; + private final double value; + + public PerClassSingleValue(String className, double value) { + this.className = ExceptionsHelper.requireNonNull(className, CLASS_NAME); + this.value = value; + } + + public PerClassSingleValue(StreamInput in) throws IOException { + this.className = in.readString(); + this.value = in.readDouble(); + } + + public String getClassName() { + return className; + } + + public double getValue() { + return value; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(className); + out.writeDouble(value); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CLASS_NAME.getPreferredName(), className); + builder.field(VALUE.getPreferredName(), value); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PerClassSingleValue that = (PerClassSingleValue) o; + return Objects.equals(this.className, that.className) + && this.value == that.value; + } + + @Override + public int hashCode() { + return Objects.hash(className, value); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java index e5cabf6e90db2..f49ee69f9c3ff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Precision.java @@ -10,11 +10,9 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryBuilders; @@ -149,13 +147,13 @@ public void process(Aggregations aggs) { aggs.get(AVG_PRECISION_AGG_NAME) instanceof NumericMetricsAggregation.SingleValue) { Filters byPredictedClassAgg = aggs.get(BY_PREDICTED_CLASS_AGG_NAME); NumericMetricsAggregation.SingleValue avgPrecisionAgg = aggs.get(AVG_PRECISION_AGG_NAME); - List classes = new ArrayList<>(byPredictedClassAgg.getBuckets().size()); + List classes = new ArrayList<>(byPredictedClassAgg.getBuckets().size()); for (Filters.Bucket bucket : byPredictedClassAgg.getBuckets()) { String className = bucket.getKeyAsString(); NumericMetricsAggregation.SingleValue precisionAgg = bucket.getAggregations().get(PER_PREDICTED_CLASS_PRECISION_AGG_NAME); double precision = precisionAgg.value(); if (Double.isFinite(precision)) { - classes.add(new PerClassResult(className, precision)); + classes.add(new PerClassSingleValue(className, precision)); } } result.set(new Result(classes, avgPrecisionAgg.value())); @@ -197,10 +195,10 @@ public static class Result implements EvaluationMetricResult { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("precision_result", true, a -> new Result((List) a[0], (double) a[1])); + new ConstructingObjectParser<>("precision_result", true, a -> new Result((List) a[0], (double) a[1])); static { - PARSER.declareObjectArray(constructorArg(), PerClassResult.PARSER, CLASSES); + PARSER.declareObjectArray(constructorArg(), PerClassSingleValue.PARSER, CLASSES); PARSER.declareDouble(constructorArg(), AVG_PRECISION); } @@ -209,17 +207,17 @@ public static Result fromXContent(XContentParser parser) { } /** List of per-class results. */ - private final List classes; + private final List classes; /** Average of per-class precisions. */ private final double avgPrecision; - public Result(List classes, double avgPrecision) { + public Result(List classes, double avgPrecision) { this.classes = Collections.unmodifiableList(ExceptionsHelper.requireNonNull(classes, CLASSES)); this.avgPrecision = avgPrecision; } public Result(StreamInput in) throws IOException { - this.classes = Collections.unmodifiableList(in.readList(PerClassResult::new)); + this.classes = Collections.unmodifiableList(in.readList(PerClassSingleValue::new)); this.avgPrecision = in.readDouble(); } @@ -233,7 +231,7 @@ public String getMetricName() { return NAME.getPreferredName(); } - public List getClasses() { + public List getClasses() { return classes; } @@ -270,71 +268,4 @@ public int hashCode() { return Objects.hash(classes, avgPrecision); } } - - public static class PerClassResult implements ToXContentObject, Writeable { - - private static final ParseField CLASS_NAME = new ParseField("class_name"); - private static final ParseField PRECISION = new ParseField("precision"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("precision_per_class_result", true, a -> new PerClassResult((String) a[0], (double) a[1])); - - static { - PARSER.declareString(constructorArg(), CLASS_NAME); - PARSER.declareDouble(constructorArg(), PRECISION); - } - - /** Name of the class. */ - private final String className; - /** Fraction of documents predicted as belonging to the {@code predictedClass} class predicted correctly. */ - private final double precision; - - public PerClassResult(String className, double precision) { - this.className = ExceptionsHelper.requireNonNull(className, CLASS_NAME); - this.precision = precision; - } - - public PerClassResult(StreamInput in) throws IOException { - this.className = in.readString(); - this.precision = in.readDouble(); - } - - public String getClassName() { - return className; - } - - public double getPrecision() { - return precision; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(className); - out.writeDouble(precision); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(CLASS_NAME.getPreferredName(), className); - builder.field(PRECISION.getPreferredName(), precision); - builder.endObject(); - return builder; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PerClassResult that = (PerClassResult) o; - return Objects.equals(this.className, that.className) - && this.precision == that.precision; - } - - @Override - public int hashCode() { - return Objects.hash(className, precision); - } - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java index 8592a33ac0436..cfc7f810e4edb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Recall.java @@ -10,11 +10,9 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.Script; @@ -128,11 +126,11 @@ public void process(Aggregations aggs) { "Cannot calculate average recall. Cardinality of field [{}] is too high", actualField.get()); } NumericMetricsAggregation.SingleValue avgRecallAgg = aggs.get(AVG_RECALL_AGG_NAME); - List classes = new ArrayList<>(byActualClassAgg.getBuckets().size()); + List classes = new ArrayList<>(byActualClassAgg.getBuckets().size()); for (Terms.Bucket bucket : byActualClassAgg.getBuckets()) { String className = bucket.getKeyAsString(); NumericMetricsAggregation.SingleValue recallAgg = bucket.getAggregations().get(PER_ACTUAL_CLASS_RECALL_AGG_NAME); - classes.add(new PerClassResult(className, recallAgg.value())); + classes.add(new PerClassSingleValue(className, recallAgg.value())); } result.set(new Result(classes, avgRecallAgg.value())); } @@ -173,10 +171,10 @@ public static class Result implements EvaluationMetricResult { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("recall_result", true, a -> new Result((List) a[0], (double) a[1])); + new ConstructingObjectParser<>("recall_result", true, a -> new Result((List) a[0], (double) a[1])); static { - PARSER.declareObjectArray(constructorArg(), PerClassResult.PARSER, CLASSES); + PARSER.declareObjectArray(constructorArg(), PerClassSingleValue.PARSER, CLASSES); PARSER.declareDouble(constructorArg(), AVG_RECALL); } @@ -185,17 +183,17 @@ public static Result fromXContent(XContentParser parser) { } /** List of per-class results. */ - private final List classes; + private final List classes; /** Average of per-class recalls. */ private final double avgRecall; - public Result(List classes, double avgRecall) { + public Result(List classes, double avgRecall) { this.classes = Collections.unmodifiableList(ExceptionsHelper.requireNonNull(classes, CLASSES)); this.avgRecall = avgRecall; } public Result(StreamInput in) throws IOException { - this.classes = Collections.unmodifiableList(in.readList(PerClassResult::new)); + this.classes = Collections.unmodifiableList(in.readList(PerClassSingleValue::new)); this.avgRecall = in.readDouble(); } @@ -209,7 +207,7 @@ public String getMetricName() { return NAME.getPreferredName(); } - public List getClasses() { + public List getClasses() { return classes; } @@ -246,71 +244,4 @@ public int hashCode() { return Objects.hash(classes, avgRecall); } } - - public static class PerClassResult implements ToXContentObject, Writeable { - - private static final ParseField CLASS_NAME = new ParseField("class_name"); - private static final ParseField RECALL = new ParseField("recall"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("recall_per_class_result", true, a -> new PerClassResult((String) a[0], (double) a[1])); - - static { - PARSER.declareString(constructorArg(), CLASS_NAME); - PARSER.declareDouble(constructorArg(), RECALL); - } - - /** Name of the class. */ - private final String className; - /** Fraction of documents actually belonging to the {@code actualClass} class predicted correctly. */ - private final double recall; - - public PerClassResult(String className, double recall) { - this.className = ExceptionsHelper.requireNonNull(className, CLASS_NAME); - this.recall = recall; - } - - public PerClassResult(StreamInput in) throws IOException { - this.className = in.readString(); - this.recall = in.readDouble(); - } - - public String getClassName() { - return className; - } - - public double getRecall() { - return recall; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(className); - out.writeDouble(recall); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(CLASS_NAME.getPreferredName(), className); - builder.field(RECALL.getPreferredName(), recall); - builder.endObject(); - return builder; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PerClassResult that = (PerClassResult) o; - return Objects.equals(this.className, that.className) - && this.recall == that.recall; - } - - @Override - public int hashCode() { - return Objects.hash(className, recall); - } - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AbstractAucRoc.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/common/AbstractAucRoc.java similarity index 92% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AbstractAucRoc.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/common/AbstractAucRoc.java index b3bb6dcefae19..5d91e6e362669 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AbstractAucRoc.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/common/AbstractAucRoc.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification; +package org.elasticsearch.xpack.core.ml.dataframe.evaluation.common; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; @@ -25,8 +25,6 @@ import java.util.List; import java.util.Objects; -import static org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider.registeredMetricName; - /** * Area under the curve (AUC) of the receiver operating characteristic (ROC). * The ROC curve is a plot of the TPR (true positive rate) against @@ -165,7 +163,7 @@ public static final class AucRocPoint implements Comparable, ToXCon private final double fpr; private final double threshold; - AucRocPoint(double tpr, double fpr, double threshold) { + public AucRocPoint(double tpr, double fpr, double threshold) { this.tpr = tpr; this.fpr = fpr; this.threshold = threshold; @@ -229,24 +227,26 @@ private static double interpolate(double x, double x1, double y1, double x2, dou public static class Result implements EvaluationMetricResult { - private static final String SCORE = "score"; + public static final String NAME = "auc_roc_result"; + + private static final String VALUE = "value"; private static final String CURVE = "curve"; - private final double score; + private final double value; private final List curve; - public Result(double score, List curve) { - this.score = score; + public Result(double value, List curve) { + this.value = value; this.curve = Objects.requireNonNull(curve); } public Result(StreamInput in) throws IOException { - this.score = in.readDouble(); + this.value = in.readDouble(); this.curve = in.readList(AucRocPoint::new); } - public double getScore() { - return score; + public double getValue() { + return value; } public List getCurve() { @@ -255,24 +255,24 @@ public List getCurve() { @Override public String getWriteableName() { - return registeredMetricName(Classification.NAME, NAME); + return NAME; } @Override public String getMetricName() { - return NAME.getPreferredName(); + return AbstractAucRoc.NAME.getPreferredName(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(score); + out.writeDouble(value); out.writeList(curve); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(SCORE, score); + builder.field(VALUE, value); if (curve.isEmpty() == false) { builder.field(CURVE, curve); } @@ -285,13 +285,13 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Result that = (Result) o; - return score == that.score + return value == that.value && Objects.equals(curve, that.curve); } @Override public int hashCode() { - return Objects.hash(score, curve); + return Objects.hash(value, curve); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/AucRoc.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/AucRoc.java index fb3ce8da24bef..7d4a50ddb9e11 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/AucRoc.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/AucRoc.java @@ -23,7 +23,7 @@ import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationFields; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters; -import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.AbstractAucRoc; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.common.AbstractAucRoc; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyResultTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyResultTests.java index 176aa6e9a309b..2dfcdcc48978a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyResultTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyResultTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; -import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy.PerClassResult; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy.Result; import java.util.ArrayList; @@ -22,10 +21,10 @@ public class AccuracyResultTests extends AbstractWireSerializingTestCase public static Result createRandom() { int numClasses = randomIntBetween(2, 100); List classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList()); - List classes = new ArrayList<>(numClasses); + List classes = new ArrayList<>(numClasses); for (int i = 0; i < numClasses; i++) { double accuracy = randomDoubleBetween(0.0, 1.0, true); - classes.add(new PerClassResult(classNames.get(i), accuracy)); + classes.add(new PerClassSingleValue(classNames.get(i), accuracy)); } double overallAccuracy = randomDoubleBetween(0.0, 1.0, true); return new Result(classes, overallAccuracy); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java index 44f9bd653c8f0..e9a95df6a3452 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationFields; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationParameters; -import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy.PerClassResult; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy.Result; import java.io.IOException; @@ -95,8 +94,8 @@ public void testProcess() { result.getClasses(), equalTo( List.of( - new PerClassResult("dog", 0.5), - new PerClassResult("cat", 0.5)))); + new PerClassSingleValue("dog", 0.5), + new PerClassSingleValue("cat", 0.5)))); assertThat(result.getOverallAccuracy(), equalTo(0.5)); } @@ -155,9 +154,9 @@ public void testComputePerClassAccuracy() { 0)), equalTo( List.of( - new Accuracy.PerClassResult("A", 25.0 / 51), // 13 false positives, 13 false negatives - new Accuracy.PerClassResult("B", 26.0 / 51), // 8 false positives, 17 false negatives - new Accuracy.PerClassResult("C", 28.0 / 51))) // 13 false positives, 10 false negatives + new PerClassSingleValue("A", 25.0 / 51), // 13 false positives, 13 false negatives + new PerClassSingleValue("B", 26.0 / 51), // 8 false positives, 17 false negatives + new PerClassSingleValue("C", 28.0 / 51))) // 13 false positives, 10 false negatives ); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocResultTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocResultTests.java index 580c1c85fbf95..ddc39ec6d9c1c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocResultTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocResultTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; -import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.AbstractAucRoc.AucRocPoint; -import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.AbstractAucRoc.Result; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.common.AbstractAucRoc.AucRocPoint; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.common.AbstractAucRoc.Result; import java.util.List; import java.util.stream.Collectors; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java new file mode 100644 index 0000000000000..4cbed2a90c429 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; + +public class PerClassSingleValueTests extends AbstractSerializingTestCase { + + @Override + protected PerClassSingleValue doParseInstance(XContentParser parser) throws IOException { + return PerClassSingleValue.PARSER.apply(parser, null); + } + + @Override + protected Writeable.Reader instanceReader() { + return PerClassSingleValue::new; + } + + @Override + protected PerClassSingleValue createTestInstance() { + return new PerClassSingleValue(randomAlphaOfLength(10), randomDouble()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionResultTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionResultTests.java index b86448a4daacb..c9cdfb8826f27 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionResultTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionResultTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; -import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision.PerClassResult; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision.Result; import java.util.ArrayList; @@ -22,10 +21,10 @@ public class PrecisionResultTests extends AbstractWireSerializingTestCase classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList()); - List classes = new ArrayList<>(numClasses); + List classes = new ArrayList<>(numClasses); for (int i = 0; i < numClasses; i++) { double precision = randomDoubleBetween(0.0, 1.0, true); - classes.add(new PerClassResult(classNames.get(i), precision)); + classes.add(new PerClassSingleValue(classNames.get(i), precision)); } double avgPrecision = randomDoubleBetween(0.0, 1.0, true); return new Result(classes, avgPrecision); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallResultTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallResultTests.java index a2a44ded76189..7d79a5363b83a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallResultTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallResultTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; -import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall.PerClassResult; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall.Result; import java.util.ArrayList; @@ -22,10 +21,10 @@ public class RecallResultTests extends AbstractWireSerializingTestCase { public static Result createRandom() { int numClasses = randomIntBetween(2, 100); List classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList()); - List classes = new ArrayList<>(numClasses); + List classes = new ArrayList<>(numClasses); for (int i = 0; i < numClasses; i++) { double recall = randomDoubleBetween(0.0, 1.0, true); - classes.add(new PerClassResult(classNames.get(i), recall)); + classes.add(new PerClassSingleValue(classNames.get(i), recall)); } double avgRecall = randomDoubleBetween(0.0, 1.0, true); return new Result(classes, avgRecall); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AbstractAucRocTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/common/AbstractAucRocTests.java similarity index 94% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AbstractAucRocTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/common/AbstractAucRocTests.java index fef8418edb9d7..71666b6972d94 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AbstractAucRocTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/common/AbstractAucRocTests.java @@ -3,9 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification; +package org.elasticsearch.xpack.core.ml.dataframe.evaluation.common; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.AucRoc; import java.util.Arrays; import java.util.List; @@ -20,7 +21,7 @@ public void testCalculateAucScore_GivenZeroPercentiles() { double[] tpPercentiles = zeroPercentiles(); double[] fpPercentiles = zeroPercentiles(); - List curve = AucRoc.buildAucRocCurve(tpPercentiles, fpPercentiles); + List curve = AucRoc.buildAucRocCurve(tpPercentiles, fpPercentiles); double aucRocScore = AucRoc.calculateAucScore(curve); assertThat(aucRocScore, closeTo(0.5, 0.01)); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java index b8395940880e8..3deebf757c3a5 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.AucRoc; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Classification; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.PerClassSingleValue; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall; import org.junit.After; @@ -129,7 +130,7 @@ public void testEvaluate_AllMetrics_KeywordField_CaseSensitivity() { List.of(new Accuracy(), new MulticlassConfusionMatrix(), new Precision(), new Recall()))); Accuracy.Result accuracyResult = (Accuracy.Result) evaluateDataFrameResponse.getMetrics().get(0); - assertThat(accuracyResult.getClasses(), contains(new Accuracy.PerClassResult("crocodile", 0.0))); + assertThat(accuracyResult.getClasses(), contains(new PerClassSingleValue("crocodile", 0.0))); assertThat(accuracyResult.getOverallAccuracy(), equalTo(0.0)); MulticlassConfusionMatrix.Result confusionMatrixResult = @@ -145,7 +146,7 @@ public void testEvaluate_AllMetrics_KeywordField_CaseSensitivity() { assertThat(precisionResult.getAvgPrecision(), is(notANumber())); Recall.Result recallResult = (Recall.Result) evaluateDataFrameResponse.getMetrics().get(3); - assertThat(recallResult.getClasses(), contains(new Recall.PerClassResult("crocodile", 0.0))); + assertThat(recallResult.getClasses(), contains(new PerClassSingleValue("crocodile", 0.0))); assertThat(recallResult.getAvgRecall(), equalTo(0.0)); } @@ -165,13 +166,13 @@ private AucRoc.Result evaluateAucRoc(boolean includeCurve) { public void testEvaluate_AucRoc_DoNotIncludeCurve() { AucRoc.Result aucrocResult = evaluateAucRoc(false); - assertThat(aucrocResult.getScore(), is(closeTo(0.5, 0.0001))); + assertThat(aucrocResult.getValue(), is(closeTo(0.5, 0.0001))); assertThat(aucrocResult.getCurve(), hasSize(0)); } public void testEvaluate_AucRoc_IncludeCurve() { AucRoc.Result aucrocResult = evaluateAucRoc(true); - assertThat(aucrocResult.getScore(), is(closeTo(0.5, 0.0001))); + assertThat(aucrocResult.getValue(), is(closeTo(0.5, 0.0001))); assertThat(aucrocResult.getCurve(), hasSize(greaterThan(0))); } @@ -188,13 +189,13 @@ private Accuracy.Result evaluateAccuracy(String actualField, String predictedFie } public void testEvaluate_Accuracy_KeywordField() { - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Accuracy.PerClassResult("ant", 47.0 / 75), - new Accuracy.PerClassResult("cat", 47.0 / 75), - new Accuracy.PerClassResult("dog", 47.0 / 75), - new Accuracy.PerClassResult("fox", 47.0 / 75), - new Accuracy.PerClassResult("mouse", 47.0 / 75)); + new PerClassSingleValue("ant", 47.0 / 75), + new PerClassSingleValue("cat", 47.0 / 75), + new PerClassSingleValue("dog", 47.0 / 75), + new PerClassSingleValue("fox", 47.0 / 75), + new PerClassSingleValue("mouse", 47.0 / 75)); double expectedOverallAccuracy = 5.0 / 75; Accuracy.Result accuracyResult = evaluateAccuracy(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_KEYWORD_FIELD); @@ -208,13 +209,13 @@ public void testEvaluate_Accuracy_KeywordField() { } public void testEvaluate_Accuracy_IntegerField() { - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Accuracy.PerClassResult("1", 57.0 / 75), - new Accuracy.PerClassResult("2", 54.0 / 75), - new Accuracy.PerClassResult("3", 51.0 / 75), - new Accuracy.PerClassResult("4", 48.0 / 75), - new Accuracy.PerClassResult("5", 45.0 / 75)); + new PerClassSingleValue("1", 57.0 / 75), + new PerClassSingleValue("2", 54.0 / 75), + new PerClassSingleValue("3", 51.0 / 75), + new PerClassSingleValue("4", 48.0 / 75), + new PerClassSingleValue("5", 45.0 / 75)); double expectedOverallAccuracy = 15.0 / 75; Accuracy.Result accuracyResult = evaluateAccuracy(NO_LEGS_INTEGER_FIELD, NO_LEGS_PREDICTION_INTEGER_FIELD); @@ -238,10 +239,10 @@ public void testEvaluate_Accuracy_IntegerField() { } public void testEvaluate_Accuracy_BooleanField() { - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Accuracy.PerClassResult("false", 18.0 / 30), - new Accuracy.PerClassResult("true", 27.0 / 45)); + new PerClassSingleValue("false", 18.0 / 30), + new PerClassSingleValue("true", 27.0 / 45)); double expectedOverallAccuracy = 45.0 / 75; Accuracy.Result accuracyResult = evaluateAccuracy(IS_PREDATOR_BOOLEAN_FIELD, IS_PREDATOR_PREDICTION_BOOLEAN_FIELD); @@ -267,13 +268,13 @@ public void testEvaluate_Accuracy_BooleanField() { public void testEvaluate_Accuracy_FieldTypeMismatch() { { // When actual and predicted fields have different types, the sets of classes are disjoint - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Accuracy.PerClassResult("1", 0.8), - new Accuracy.PerClassResult("2", 0.8), - new Accuracy.PerClassResult("3", 0.8), - new Accuracy.PerClassResult("4", 0.8), - new Accuracy.PerClassResult("5", 0.8)); + new PerClassSingleValue("1", 0.8), + new PerClassSingleValue("2", 0.8), + new PerClassSingleValue("3", 0.8), + new PerClassSingleValue("4", 0.8), + new PerClassSingleValue("5", 0.8)); double expectedOverallAccuracy = 0.0; Accuracy.Result accuracyResult = evaluateAccuracy(NO_LEGS_INTEGER_FIELD, IS_PREDATOR_BOOLEAN_FIELD); @@ -282,10 +283,10 @@ public void testEvaluate_Accuracy_FieldTypeMismatch() { } { // When actual and predicted fields have different types, the sets of classes are disjoint - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Accuracy.PerClassResult("false", 0.6), - new Accuracy.PerClassResult("true", 0.4)); + new PerClassSingleValue("false", 0.6), + new PerClassSingleValue("true", 0.4)); double expectedOverallAccuracy = 0.0; Accuracy.Result accuracyResult = evaluateAccuracy(IS_PREDATOR_BOOLEAN_FIELD, NO_LEGS_INTEGER_FIELD); @@ -307,13 +308,13 @@ private Precision.Result evaluatePrecision(String actualField, String predictedF } public void testEvaluate_Precision_KeywordField() { - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Precision.PerClassResult("ant", 1.0 / 15), - new Precision.PerClassResult("cat", 1.0 / 15), - new Precision.PerClassResult("dog", 1.0 / 15), - new Precision.PerClassResult("fox", 1.0 / 15), - new Precision.PerClassResult("mouse", 1.0 / 15)); + new PerClassSingleValue("ant", 1.0 / 15), + new PerClassSingleValue("cat", 1.0 / 15), + new PerClassSingleValue("dog", 1.0 / 15), + new PerClassSingleValue("fox", 1.0 / 15), + new PerClassSingleValue("mouse", 1.0 / 15)); double expectedAvgPrecision = 5.0 / 75; Precision.Result precisionResult = evaluatePrecision(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_KEYWORD_FIELD); @@ -324,13 +325,13 @@ public void testEvaluate_Precision_KeywordField() { } public void testEvaluate_Precision_IntegerField() { - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Precision.PerClassResult("1", 0.2), - new Precision.PerClassResult("2", 0.2), - new Precision.PerClassResult("3", 0.2), - new Precision.PerClassResult("4", 0.2), - new Precision.PerClassResult("5", 0.2)); + new PerClassSingleValue("1", 0.2), + new PerClassSingleValue("2", 0.2), + new PerClassSingleValue("3", 0.2), + new PerClassSingleValue("4", 0.2), + new PerClassSingleValue("5", 0.2)); double expectedAvgPrecision = 0.2; Precision.Result precisionResult = evaluatePrecision(NO_LEGS_INTEGER_FIELD, NO_LEGS_PREDICTION_INTEGER_FIELD); @@ -348,10 +349,10 @@ public void testEvaluate_Precision_IntegerField() { } public void testEvaluate_Precision_BooleanField() { - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Precision.PerClassResult("false", 0.5), - new Precision.PerClassResult("true", 9.0 / 13)); + new PerClassSingleValue("false", 0.5), + new PerClassSingleValue("true", 9.0 / 13)); double expectedAvgPrecision = 31.0 / 52; Precision.Result precisionResult = evaluatePrecision(IS_PREDATOR_BOOLEAN_FIELD, IS_PREDATOR_PREDICTION_BOOLEAN_FIELD); @@ -407,13 +408,13 @@ private Recall.Result evaluateRecall(String actualField, String predictedField) } public void testEvaluate_Recall_KeywordField() { - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Recall.PerClassResult("ant", 1.0 / 15), - new Recall.PerClassResult("cat", 1.0 / 15), - new Recall.PerClassResult("dog", 1.0 / 15), - new Recall.PerClassResult("fox", 1.0 / 15), - new Recall.PerClassResult("mouse", 1.0 / 15)); + new PerClassSingleValue("ant", 1.0 / 15), + new PerClassSingleValue("cat", 1.0 / 15), + new PerClassSingleValue ("dog", 1.0 / 15), + new PerClassSingleValue("fox", 1.0 / 15), + new PerClassSingleValue("mouse", 1.0 / 15)); double expectedAvgRecall = 5.0 / 75; Recall.Result recallResult = evaluateRecall(ANIMAL_NAME_KEYWORD_FIELD, ANIMAL_NAME_PREDICTION_KEYWORD_FIELD); @@ -424,13 +425,13 @@ public void testEvaluate_Recall_KeywordField() { } public void testEvaluate_Recall_IntegerField() { - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Recall.PerClassResult("1", 1.0 / 15), - new Recall.PerClassResult("2", 2.0 / 15), - new Recall.PerClassResult("3", 3.0 / 15), - new Recall.PerClassResult("4", 4.0 / 15), - new Recall.PerClassResult("5", 5.0 / 15)); + new PerClassSingleValue("1", 1.0 / 15), + new PerClassSingleValue("2", 2.0 / 15), + new PerClassSingleValue("3", 3.0 / 15), + new PerClassSingleValue("4", 4.0 / 15), + new PerClassSingleValue("5", 5.0 / 15)); double expectedAvgRecall = 3.0 / 15; Recall.Result recallResult = evaluateRecall(NO_LEGS_INTEGER_FIELD, NO_LEGS_PREDICTION_INTEGER_FIELD); @@ -448,10 +449,10 @@ public void testEvaluate_Recall_IntegerField() { } public void testEvaluate_Recall_BooleanField() { - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Recall.PerClassResult("true", 0.6), - new Recall.PerClassResult("false", 0.6)); + new PerClassSingleValue("true", 0.6), + new PerClassSingleValue("false", 0.6)); double expectedAvgRecall = 0.6; Recall.Result recallResult = evaluateRecall(IS_PREDATOR_BOOLEAN_FIELD, IS_PREDATOR_PREDICTION_BOOLEAN_FIELD); @@ -471,13 +472,13 @@ public void testEvaluate_Recall_BooleanField() { public void testEvaluate_Recall_FieldTypeMismatch() { { // When actual and predicted fields have different types, the sets of classes are disjoint, hence 0.0 results here - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Recall.PerClassResult("1", 0.0), - new Recall.PerClassResult("2", 0.0), - new Recall.PerClassResult("3", 0.0), - new Recall.PerClassResult("4", 0.0), - new Recall.PerClassResult("5", 0.0)); + new PerClassSingleValue("1", 0.0), + new PerClassSingleValue("2", 0.0), + new PerClassSingleValue("3", 0.0), + new PerClassSingleValue("4", 0.0), + new PerClassSingleValue("5", 0.0)); double expectedAvgRecall = 0.0; Recall.Result recallResult = evaluateRecall(NO_LEGS_INTEGER_FIELD, IS_PREDATOR_BOOLEAN_FIELD); @@ -486,10 +487,10 @@ public void testEvaluate_Recall_FieldTypeMismatch() { } { // When actual and predicted fields have different types, the sets of classes are disjoint, hence 0.0 results here - List expectedPerClassResults = + List expectedPerClassResults = List.of( - new Recall.PerClassResult("true", 0.0), - new Recall.PerClassResult("false", 0.0)); + new PerClassSingleValue("true", 0.0), + new PerClassSingleValue("false", 0.0)); double expectedAvgRecall = 0.0; Recall.Result recallResult = evaluateRecall(IS_PREDATOR_BOOLEAN_FIELD, NO_LEGS_INTEGER_FIELD); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java index 81303e647f789..c544ee995a2a1 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java @@ -40,6 +40,7 @@ import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Accuracy; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.AucRoc; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.MulticlassConfusionMatrix; +import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.PerClassSingleValue; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Precision; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.classification.Recall; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; @@ -969,16 +970,16 @@ private void assertEvaluation(String dependentVariable, List dependentVar { // Accuracy Accuracy.Result accuracyResult = (Accuracy.Result) evaluateDataFrameResponse.getMetrics().get(0); assertThat(accuracyResult.getMetricName(), equalTo(Accuracy.NAME.getPreferredName())); - for (Accuracy.PerClassResult klass : accuracyResult.getClasses()) { + for (PerClassSingleValue klass : accuracyResult.getClasses()) { assertThat(klass.getClassName(), is(in(dependentVariableValuesAsStrings))); - assertThat(klass.getAccuracy(), allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(1.0))); + assertThat(klass.getValue(), allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(1.0))); } } { // AucRoc AucRoc.Result aucRocResult = (AucRoc.Result) evaluateDataFrameResponse.getMetrics().get(1); assertThat(aucRocResult.getMetricName(), equalTo(AucRoc.NAME.getPreferredName())); - assertThat(aucRocResult.getScore(), allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(1.0))); + assertThat(aucRocResult.getValue(), allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(1.0))); assertThat(aucRocResult.getCurve(), hasSize(greaterThan(0))); } @@ -1004,18 +1005,18 @@ private void assertEvaluation(String dependentVariable, List dependentVar { // Precision Precision.Result precisionResult = (Precision.Result) evaluateDataFrameResponse.getMetrics().get(3); assertThat(precisionResult.getMetricName(), equalTo(Precision.NAME.getPreferredName())); - for (Precision.PerClassResult klass : precisionResult.getClasses()) { + for (PerClassSingleValue klass : precisionResult.getClasses()) { assertThat(klass.getClassName(), is(in(dependentVariableValuesAsStrings))); - assertThat(klass.getPrecision(), allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(1.0))); + assertThat(klass.getValue(), allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(1.0))); } } { // Recall Recall.Result recallResult = (Recall.Result) evaluateDataFrameResponse.getMetrics().get(4); assertThat(recallResult.getMetricName(), equalTo(Recall.NAME.getPreferredName())); - for (Recall.PerClassResult klass : recallResult.getClasses()) { + for (PerClassSingleValue klass : recallResult.getClasses()) { assertThat(klass.getClassName(), is(in(dependentVariableValuesAsStrings))); - assertThat(klass.getRecall(), allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(1.0))); + assertThat(klass.getValue(), allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(1.0))); } } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionEvaluationIT.java index 22a18654dc872..3bd7be05236da 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionEvaluationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionEvaluationIT.java @@ -97,13 +97,13 @@ private AucRoc.Result evaluateAucRoc(String actualField, String predictedField, public void testEvaluate_AucRoc_DoNotIncludeCurve() { AucRoc.Result aucrocResult = evaluateAucRoc(IS_PREDATOR_BOOLEAN_FIELD, IS_PREDATOR_PREDICTION_PROBABILITY_FIELD, false); - assertThat(aucrocResult.getScore(), is(closeTo(1.0, 0.0001))); + assertThat(aucrocResult.getValue(), is(closeTo(1.0, 0.0001))); assertThat(aucrocResult.getCurve(), hasSize(0)); } public void testEvaluate_AucRoc_IncludeCurve() { AucRoc.Result aucrocResult = evaluateAucRoc(IS_PREDATOR_BOOLEAN_FIELD, IS_PREDATOR_PREDICTION_PROBABILITY_FIELD, true); - assertThat(aucrocResult.getScore(), is(closeTo(1.0, 0.0001))); + assertThat(aucrocResult.getValue(), is(closeTo(1.0, 0.0001))); assertThat(aucrocResult.getCurve(), hasSize(greaterThan(0))); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml index 944d6a3a80a11..09cf11d266612 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml @@ -206,7 +206,7 @@ setup: } } } - - match: { outlier_detection.auc_roc.score: 0.9899 } + - match: { outlier_detection.auc_roc.value: 0.9899 } - is_false: outlier_detection.auc_roc.curve --- @@ -226,7 +226,7 @@ setup: } } } - - match: { outlier_detection.auc_roc.score: 0.9899 } + - match: { outlier_detection.auc_roc.value: 0.9899 } - is_false: outlier_detection.auc_roc.curve --- @@ -246,7 +246,7 @@ setup: } } } - - match: { outlier_detection.auc_roc.score: 0.9899 } + - match: { outlier_detection.auc_roc.value: 0.9899 } - is_true: outlier_detection.auc_roc.curve --- @@ -411,7 +411,7 @@ setup: } } } - - is_true: outlier_detection.auc_roc.score + - is_true: outlier_detection.auc_roc.value - is_true: outlier_detection.precision.0\.25 - is_true: outlier_detection.precision.0\.5 - is_true: outlier_detection.precision.0\.75 @@ -721,7 +721,7 @@ setup: } } } - - match: { classification.auc_roc.score: 0.8050111095212122 } + - match: { classification.auc_roc.value: 0.8050111095212122 } - is_false: classification.auc_roc.curve --- "Test classification auc_roc with default top_classes_field": @@ -741,7 +741,7 @@ setup: } } } - - match: { classification.auc_roc.score: 0.8050111095212122 } + - match: { classification.auc_roc.value: 0.8050111095212122 } - is_false: classification.auc_roc.curve --- "Test classification accuracy with missing predicted_field": @@ -778,11 +778,11 @@ setup: classification.accuracy: classes: - class_name: "cat" - accuracy: 0.625 # 5 out of 8 + value: 0.625 # 5 out of 8 - class_name: "dog" - accuracy: 0.75 # 6 out of 8 + value: 0.75 # 6 out of 8 - class_name: "mouse" - accuracy: 0.875 # 7 out of 8 + value: 0.875 # 7 out of 8 overall_accuracy: 0.625 # 5 out of 8 --- "Test classification precision": @@ -804,11 +804,11 @@ setup: classification.precision: classes: - class_name: "cat" - precision: 0.5 # 2 out of 4 + value: 0.5 # 2 out of 4 - class_name: "dog" - precision: 0.6666666666666666 # 2 out of 3 + value: 0.6666666666666666 # 2 out of 3 - class_name: "mouse" - precision: 1.0 # 1 out of 1 + value: 1.0 # 1 out of 1 avg_precision: 0.7222222222222222 --- "Test classification recall": @@ -830,11 +830,11 @@ setup: classification.recall: classes: - class_name: "cat" - recall: 0.6666666666666666 # 2 out of 3 + value: 0.6666666666666666 # 2 out of 3 - class_name: "dog" - recall: 0.6666666666666666 # 2 out of 3 + value: 0.6666666666666666 # 2 out of 3 - class_name: "mouse" - recall: 0.5 # 1 out of 2 + value: 0.5 # 1 out of 2 avg_recall: 0.611111111111111 --- "Test classification multiclass_confusion_matrix": From 9e8221a0ef4c1543184ace665be92ea23410ca12 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 20 Oct 2020 09:46:28 +0200 Subject: [PATCH 35/60] Update gradle wrapper to Gradle 6.7 (#62386) - Ignore Memory Manager thread in Gradle thread filter - Update gradle wrapper to 6.7 GA release --- buildSrc/src/main/resources/minimumGradleVersion | 2 +- .../org/elasticsearch/gradle/test/GradleThreadsFilter.java | 5 ++++- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/resources/minimumGradleVersion b/buildSrc/src/main/resources/minimumGradleVersion index ba92e72f5775b..5840475a6c30a 100644 --- a/buildSrc/src/main/resources/minimumGradleVersion +++ b/buildSrc/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -6.6.1 \ No newline at end of file +6.7 \ No newline at end of file diff --git a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleThreadsFilter.java b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleThreadsFilter.java index e216c123001e3..99268a800d2d7 100644 --- a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleThreadsFilter.java +++ b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleThreadsFilter.java @@ -30,6 +30,9 @@ public class GradleThreadsFilter implements ThreadFilter { @Override public boolean reject(Thread t) { - return t.getName().startsWith("Exec process") || t.getName().startsWith("File watcher consumer"); + return t.getName().startsWith("Exec process") + || t.getName().startsWith("File watcher consumer") + || t.getName().startsWith("Memory manager"); + } } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 0c2d86efbc66d..0cb16492aa0be 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=11657af6356b7587bfb37287b5992e94a9686d5c8a0a1b60b87b9928a2decde5 +distributionSha256Sum=0080de8491f0918e4f529a6db6820fa0b9e818ee2386117f4394f95feb1d5583 From dd7a22fcabb764f123074ede7e0ea258e196513a Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 20 Oct 2020 10:16:43 +0200 Subject: [PATCH 36/60] Add snapshot shard size based test in DiskThresholdDeciderTests (#63546) This commit adds a test in DiskThresholdDeciderTests that verifies the allocation of a snapshot recovery source based shard in the situation where the snapshot shard size was successfully provided by the SnapshotInfoService introduced in #61906 and when the service failed to provide the size. Relates #61906 --- .../decider/DiskThresholdDeciderTests.java | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 3000336e88a77..ccab74674a62e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.routing.allocation.decider; +import com.carrotsearch.hppc.IntHashSet; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterInfoService; @@ -26,11 +27,13 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.DiskUsage; import org.elasticsearch.cluster.ESAllocationTestCase; +import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RecoverySource; @@ -40,6 +43,8 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus; +import org.elasticsearch.cluster.routing.UnassignedInfo.Reason; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; @@ -47,11 +52,18 @@ import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand; import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.repositories.IndexId; import org.elasticsearch.snapshots.EmptySnapshotsInfoService; +import org.elasticsearch.snapshots.InternalSnapshotsInfoService.SnapshotShard; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotShardSizeInfo; import org.elasticsearch.test.gateway.TestGatewayAllocator; import java.util.Arrays; @@ -59,6 +71,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.emptyMap; @@ -70,6 +83,7 @@ import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.oneOf; @@ -1130,6 +1144,108 @@ Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLU " actual free: [20.0%]")); } + public void testDiskThresholdWithSnapshotShardSizes() { + final long shardSizeInBytes = randomBoolean() ? 10L : 50L; + logger.info("--> using shard size [{}]", shardSizeInBytes); + + final Settings diskSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "95%") + .build(); + + final ImmutableOpenMap.Builder usagesBuilder = ImmutableOpenMap.builder(); + usagesBuilder.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 21)); // 79% used + usagesBuilder.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 1)); // 99% used + final ImmutableOpenMap usages = usagesBuilder.build(); + final ClusterInfoService clusterInfoService = () -> new DevNullClusterInfo(usages, usages, ImmutableOpenMap.of()); + + final AllocationDeciders deciders = new AllocationDeciders( + new HashSet<>(Arrays.asList( + new RestoreInProgressAllocationDecider(), + new SameShardAllocationDecider( + Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ), + makeDecider(diskSettings)))); + + final Snapshot snapshot = new Snapshot("_repository", new SnapshotId("_snapshot_name", UUIDs.randomBase64UUID(random()))); + final IndexId indexId = new IndexId("_indexid_name", UUIDs.randomBase64UUID(random())); + final ShardId shardId = new ShardId(new Index("test", IndexMetadata.INDEX_UUID_NA_VALUE), 0); + + final Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putInSyncAllocationIds(0, Set.of(AllocationId.newInitializing().getId())) + ).build(); + + final RoutingTable routingTable = RoutingTable.builder() + .addAsNewRestore( + metadata.index("test"), + new RecoverySource.SnapshotRecoverySource("_restore_uuid", snapshot, Version.CURRENT, indexId), + new IntHashSet() + ).build(); + + final ImmutableOpenMap.Builder shards = ImmutableOpenMap.builder(); + shards.put(shardId, new RestoreInProgress.ShardRestoreStatus("node1")); + + final RestoreInProgress.Builder restores = new RestoreInProgress.Builder() + .add(new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, List.of("test"), shards.build())); + + ClusterState clusterState = ClusterState.builder(new ClusterName(getTestName())) + .metadata(metadata) + .routingTable(routingTable) + .putCustom(RestoreInProgress.TYPE, restores.build()) + .nodes(DiscoveryNodes.builder() + .add(newNode("node1")) + .add(newNode("node2")) // node2 is added because DiskThresholdDecider automatically ignore single-node clusters + ).build(); + + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().map(ShardRouting::unassignedInfo) + .allMatch(unassignedInfo -> Reason.NEW_INDEX_RESTORED.equals(unassignedInfo.getReason())), is(true)); + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().map(ShardRouting::unassignedInfo) + .allMatch(unassignedInfo -> AllocationStatus.NO_ATTEMPT.equals(unassignedInfo.getLastAllocationStatus())), is(true)); + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1)); + + final AtomicReference snapshotShardSizeInfoRef = new AtomicReference<>(SnapshotShardSizeInfo.EMPTY); + final AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(), + new BalancedShardsAllocator(Settings.EMPTY), clusterInfoService, snapshotShardSizeInfoRef::get); + + // reroute triggers snapshot shard size fetching + clusterState = strategy.reroute(clusterState, "reroute"); + logShardStates(clusterState); + + // shard cannot be assigned yet as the snapshot shard size is unknown + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().map(ShardRouting::unassignedInfo) + .allMatch(unassignedInfo -> AllocationStatus.FETCHING_SHARD_DATA.equals(unassignedInfo.getLastAllocationStatus())), is(true)); + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1)); + + final SnapshotShard snapshotShard = new SnapshotShard(snapshot, indexId, shardId); + final ImmutableOpenMap.Builder snapshotShardSizes = ImmutableOpenMap.builder(); + + final boolean shouldAllocate; + if (randomBoolean()) { + logger.info("--> simulating snapshot shards size retrieval success"); + snapshotShardSizes.put(snapshotShard, shardSizeInBytes); + logger.info("--> shard allocation depends on its size"); + shouldAllocate = shardSizeInBytes < usages.get("node1").getFreeBytes(); + } else { + logger.info("--> simulating snapshot shards size retrieval failure"); + snapshotShardSizes.put(snapshotShard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE); + logger.info("--> shard is always allocated when its size could not be retrieved"); + shouldAllocate = true; + } + snapshotShardSizeInfoRef.set(new SnapshotShardSizeInfo(snapshotShardSizes.build())); + + // reroute uses the previous snapshot shard size + clusterState = startInitializingShardsAndReroute(strategy, clusterState); + logShardStates(clusterState); + + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(shouldAllocate ? 0 : 1)); + assertThat(clusterState.getRoutingNodes().shardsWithState("test", INITIALIZING, STARTED).size(), equalTo(shouldAllocate ? 1 : 0)); + } + public void logShardStates(ClusterState state) { RoutingNodes rn = state.getRoutingNodes(); logger.info("--> counts: total: {}, unassigned: {}, initializing: {}, relocating: {}, started: {}", From cfca1e4186933d99d9b0cbd028d8ee05fac827ce Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 20 Oct 2020 11:10:05 +0200 Subject: [PATCH 37/60] DocumentMapperParser to no longer depend directly on MapperService (#63850) This change was mainly triggered by the need for `MapperService` to pass `this` during its constructor when creating `DocumentMapperParser`. Also, `MapperService` is carried around in some places where only a subset of it is needed. With this change we rather carry around the components that are strictly needed, in a couple of cases functions that `MapperService` provides, which helps clarifying the dependency between `DocumentMapperParser`, `DocumentMapper` and `MapperService`, as well as removing the need for MapperService to pass `this` to `DocumentMapperParser` --- .../percolator/CandidateQueryTests.java | 8 +-- .../PercolatorFieldMapperTests.java | 30 ++------ .../index/mapper/DocumentMapper.java | 54 +++++---------- .../index/mapper/DocumentMapperParser.java | 68 ++++++++----------- .../index/mapper/DocumentParser.java | 8 ++- .../index/mapper/IdFieldMapper.java | 11 +-- .../elasticsearch/index/mapper/Mapper.java | 4 +- .../index/mapper/MapperService.java | 53 ++++++++++----- .../index/mapper/ParseContext.java | 10 ++- .../index/query/QueryShardContext.java | 2 +- .../index/mapper/DocumentParserTests.java | 10 +-- .../index/mapper/ParametrizedMapperTests.java | 4 +- .../index/mapper/TypeParsersTests.java | 8 ++- .../index/query/QueryShardContextTests.java | 9 ++- .../indices/IndicesServiceTests.java | 4 +- 15 files changed, 127 insertions(+), 156 deletions(-) diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index a73436e0871f0..137f87d1e8ce3 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -77,7 +77,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -88,7 +87,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; @@ -1115,11 +1113,7 @@ private void duelRun(PercolateQuery.QueryStore queryStore, MemoryIndex memoryInd } private void addQuery(Query query, List docs) { - IndexMetadata build = IndexMetadata.builder("") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(0).build(); - IndexSettings settings = new IndexSettings(build, Settings.EMPTY); - ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null); + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null); fieldMapper.processQuery(query, parseContext); ParseContext.Document queryDocument = parseContext.doc(); // Add to string representation of the query to make debugging easier: diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 939ff5a703f7f..27ae625d50b96 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -42,7 +42,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -59,7 +58,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; @@ -183,11 +181,7 @@ public void testExtractTerms() throws Exception { DocumentMapper documentMapper = mapperService.documentMapper(); PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); - IndexMetadata build = IndexMetadata.builder("") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(0).build(); - IndexSettings settings = new IndexSettings(build, Settings.EMPTY); - ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null); + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null); fieldMapper.processQuery(bq.build(), parseContext); ParseContext.Document document = parseContext.doc(); @@ -208,7 +202,7 @@ public void testExtractTerms() throws Exception { bq.add(termQuery1, Occur.MUST); bq.add(termQuery2, Occur.MUST); - parseContext = new ParseContext.InternalParseContext(documentMapper, null, null); + parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null); fieldMapper.processQuery(bq.build(), parseContext); document = parseContext.doc(); @@ -236,12 +230,8 @@ public void testExtractRanges() throws Exception { bq.add(rangeQuery2, Occur.MUST); DocumentMapper documentMapper = mapperService.documentMapper(); - IndexMetadata build = IndexMetadata.builder("") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(0).build(); - IndexSettings settings = new IndexSettings(build, Settings.EMPTY); PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); - ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null); + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null); fieldMapper.processQuery(bq.build(), parseContext); ParseContext.Document document = parseContext.doc(); @@ -266,7 +256,7 @@ public void testExtractRanges() throws Exception { .rangeQuery(15, 20, true, true, null, null, null, context); bq.add(rangeQuery2, Occur.MUST); - parseContext = new ParseContext.InternalParseContext(documentMapper, null, null); + parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null); fieldMapper.processQuery(bq.build(), parseContext); document = parseContext.doc(); @@ -289,11 +279,7 @@ public void testExtractTermsAndRanges_failed() throws Exception { TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true); DocumentMapper documentMapper = mapperService.documentMapper(); PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); - IndexMetadata build = IndexMetadata.builder("") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(0).build(); - IndexSettings settings = new IndexSettings(build, Settings.EMPTY); - ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null); + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null); fieldMapper.processQuery(query, parseContext); ParseContext.Document document = parseContext.doc(); @@ -307,11 +293,7 @@ public void testExtractTermsAndRanges_partial() throws Exception { PhraseQuery phraseQuery = new PhraseQuery("field", "term"); DocumentMapper documentMapper = mapperService.documentMapper(); PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); - IndexMetadata build = IndexMetadata.builder("") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(0).build(); - IndexSettings settings = new IndexSettings(build, Settings.EMPTY); - ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null); + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null); fieldMapper.processQuery(phraseQuery, parseContext); ParseContext.Document document = parseContext.doc(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 9d4a57150486f..2bcb15326f9a3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -37,13 +37,11 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.index.mapper.MetadataFieldMapper.TypeParser; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.stream.Stream; @@ -51,43 +49,32 @@ public class DocumentMapper implements ToXContentFragment { - public static class Builder { - - private final Map, MetadataFieldMapper> metadataMappers = new LinkedHashMap<>(); + public static final class Builder { + private final Map, MetadataFieldMapper> metadataMappers; private final RootObjectMapper rootObjectMapper; private final Mapper.BuilderContext builderContext; private final IndexSettings indexSettings; private final IndexAnalyzers indexAnalyzers; - private final DocumentMapperParser documentMapperParser; private final DocumentParser documentParser; private Map meta; public Builder(RootObjectMapper.Builder builder, MapperService mapperService) { - this.indexSettings = mapperService.getIndexSettings(); - this.indexAnalyzers = mapperService.getIndexAnalyzers(); - this.documentMapperParser = mapperService.documentMapperParser(); - this.documentParser = mapperService.documentParser(); + this(builder, mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), mapperService.documentParser(), + mapperService.getMetadataMappers()); + } + + Builder(RootObjectMapper.Builder builder, + IndexSettings indexSettings, + IndexAnalyzers indexAnalyzers, + DocumentParser documentParser, + Map, MetadataFieldMapper> metadataMappers) { + this.indexSettings = indexSettings; + this.indexAnalyzers = indexAnalyzers; + this.documentParser = documentParser; this.builderContext = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(1)); this.rootObjectMapper = builder.build(builderContext); - - final DocumentMapper existingMapper = mapperService.documentMapper(); - final Map metadataMapperParsers = - mapperService.mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated()); - for (Map.Entry entry : metadataMapperParsers.entrySet()) { - final String name = entry.getKey(); - final MetadataFieldMapper existingMetadataMapper = existingMapper == null - ? null - : (MetadataFieldMapper) existingMapper.mappers().getMapper(name); - final MetadataFieldMapper metadataMapper; - if (existingMetadataMapper == null) { - final TypeParser parser = entry.getValue(); - metadataMapper = parser.getDefault(mapperService.documentMapperParser().parserContext()); - } else { - metadataMapper = existingMetadataMapper; - } - metadataMappers.put(metadataMapper.getClass(), metadataMapper); - } + this.metadataMappers = metadataMappers; } public Builder meta(Map meta) { @@ -108,7 +95,7 @@ public DocumentMapper build() { rootObjectMapper, metadataMappers.values().toArray(new MetadataFieldMapper[0]), meta); - return new DocumentMapper(indexSettings, documentMapperParser, indexAnalyzers, documentParser, mapping); + return new DocumentMapper(indexSettings, indexAnalyzers, documentParser, mapping); } } @@ -120,19 +107,16 @@ public DocumentMapper build() { private final MappingLookup fieldMappers; private final IndexSettings indexSettings; private final IndexAnalyzers indexAnalyzers; - private final DocumentMapperParser documentMapperParser; private final MetadataFieldMapper[] deleteTombstoneMetadataFieldMappers; private final MetadataFieldMapper[] noopTombstoneMetadataFieldMappers; private DocumentMapper(IndexSettings indexSettings, - DocumentMapperParser documentMapperParser, IndexAnalyzers indexAnalyzers, DocumentParser documentParser, Mapping mapping) { this.type = mapping.root().name(); this.typeText = new Text(this.type); this.mapping = mapping; - this.documentMapperParser = documentMapperParser; this.documentParser = documentParser; this.indexSettings = indexSettings; this.indexAnalyzers = indexAnalyzers; @@ -158,10 +142,6 @@ IndexSettings indexSettings() { return indexSettings; } - DocumentMapperParser documentMapperParser() { - return documentMapperParser; - } - IndexAnalyzers indexAnalyzers() { return indexAnalyzers; } @@ -282,7 +262,7 @@ public ObjectMapper findNestedObjectMapper(int nestedDocId, SearchContext sc, Le public DocumentMapper merge(Mapping mapping, MergeReason reason) { Mapping merged = this.mapping.merge(mapping, reason); - return new DocumentMapper(this.indexSettings, this.documentMapperParser, this.indexAnalyzers, this.documentParser, merged); + return new DocumentMapper(this.indexSettings, this.indexAnalyzers, this.documentParser, merged); } public void validate(IndexSettings settings, boolean checkLimits) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 42f0f4469539d..ae84088a1fee8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -22,57 +22,42 @@ import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.script.ScriptService; +import org.elasticsearch.index.analysis.IndexAnalyzers; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.function.Function; import java.util.function.Supplier; public class DocumentMapperParser { - - final MapperService mapperService; - private final SimilarityService similarityService; - private final Supplier queryShardContextSupplier; + private final IndexSettings indexSettings; + private final IndexAnalyzers indexAnalyzers; + private final Function documentTypeResolver; + private final DocumentParser documentParser; + private final Supplier, MetadataFieldMapper>> metadataMappersSupplier; + private final Supplier parserContextSupplier; private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser(); - private final Version indexVersionCreated; - private final Map typeParsers; private final Map rootTypeParsers; - private final ScriptService scriptService; - - public DocumentMapperParser(IndexSettings indexSettings, - MapperService mapperService, - SimilarityService similarityService, - MapperRegistry mapperRegistry, - Supplier queryShardContextSupplier, - ScriptService scriptService) { - this.mapperService = mapperService; - this.similarityService = similarityService; - this.queryShardContextSupplier = queryShardContextSupplier; - this.scriptService = scriptService; - this.typeParsers = mapperRegistry.getMapperParsers(); - this.indexVersionCreated = indexSettings.getIndexVersionCreated(); - this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(indexVersionCreated); - } - - public Mapper.TypeParser.ParserContext parserContext() { - return new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, typeParsers::get, indexVersionCreated, - queryShardContextSupplier, null, scriptService, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - mapperService::isIdFieldDataEnabled); - } - public Mapper.TypeParser.ParserContext parserContext(DateFormatter dateFormatter) { - return new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, typeParsers::get, indexVersionCreated, - queryShardContextSupplier, dateFormatter, scriptService, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - mapperService::isIdFieldDataEnabled); + DocumentMapperParser(IndexSettings indexSettings, + IndexAnalyzers indexAnalyzers, + Function documentTypeResolver, + DocumentParser documentParser, + Supplier, MetadataFieldMapper>> metadataMappersSupplier, + Supplier parserContextSupplier, + Map metadataMapperParsers) { + this.indexSettings = indexSettings; + this.indexAnalyzers = indexAnalyzers; + this.documentTypeResolver = documentTypeResolver; + this.documentParser = documentParser; + this.metadataMappersSupplier = metadataMappersSupplier; + this.parserContextSupplier = parserContextSupplier; + this.rootTypeParsers = metadataMapperParsers; } @SuppressWarnings("unchecked") @@ -86,7 +71,7 @@ public DocumentMapper parse(@Nullable String type, CompressedXContent source) th } } else { String rootName = mapping.keySet().iterator().next(); - if (type == null || type.equals(rootName) || mapperService.resolveDocumentType(type).equals(rootName)) { + if (type == null || type.equals(rootName) || documentTypeResolver.apply(type).equals(rootName)) { type = rootName; mapping = (Map) mapping.get(rootName); } @@ -104,10 +89,11 @@ private DocumentMapper parse(String type, Map mapping) throws Ma throw new MapperParsingException("Failed to derive type"); } - Mapper.TypeParser.ParserContext parserContext = parserContext(); + Mapper.TypeParser.ParserContext parserContext = parserContextSupplier.get(); // parse RootObjectMapper - DocumentMapper.Builder docBuilder = new DocumentMapper.Builder( - (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), mapperService); + RootObjectMapper.Builder root = (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext); + DocumentMapper.Builder docBuilder = new DocumentMapper.Builder(root, indexSettings, indexAnalyzers, documentParser, + metadataMappersSupplier.get()); Iterator> iterator = mapping.entrySet().iterator(); // parse DocumentMapper while(iterator.hasNext()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 353e04d197317..bd5f0cd1f840c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -40,6 +40,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.function.Function; import static org.elasticsearch.index.mapper.FieldMapper.IGNORE_MALFORMED_SETTING; @@ -47,9 +48,12 @@ final class DocumentParser { private final NamedXContentRegistry xContentRegistry; + private final Function dateParserContext; - DocumentParser(NamedXContentRegistry xContentRegistry) { + DocumentParser(NamedXContentRegistry xContentRegistry, + Function dateParserContext) { this.xContentRegistry = xContentRegistry; + this.dateParserContext = dateParserContext; } ParsedDocument parseDocument(SourceToParse source, @@ -62,7 +66,7 @@ ParsedDocument parseDocument(SourceToParse source, try (XContentParser parser = XContentHelper.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, source.source(), xContentType)) { - context = new ParseContext.InternalParseContext(docMapper, source, parser); + context = new ParseContext.InternalParseContext(docMapper, dateParserContext, source, parser); validateStart(parser); internalParseDocument(mapping, metadataFieldsMappers, context, parser); validateEnd(parser); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index e6c5dbd8364d1..23a8d185f9f1b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -54,6 +54,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.BooleanSupplier; import java.util.function.Supplier; /** @@ -95,13 +96,13 @@ public static class Defaults { } } - public static final TypeParser PARSER = new FixedTypeParser(c -> new IdFieldMapper(c::isIdFieldDataEnabled)); + public static final TypeParser PARSER = new FixedTypeParser(c -> new IdFieldMapper(c.isIdFieldDataEnabled())); static final class IdFieldType extends TermBasedFieldType { - private final Supplier fieldDataEnabled; + private final BooleanSupplier fieldDataEnabled; - IdFieldType(Supplier fieldDataEnabled) { + IdFieldType(BooleanSupplier fieldDataEnabled) { super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); this.fieldDataEnabled = fieldDataEnabled; setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); @@ -149,7 +150,7 @@ public Query termsQuery(List values, QueryShardContext context) { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - if (fieldDataEnabled.get() == false) { + if (fieldDataEnabled.getAsBoolean() == false) { throw new IllegalArgumentException("Fielddata access on the _id field is disallowed, " + "you can re-enable it by updating the dynamic cluster setting: " + IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey()); @@ -256,7 +257,7 @@ public boolean advanceExact(int doc) throws IOException { }; } - private IdFieldMapper(Supplier fieldDataEnabled) { + private IdFieldMapper(BooleanSupplier fieldDataEnabled) { super(new IdFieldType(fieldDataEnabled)); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 5531e6f0a8969..4e9249c62a4dc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -118,8 +118,8 @@ public IndexSettings getIndexSettings() { return indexSettings; } - public boolean isIdFieldDataEnabled() { - return idFieldDataEnabled.getAsBoolean(); + public BooleanSupplier isIdFieldDataEnabled() { + return idFieldDataEnabled; } public Settings getSettings() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 18b78a03d27d6..febc940ad40ac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; @@ -56,6 +57,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -107,8 +109,8 @@ public enum MergeReason { private final DocumentParser documentParser; private final Version indexVersionCreated; private final MapperAnalyzerWrapper indexAnalyzer; - final MapperRegistry mapperRegistry; - private final BooleanSupplier idFieldDataEnabled; + private final MapperRegistry mapperRegistry; + private final Supplier parserContextSupplier; private volatile DocumentMapper mapper; @@ -119,12 +121,18 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, super(indexSettings); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); this.indexAnalyzers = indexAnalyzers; - this.documentParser = new DocumentParser(xContentRegistry); - this.documentMapperParser = new DocumentMapperParser(indexSettings, this, similarityService, mapperRegistry, - queryShardContextSupplier, scriptService); this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), MappedFieldType::indexAnalyzer); this.mapperRegistry = mapperRegistry; - this.idFieldDataEnabled = idFieldDataEnabled; + Function parserContextFunction = + dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get, + indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService, indexAnalyzers, indexSettings, + idFieldDataEnabled); + this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction); + Map metadataMapperParsers = + mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated()); + this.parserContextSupplier = () -> parserContextFunction.apply(null); + this.documentMapperParser = new DocumentMapperParser(indexSettings, indexAnalyzers, this::resolveDocumentType, documentParser, + this::getMetadataMappers, parserContextSupplier, metadataMapperParsers); } public boolean hasNested() { @@ -139,14 +147,36 @@ public NamedAnalyzer getNamedAnalyzer(String analyzerName) { return this.indexAnalyzers.get(analyzerName); } + //TODO This is only used in tests, we may want to look into replacing those usages? public DocumentMapperParser documentMapperParser() { return this.documentMapperParser; } + public Mapper.TypeParser.ParserContext parserContext() { + return parserContextSupplier.get(); + } + DocumentParser documentParser() { return this.documentParser; } + Map, MetadataFieldMapper> getMetadataMappers() { + final DocumentMapper existingMapper = mapper; + final Map metadataMapperParsers = + mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated()); + Map, MetadataFieldMapper> metadataMappers = new LinkedHashMap<>(); + if (existingMapper == null) { + for (MetadataFieldMapper.TypeParser parser : metadataMapperParsers.values()) { + MetadataFieldMapper metadataFieldMapper = parser.getDefault(parserContext()); + metadataMappers.put(metadataFieldMapper.getClass(), metadataFieldMapper); + } + + } else { + metadataMappers.putAll(existingMapper.mapping().metadataMappersMap); + } + return metadataMappers; + } + /** * Parses the mappings (formatted as JSON) into a map */ @@ -207,8 +237,6 @@ public boolean updateMapping(final IndexMetadata currentIndexMetadata, final Ind requireRefresh = true; } - - return requireRefresh; } @@ -354,7 +382,7 @@ public static boolean isMappingSourceTyped(String type, Map mapp * being used in the mappings. This allows typeless APIs such as 'index' or 'put mappings' * to work against indices with a custom type name. */ - public String resolveDocumentType(String type) { + private String resolveDocumentType(String type) { if (MapperService.SINGLE_MAPPING_NAME.equals(type)) { if (mapper != null) { return mapper.type(); @@ -421,13 +449,6 @@ public Analyzer indexAnalyzer() { return this.indexAnalyzer; } - /** - * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. - */ - public boolean isIdFieldDataEnabled() { - return idFieldDataEnabled.getAsBoolean(); - } - @Override public void close() throws IOException { indexAnalyzers.close(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java index 118b31bb3f759..2fe26ddd7824d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java @@ -37,6 +37,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; +import java.util.function.Function; public abstract class ParseContext { @@ -286,6 +287,7 @@ public Collection getIgnoredFields() { public static class InternalParseContext extends ParseContext { private final DocumentMapper docMapper; + private final Function parserContextFunction; private final ContentPath path; private final XContentParser parser; private final Document document; @@ -299,8 +301,12 @@ public static class InternalParseContext extends ParseContext { private long numNestedDocs; private boolean docsReversed = false; - public InternalParseContext(DocumentMapper docMapper, SourceToParse source, XContentParser parser) { + public InternalParseContext(DocumentMapper docMapper, + Function parserContextFunction, + SourceToParse source, + XContentParser parser) { this.docMapper = docMapper; + this.parserContextFunction = parserContextFunction; this.path = new ContentPath(0); this.parser = parser; this.document = new Document(); @@ -315,7 +321,7 @@ public InternalParseContext(DocumentMapper docMapper, SourceToParse source, XCon @Override public Mapper.TypeParser.ParserContext parserContext(DateFormatter dateFormatter) { - return docMapper.documentMapperParser().parserContext(dateFormatter); + return parserContextFunction.apply(dateFormatter); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index f8f91c54de186..28267b009247e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -253,7 +253,7 @@ public ObjectMapper getObjectMapper(String name) { * Generally used to handle unmapped fields in the context of sorting. */ public MappedFieldType buildAnonymousFieldType(String type) { - final Mapper.TypeParser.ParserContext parserContext = mapperService.documentMapperParser().parserContext(); + final Mapper.TypeParser.ParserContext parserContext = mapperService.parserContext(); Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new IllegalArgumentException("No mapper found for type [" + type + "]"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 37c4ded743a5c..bb51cf093bddb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -21,15 +21,11 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.plugins.Plugin; @@ -347,11 +343,7 @@ MapperService createMapperService() throws Exception { // creates an object mapper, which is about 100x harder than it should be.... ObjectMapper createObjectMapper(MapperService mapperService, String name) { - IndexMetadata build = IndexMetadata.builder("") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(0).build(); - IndexSettings settings = new IndexSettings(build, Settings.EMPTY); - ParseContext context = new ParseContext.InternalParseContext(mapperService.documentMapper(), null, null); + ParseContext context = new ParseContext.InternalParseContext(mapperService.documentMapper(), null, null, null); String[] nameParts = name.split("\\."); for (int i = 0; i < nameParts.length - 1; ++i) { context.path().add(nameParts[i]); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 244dbef598d97..4fb708398058a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -214,7 +214,9 @@ private static TestMapper fromMapping(String mapping, Version version) { } return null; }, version, () -> null, null, null, - mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), mapperService::isIdFieldDataEnabled); + mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { + throw new UnsupportedOperationException(); + }); return (TestMapper) new TypeParser() .parse("field", XContentHelper.convertToMap(JsonXContent.jsonXContent, mapping, true), pc) .build(new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0))); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 553261a0dcc45..daf34df975157 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -84,7 +84,9 @@ public void testMultiFieldWithinMultiField() throws IOException { when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); Version olderVersion = VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0); Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext(null, type -> typeParser, olderVersion, null, - null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), mapperService::isIdFieldDataEnabled); + null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { + throw new UnsupportedOperationException(); + }); builder.parse("some-field", olderContext, fieldNode); assertWarnings("At least one multi-field, [sub-field], " + @@ -99,7 +101,9 @@ public void testMultiFieldWithinMultiField() throws IOException { Version version = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.CURRENT); Mapper.TypeParser.ParserContext context = new Mapper.TypeParser.ParserContext(null, type -> typeParser, version, null, null, - null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), mapperService::isIdFieldDataEnabled); + null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { + throw new UnsupportedOperationException(); + }); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { TextFieldMapper.Builder bad = new TextFieldMapper.Builder("textField", () -> Lucene.STANDARD_ANALYZER); diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 48210793e9ddb..c4d63e051230d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -50,7 +50,6 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; -import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -312,13 +311,13 @@ private static QueryShardContext createQueryShardContext(String indexUuid, Strin when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.index()).thenReturn(indexMetadata.getIndex()); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - DocumentMapperParser documentMapperParser = mock(DocumentMapperParser.class); Map typeParserMap = IndicesModule.getMappers(Collections.emptyList()); Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(name -> null, typeParserMap::get, Version.CURRENT, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - mapperService::isIdFieldDataEnabled); - when(documentMapperParser.parserContext()).thenReturn(parserContext); - when(mapperService.documentMapperParser()).thenReturn(documentMapperParser); + () -> { + throw new UnsupportedOperationException(); + }); + when(mapperService.parserContext()).thenReturn(parserContext); if (runtimeDocValues != null) { when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { String fieldName = (String)fieldTypeInv.getArguments()[0]; diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index b4d65502f3f3d..f4e63c5ad3b67 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -458,8 +458,8 @@ public void testStandAloneMapperServiceWithPlugins() throws IOException { .numberOfReplicas(0) .build(); MapperService mapperService = indicesService.createIndexMapperService(indexMetadata); - assertNotNull(mapperService.documentMapperParser().parserContext().typeParser("fake-mapper")); - Similarity sim = mapperService.documentMapperParser().parserContext().getSimilarity("test").get(); + assertNotNull(mapperService.parserContext().typeParser("fake-mapper")); + Similarity sim = mapperService.parserContext().getSimilarity("test").get(); assertThat(sim, instanceOf(NonNegativeScoresSimilarity.class)); sim = ((NonNegativeScoresSimilarity) sim).getDelegate(); assertThat(sim, instanceOf(BM25Similarity.class)); From 85c0b5483d79bff8768d5140527d2d3dfaf52074 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 20 Oct 2020 11:39:25 +0200 Subject: [PATCH 38/60] Composite aggregation must check live docs when the index is sorted (#63864) This change ensures that the live docs are checked in the composite aggregator when the index is sorted. --- .../bucket/composite/CompositeAggregator.java | 6 +++++- .../composite/CompositeAggregatorTests.java | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java index 52defae72c454..2791e1edd72c7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java @@ -42,6 +42,7 @@ import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.Weight; import org.apache.lucene.search.comparators.LongComparator; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.RoaringDocIdSet; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.index.IndexSortConfig; @@ -367,8 +368,11 @@ private void processLeafFromQuery(LeafReaderContext ctx, Sort indexSortPrefix) t final LeafBucketCollector inner = queue.getLeafCollector(ctx, getFirstPassCollector(docIdSetBuilder, indexSortPrefix.getSort().length)); inner.setScorer(scorer); + final Bits liveDocs = ctx.reader().getLiveDocs(); while (docIt.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { - inner.collect(docIt.docID()); + if (liveDocs == null || liveDocs.get(docIt.docID())) { + inner.collect(docIt.docID()); + } } } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index e0a8e632d05e7..ee59cdcbc2fe4 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -2252,14 +2252,26 @@ private void executeTestCase(boolean useIndexSort, } try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, config)) { Document document = new Document(); + int id = 0; for (Map> fields : dataset) { - addToDocument(document, fields); - indexWriter.addDocument(document); document.clear(); + addToDocument(id, document, fields); + indexWriter.addDocument(document); + id++; } if (rarely()) { indexWriter.forceMerge(1); } + if (dataset.size() > 0) { + int numDeletes = randomIntBetween(1, 25); + for (int i = 0; i < numDeletes; i++) { + id = randomIntBetween(0, dataset.size() - 1); + indexWriter.deleteDocuments(new Term("id", Integer.toString(id))); + document.clear(); + addToDocument(id, document, dataset.get(id)); + indexWriter.addDocument(document); + } + } } try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = new IndexSearcher(indexReader); @@ -2284,7 +2296,8 @@ private static IndexSettings createIndexSettings(Sort sort) { return IndexSettingsModule.newIndexSettings(new Index("_index", "0"), builder.build()); } - private void addToDocument(Document doc, Map> keys) { + private void addToDocument(int id, Document doc, Map> keys) { + doc.add(new StringField("id", Integer.toString(id), Field.Store.NO)); for (Map.Entry> entry : keys.entrySet()) { final String name = entry.getKey(); for (Object value : entry.getValue()) { From 0204f0af7a9f0544420d96be865fad39869747bd Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 20 Oct 2020 12:01:20 +0200 Subject: [PATCH 39/60] Fix build tools integTest failure after vault update (#63914) --- .../fake_git/remote/gradle.properties | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/gradle.properties diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/gradle.properties b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/gradle.properties new file mode 100644 index 0000000000000..6761776ee0ce4 --- /dev/null +++ b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/gradle.properties @@ -0,0 +1,22 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# forcing to use TLS1.2 to avoid failure in vault +# see https://github.com/hashicorp/vault/issues/8750#issuecomment-631236121 +systemProp.jdk.tls.client.protocols=TLSv1.2 \ No newline at end of file From eae0eaac87f06dcf903203d570d2bec2ab73237a Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 20 Oct 2020 13:33:32 +0200 Subject: [PATCH 40/60] Upgrade to lucene-8.7.0-snapshot-72d8528c3a6 (#63912) --- buildSrc/version.properties | 2 +- .../licenses/lucene-expressions-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-expressions-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-analyzers-icu-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-analyzers-icu-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-analyzers-kuromoji-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - ...ucene-analyzers-kuromoji-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-analyzers-nori-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-analyzers-nori-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-analyzers-phonetic-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - ...ucene-analyzers-phonetic-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-analyzers-smartcn-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - ...lucene-analyzers-smartcn-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-analyzers-stempel-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - ...lucene-analyzers-stempel-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-analyzers-morfologik-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - ...ene-analyzers-morfologik-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-analyzers-common-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-analyzers-common-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-backward-codecs-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-backward-codecs-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + server/licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - server/licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + server/licenses/lucene-grouping-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-grouping-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../licenses/lucene-highlighter-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-highlighter-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + server/licenses/lucene-join-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - server/licenses/lucene-join-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + server/licenses/lucene-memory-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../licenses/lucene-memory-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + server/licenses/lucene-misc-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - server/licenses/lucene-misc-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + server/licenses/lucene-queries-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../licenses/lucene-queries-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../licenses/lucene-queryparser-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-queryparser-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + server/licenses/lucene-sandbox-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../licenses/lucene-sandbox-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../lucene-spatial-extras-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-spatial-extras-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../licenses/lucene-spatial3d-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../lucene-spatial3d-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + server/licenses/lucene-suggest-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../licenses/lucene-suggest-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + .../licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 | 1 - .../licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 | 1 + 47 files changed, 24 insertions(+), 24 deletions(-) delete mode 100644 modules/lang-expression/licenses/lucene-expressions-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-analyzers-common-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-analyzers-common-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-grouping-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-join-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-join-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-memory-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-memory-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-misc-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-misc-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-queries-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-queries-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 server/licenses/lucene-suggest-8.7.0-snapshot-72d8528c3a6.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index c6bf016a4c58c..424eb793a5578 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.0.0 -lucene = 8.7.0-snapshot-5c4168d +lucene = 8.7.0-snapshot-72d8528c3a6 bundled_jdk_vendor = openjdk bundled_jdk = 15+36@779bf45e88a44cbd9ea6621d33e33db1 diff --git a/modules/lang-expression/licenses/lucene-expressions-8.7.0-snapshot-5c4168d.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 8195cc3b93071..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a2576ccfd677b987ba2db31a179f485ebc07a839 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..a27a199ba4266 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +80caf0f653da6497532d1c0a4f82686fe5219f20 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0-snapshot-5c4168d.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 5e66b08b01e02..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -033f8d4270a4882a71c68c872c577431b657300c \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..1844e368d5e3e --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +4d96962c085d360c2d50fd8031b9f4d862b9efa2 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0-snapshot-5c4168d.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index cef78fe791142..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1ab5d0e206561a528c6e020932950de6ca9b9d0 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..4d91439d94801 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +41efebeeafa9007b3e1167e1904f458e49533607 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0-snapshot-5c4168d.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 6be5d5c09e246..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cdbe5c7655215d04061721ee7256719002fea5ee \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..d1f080bd940e4 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +66ce33df9033cfbfac24594f0b2b58f68a77580c \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0-snapshot-5c4168d.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index cbf61fe31debd..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fa1e1d6692efd537ae3a691494ebfd4a4b8d223a \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..46adff460101b --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +b2791388ee7fba44a8675d176634724251f308e1 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0-snapshot-5c4168d.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 690be950ea32e..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -64b1b3f1384957639e372a795bbb5c4cd552752e \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..8206a5ede939b --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +9482f7433edfba7f1ebfcb908efc4a4153842dd6 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0-snapshot-5c4168d.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index dd4e48a15abf4..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -21760e5ba8b9979f3fd396e607fd029b74fd087d \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..bf3f24ae9811d --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +029a82805fcbf8e3749851f972d1054438a144c6 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0-snapshot-5c4168d.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index f718fb9dc8006..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9d5e5028b8bee18526a449d660da7580e25c2605 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..7728687c415c3 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +0f1b11d41ae030711813e38b2f477c5c3124223a \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-analyzers-common-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 4adac089119c0..0000000000000 --- a/server/licenses/lucene-analyzers-common-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3dc61837524a79e9b381bc374a3066bb18dd68cc \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-analyzers-common-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..7bb882591fe26 --- /dev/null +++ b/server/licenses/lucene-analyzers-common-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +74c3a717e0d693ee91676558cec21b0d2af9120b \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-backward-codecs-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 96b92ba677a42..0000000000000 --- a/server/licenses/lucene-backward-codecs-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb0ba162d80777dc826f7a94e37e974c35083f34 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-backward-codecs-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..33b81cb5d886b --- /dev/null +++ b/server/licenses/lucene-backward-codecs-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +cab0e149a9c0a8dad4f7602a6782ba65d8c6e493 \ No newline at end of file diff --git a/server/licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index f45eb8b6bbf11..0000000000000 --- a/server/licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aa783fcf42ca8f781c9e71127f2bea38021f253c \ No newline at end of file diff --git a/server/licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..4a8cbc9bc49c0 --- /dev/null +++ b/server/licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +c4525d38643972dda5a69206416043f592bf6ad6 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-grouping-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 1a3b78cbf358f..0000000000000 --- a/server/licenses/lucene-grouping-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb5dcf4953f1de6238dcee9b05d7f214e35c23a7 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-grouping-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..448e3951e9467 --- /dev/null +++ b/server/licenses/lucene-grouping-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +b0d7d5064c93c41f6f7d193fa08230d330fc61e1 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-highlighter-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 17a15ac783f77..0000000000000 --- a/server/licenses/lucene-highlighter-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c00159a32771bfde5af859431c5f3dd0d13b49ba \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-highlighter-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..6955cdce44262 --- /dev/null +++ b/server/licenses/lucene-highlighter-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +17020a2a3d4bf9ac1812416453eb7d640ae8b633 \ No newline at end of file diff --git a/server/licenses/lucene-join-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-join-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 0c533186c0117..0000000000000 --- a/server/licenses/lucene-join-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2ca129b6f5aa7db22e26cf70a744ba6a0ed9791c \ No newline at end of file diff --git a/server/licenses/lucene-join-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-join-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..4a7dff5cf12be --- /dev/null +++ b/server/licenses/lucene-join-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +3cef052385aa5b6dfdc7fe73c2484ea4c2def4a2 \ No newline at end of file diff --git a/server/licenses/lucene-memory-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-memory-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 354bf44c5c77d..0000000000000 --- a/server/licenses/lucene-memory-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -be31be3e22ea32b1a399556d86606414d0c40bc3 \ No newline at end of file diff --git a/server/licenses/lucene-memory-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-memory-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..f2572d1fae348 --- /dev/null +++ b/server/licenses/lucene-memory-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +703ba54f53d74bca3cc7e10ef981f559b70bafdd \ No newline at end of file diff --git a/server/licenses/lucene-misc-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-misc-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index b6f23aa9c8033..0000000000000 --- a/server/licenses/lucene-misc-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a383ed30869a94d01e38bf9de34afb9c8e4ac112 \ No newline at end of file diff --git a/server/licenses/lucene-misc-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-misc-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..407c7152dc046 --- /dev/null +++ b/server/licenses/lucene-misc-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +3e567c20bad52b49e5d58bf6c6af28ad20dc3571 \ No newline at end of file diff --git a/server/licenses/lucene-queries-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-queries-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index cd9509ce28001..0000000000000 --- a/server/licenses/lucene-queries-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -43ac03d9f172f936f454361ab34eee5cf25ef061 \ No newline at end of file diff --git a/server/licenses/lucene-queries-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-queries-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..78931cec7bffe --- /dev/null +++ b/server/licenses/lucene-queries-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +884f1db345a5c87dfa5468a4cd57126b76b36e77 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-queryparser-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index a85dff959521b..0000000000000 --- a/server/licenses/lucene-queryparser-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -207014ee624e0dbce5fa89dca94dcbbad6cf37fe \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-queryparser-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..1063380f3449d --- /dev/null +++ b/server/licenses/lucene-queryparser-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +d874b79b9f5ce80530749198864051b9d424c9bb \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-sandbox-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index f1f813b5f72fc..0000000000000 --- a/server/licenses/lucene-sandbox-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -185a2c185432ad0bd3fe8e1abf2379bf8118e054 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-sandbox-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..ea9d904fe42f3 --- /dev/null +++ b/server/licenses/lucene-sandbox-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +fedbcb7253538f9811aef44b88e3e731094104e0 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-spatial-extras-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 2be44f73dd44f..0000000000000 --- a/server/licenses/lucene-spatial-extras-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a1007d59644419987b483e15bcdbb99569cec73d \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-spatial-extras-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..739e9a7cfd752 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +be6f22e5e5325f0f9b0d7c7f696228a8aa966f51 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-spatial3d-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 83879acd3ad6c..0000000000000 --- a/server/licenses/lucene-spatial3d-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7428382a681523181b6557380693c17c327b5baa \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-spatial3d-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..55ab8bc4144b4 --- /dev/null +++ b/server/licenses/lucene-spatial3d-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +dbbc693c9d2027bae1f805f187f85698c21a8599 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-8.7.0-snapshot-5c4168d.jar.sha1 b/server/licenses/lucene-suggest-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index 03a58cb4f1eb2..0000000000000 --- a/server/licenses/lucene-suggest-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bd50cc1a08fcd07ace6795c1000cdc881d9d9907 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/server/licenses/lucene-suggest-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..9afabe74cf32c --- /dev/null +++ b/server/licenses/lucene-suggest-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +693d016e5a347cff50f450d371be67e19f16564a \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 deleted file mode 100644 index f45eb8b6bbf11..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.7.0-snapshot-5c4168d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aa783fcf42ca8f781c9e71127f2bea38021f253c \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 new file mode 100644 index 0000000000000..4a8cbc9bc49c0 --- /dev/null +++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.7.0-snapshot-72d8528c3a6.jar.sha1 @@ -0,0 +1 @@ +c4525d38643972dda5a69206416043f592bf6ad6 \ No newline at end of file From d9aea1c83beee93cbfa667b7e9e998d42cd268f8 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 20 Oct 2020 08:12:10 -0400 Subject: [PATCH 41/60] [ML] fix inference binary classification predication label and feature importance (#63688) When calculating feature importance, the leaf values directly correlate the value of the importance. Consequently, positive leaf values -> positive feature importance negative leaf values -> negative feature importance. It follows that for binary classification, this is done such that the importance relates to the leaf values, which relate directly to the "probability of class 1". So, the feature importance calculated is always for the importance as it relates to class 1. The inverse is the importance as it relates to class 0. --- .../trainedmodel/InferenceHelpers.java | 21 +++++----- .../trainedmodel/PredictionFieldType.java | 41 +++++++++++++++---- .../inference/EnsembleInferenceModel.java | 1 - .../inference/TreeInferenceModel.java | 1 - .../PredictionFieldTypeTests.java | 20 ++++++--- .../integration/ModelInferenceActionIT.java | 12 +++--- .../loadingservice/LocalModelTests.java | 12 +++--- 7 files changed, 69 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java index ed82a4da47611..ce8e9abcc2ae3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java @@ -139,7 +139,6 @@ public static List transformFeatureImportanceRegres public static List transformFeatureImportanceClassification( Map featureImportance, - final int predictedValue, @Nullable List classificationLabels, @Nullable PredictionFieldType predictionFieldType) { List importances = new ArrayList<>(featureImportance.size()); @@ -148,20 +147,20 @@ public static List transformFeatureImportanceCl // This indicates logistic regression (binary classification) // If the length > 1, we assume multi-class classification. if (v.length == 1) { - assert predictedValue == 1 || predictedValue == 0; - // If predicted value is `1`, then the other class is `0` - // If predicted value is `0`, then the other class is `1` - final int otherClass = 1 - predictedValue; - String predictedLabel = classificationLabels == null ? null : classificationLabels.get(predictedValue); - String otherLabel = classificationLabels == null ? null : classificationLabels.get(otherClass); + String zeroLabel = classificationLabels == null ? null : classificationLabels.get(0); + String oneLabel = classificationLabels == null ? null : classificationLabels.get(1); + // For feature importance, it is built off of the value in the leaves. + // These leaves indicate which direction the feature pulls the value + // The original importance is an indication of how it pushes or pulls the value towards or from `1` + // To get the importance for the `0` class, we simply invert it. importances.add(new ClassificationFeatureImportance(k, Arrays.asList( new ClassificationFeatureImportance.ClassImportance( - fieldType.transformPredictedValue((double)predictedValue, predictedLabel), - v[0]), + fieldType.transformPredictedValue(0.0, zeroLabel), + -v[0]), new ClassificationFeatureImportance.ClassImportance( - fieldType.transformPredictedValue((double)otherClass, otherLabel), - -v[0]) + fieldType.transformPredictedValue(1.0, oneLabel), + v[0]) ))); } else { List classImportance = new ArrayList<>(v.length); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PredictionFieldType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PredictionFieldType.java index f0a8f08aa0089..f94220b82ce42 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PredictionFieldType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PredictionFieldType.java @@ -52,22 +52,25 @@ public Object transformPredictedValue(Double value, String stringRep) { case STRING: return stringRep == null ? value.toString() : stringRep; case BOOLEAN: - if ((areClose(value, 1.0D) || areClose(value, 0.0D)) == false) { - throw new IllegalArgumentException( - "Cannot transform numbers other than 0.0 or 1.0 to boolean. Provided number [" + value + "]"); + if (isNumberQuickCheck(stringRep)) { + try { + // 1 is true, 0 is false + return Integer.parseInt(stringRep) == 1; + } catch (NumberFormatException nfe) { + // do nothing, allow fall through to final fromDouble + } + } else if (isBoolQuickCheck(stringRep)) { // if we start with t/f case insensitive, it indicates boolean string + return Boolean.parseBoolean(stringRep); } - return areClose(value, 1.0D); + return fromDouble(value); case NUMBER: - if (Strings.isNullOrEmpty(stringRep)) { - return value; - } // Quick check to verify that the string rep is LIKELY a number // Still handles the case where it throws and then returns the underlying value - if (stringRep.charAt(0) == '-' || Character.isDigit(stringRep.charAt(0))) { + if (isNumberQuickCheck(stringRep)) { try { return Long.parseLong(stringRep); } catch (NumberFormatException nfe) { - return value; + // do nothing, allow fall through to final return } } return value; @@ -76,7 +79,27 @@ public Object transformPredictedValue(Double value, String stringRep) { } } + private static boolean fromDouble(double value) { + if ((areClose(value, 1.0D) || areClose(value, 0.0D)) == false) { + throw new IllegalArgumentException( + "Cannot transform numbers other than 0.0 or 1.0 to boolean. Provided number [" + value + "]"); + } + return areClose(value, 1.0D); + } + private static boolean areClose(double value1, double value2) { return Math.abs(value1 - value2) < EPS; } + + private static boolean isNumberQuickCheck(String stringRep) { + return Strings.isNullOrEmpty(stringRep) == false && (stringRep.charAt(0) == '-' || Character.isDigit(stringRep.charAt(0))); + } + + private static boolean isBoolQuickCheck(String stringRep) { + if (Strings.isNullOrEmpty(stringRep)) { + return false; + } + char c = stringRep.charAt(0); + return 't' == c || 'T' == c || 'f' == c || 'F' == c; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/EnsembleInferenceModel.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/EnsembleInferenceModel.java index 72256b35e7fd4..acaa201d08949 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/EnsembleInferenceModel.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/EnsembleInferenceModel.java @@ -213,7 +213,6 @@ private InferenceResults buildResults(double[] processedInferences, classificationLabel(topClasses.v1().getValue(), classificationLabels), topClasses.v2(), transformFeatureImportanceClassification(decodedFeatureImportance, - value.getValue(), classificationLabels, classificationConfig.getPredictionFieldType()), config, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/TreeInferenceModel.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/TreeInferenceModel.java index f33a0075fa708..8fb14654d3b6f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/TreeInferenceModel.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/TreeInferenceModel.java @@ -180,7 +180,6 @@ private InferenceResults buildResult(double[] value, classificationLabel(classificationValue.getValue(), classificationLabels), topClasses.v2(), InferenceHelpers.transformFeatureImportanceClassification(decodedFeatureImportance, - classificationValue.getValue(), classificationLabels, classificationConfig.getPredictionFieldType()), config, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PredictionFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PredictionFieldTypeTests.java index 0901ee62e742c..cb7f5f1cb1870 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PredictionFieldTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PredictionFieldTypeTests.java @@ -13,17 +13,27 @@ public class PredictionFieldTypeTests extends ESTestCase { + private static final String NOT_BOOLEAN = "not_boolean"; + public void testTransformPredictedValueBoolean() { - assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(null, randomBoolean() ? null : randomAlphaOfLength(10)), + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(null, randomBoolean() ? null : NOT_BOOLEAN), is(nullValue())); - assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(1.0, randomBoolean() ? null : randomAlphaOfLength(10)), + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(1.0, randomBoolean() ? null : NOT_BOOLEAN), is(true)); - assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(0.0, randomBoolean() ? null : randomAlphaOfLength(10)), + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(0.0, randomBoolean() ? null : NOT_BOOLEAN), is(false)); + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(0.0, "1"), is(true)); + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(0.0, "0"), is(false)); + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(0.0, "TruE"), is(true)); + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(0.0, "fAlsE"), is(false)); + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(1.0, "0"), is(false)); + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(1.0, "1"), is(true)); + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(1.0, "TruE"), is(true)); + assertThat(PredictionFieldType.BOOLEAN.transformPredictedValue(1.0, "fAlse"), is(false)); expectThrows(IllegalArgumentException.class, - () -> PredictionFieldType.BOOLEAN.transformPredictedValue(0.1, randomBoolean() ? null : randomAlphaOfLength(10))); + () -> PredictionFieldType.BOOLEAN.transformPredictedValue(0.1, randomBoolean() ? null : NOT_BOOLEAN)); expectThrows(IllegalArgumentException.class, - () -> PredictionFieldType.BOOLEAN.transformPredictedValue(1.1, randomBoolean() ? null : randomAlphaOfLength(10))); + () -> PredictionFieldType.BOOLEAN.transformPredictedValue(1.1, randomBoolean() ? null : NOT_BOOLEAN)); } public void testTransformPredictedValueString() { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ModelInferenceActionIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ModelInferenceActionIT.java index 75db6fb0cc0dd..bf5d9ea9224c1 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ModelInferenceActionIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ModelInferenceActionIT.java @@ -165,7 +165,7 @@ public void testInferModels() throws Exception { .stream() .map(i -> ((SingleValueInferenceResults)i).valueAsString()) .collect(Collectors.toList()), - contains("not_to_be", "to_be")); + contains("no", "yes")); // Get top classes request = new InternalInferModelAction.Request(modelId2, toInfer, new ClassificationConfigUpdate(2, null, null, null, null), true); @@ -174,14 +174,14 @@ public void testInferModels() throws Exception { ClassificationInferenceResults classificationInferenceResults = (ClassificationInferenceResults)response.getInferenceResults().get(0); - assertThat(classificationInferenceResults.getTopClasses().get(0).getClassification(), equalTo("not_to_be")); - assertThat(classificationInferenceResults.getTopClasses().get(1).getClassification(), equalTo("to_be")); + assertThat(classificationInferenceResults.getTopClasses().get(0).getClassification(), equalTo("no")); + assertThat(classificationInferenceResults.getTopClasses().get(1).getClassification(), equalTo("yes")); assertThat(classificationInferenceResults.getTopClasses().get(0).getProbability(), greaterThan(classificationInferenceResults.getTopClasses().get(1).getProbability())); classificationInferenceResults = (ClassificationInferenceResults)response.getInferenceResults().get(1); - assertThat(classificationInferenceResults.getTopClasses().get(0).getClassification(), equalTo("to_be")); - assertThat(classificationInferenceResults.getTopClasses().get(1).getClassification(), equalTo("not_to_be")); + assertThat(classificationInferenceResults.getTopClasses().get(0).getClassification(), equalTo("yes")); + assertThat(classificationInferenceResults.getTopClasses().get(1).getClassification(), equalTo("no")); // they should always be in order of Most probable to least assertThat(classificationInferenceResults.getTopClasses().get(0).getProbability(), greaterThan(classificationInferenceResults.getTopClasses().get(1).getProbability())); @@ -192,7 +192,7 @@ public void testInferModels() throws Exception { classificationInferenceResults = (ClassificationInferenceResults)response.getInferenceResults().get(0); assertThat(classificationInferenceResults.getTopClasses(), hasSize(1)); - assertThat(classificationInferenceResults.getTopClasses().get(0).getClassification(), equalTo("to_be")); + assertThat(classificationInferenceResults.getTopClasses().get(0).getClassification(), equalTo("yes")); } public void testInferModelMultiClassModel() throws Exception { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java index e5946417fe2ed..c1d7d951d0511 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java @@ -114,13 +114,13 @@ public void testClassificationInfer() throws Exception { mock(CircuitBreaker.class)); result = getSingleValue(model, fields, ClassificationConfigUpdate.EMPTY_PARAMS); assertThat(result.value(), equalTo(0.0)); - assertThat(result.valueAsString(), equalTo("not_to_be")); + assertThat(result.valueAsString(), equalTo("no")); classificationResult = (ClassificationInferenceResults)getSingleValue(model, fields, new ClassificationConfigUpdate(1, null, null, null, null)); assertThat(classificationResult.getTopClasses().get(0).getProbability(), closeTo(0.5498339973124778, 0.0000001)); - assertThat(classificationResult.getTopClasses().get(0).getClassification(), equalTo("not_to_be")); + assertThat(classificationResult.getTopClasses().get(0).getClassification(), equalTo("no")); assertThat(model.getLatestStatsAndReset().getInferenceCount(), equalTo(2L)); classificationResult = (ClassificationInferenceResults)getSingleValue(model, @@ -169,11 +169,11 @@ public void testClassificationInferWithDifferentPredictionFieldTypes() throws Ex IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); writeResult(result, document, "result_field", modelId); - assertThat(document.getFieldValue("result_field.predicted_value", String.class), equalTo("not_to_be")); + assertThat(document.getFieldValue("result_field.predicted_value", String.class), equalTo("no")); List list = document.getFieldValue("result_field.top_classes", List.class); assertThat(list.size(), equalTo(2)); - assertThat(((Map)list.get(0)).get("class_name"), equalTo("not_to_be")); - assertThat(((Map)list.get(1)).get("class_name"), equalTo("to_be")); + assertThat(((Map)list.get(0)).get("class_name"), equalTo("no")); + assertThat(((Map)list.get(1)).get("class_name"), equalTo("yes")); result = getInferenceResult(model, fields, new ClassificationConfigUpdate(2, null, null, null, PredictionFieldType.NUMBER)); @@ -440,7 +440,7 @@ public static TrainedModel buildClassification(boolean includeLabels) { .addNode(TreeNode.builder(2).setLeafValue(0.0)) .build(); return Ensemble.builder() - .setClassificationLabels(includeLabels ? Arrays.asList("not_to_be", "to_be") : null) + .setClassificationLabels(includeLabels ? Arrays.asList("no", "yes") : null) .setTargetType(TargetType.CLASSIFICATION) .setFeatureNames(featureNames) .setTrainedModels(Arrays.asList(tree1, tree2, tree3)) From 13ef3ab3ea4c63e2235ad70c32a80d367c2a8d29 Mon Sep 17 00:00:00 2001 From: Adam Locke Date: Tue, 20 Oct 2020 08:45:38 -0400 Subject: [PATCH 42/60] Move clone snapshot API page. (#63902) --- .../snapshot-restore/apis/snapshot-restore-apis.asciidoc | 2 ++ docs/reference/snapshot-restore/index.asciidoc | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc index 8848348316eed..cf70d3bcb2eab 100644 --- a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc +++ b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc @@ -19,6 +19,7 @@ For more information, see <>. [[snapshot-management-apis]] === Snapshot management APIs * <> +* <> * <> * <> * <> @@ -29,6 +30,7 @@ include::verify-repo-api.asciidoc[] include::get-repo-api.asciidoc[] include::delete-repo-api.asciidoc[] include::clean-up-repo-api.asciidoc[] +include::clone-snapshot-api.asciidoc[] include::create-snapshot-api.asciidoc[] include::get-snapshot-api.asciidoc[] include::get-snapshot-status-api.asciidoc[] diff --git a/docs/reference/snapshot-restore/index.asciidoc b/docs/reference/snapshot-restore/index.asciidoc index 805b923c6d56d..8286c73276864 100644 --- a/docs/reference/snapshot-restore/index.asciidoc +++ b/docs/reference/snapshot-restore/index.asciidoc @@ -107,7 +107,6 @@ understand the time requirements before proceeding. -- include::register-repository.asciidoc[] -include::apis/clone-snapshot-api.asciidoc[] include::take-snapshot.asciidoc[] include::restore-snapshot.asciidoc[] include::monitor-snapshot-restore.asciidoc[] From 4bdc32227c2c178fbe7ccf38f74669667e7463e7 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 20 Oct 2020 15:03:25 +0200 Subject: [PATCH 43/60] Remove two redundant DocumentMapper methods (#63922) DocumentMapper exposes field types and object mappers through specific getter methods, that call the corresponding getters exposed by MappingLookup. MappingLookup is though exposed directly by DocumentMapper, hence there is no need for additional methods other than the one to retrieve the mapping lookup object. --- .../percolator/PercolateQueryBuilder.java | 2 +- .../index/cache/bitset/BitsetFilterCache.java | 2 +- .../index/get/ShardGetService.java | 2 +- .../index/mapper/DocumentMapper.java | 10 +-------- .../index/mapper/DocumentParser.java | 6 ++--- .../index/mapper/MapperService.java | 12 +++++----- .../mapper/DocumentMapperParserTests.java | 4 ++-- .../index/mapper/KeywordFieldMapperTests.java | 2 +- .../index/mapper/NestedObjectMapperTests.java | 22 +++++++++---------- .../index/mapper/ObjectMapperTests.java | 4 ++-- 10 files changed, 29 insertions(+), 37 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index a583c31831eb6..e901834b72b35 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -471,7 +471,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { docs.add(docMapper.parse(new SourceToParse(context.index().getName(), "_temp_id", document, documentXContentType))); } - FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer) docMapper.mappers().indexAnalyzer(); + FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer)docMapper.mappers().indexAnalyzer(); // Need to this custom impl because FieldNameAnalyzer is strict and the percolator sometimes isn't when // 'index.percolator.map_unmapped_fields_as_string' is enabled: Analyzer analyzer = new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) { diff --git a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java index 73053e6371362..9771011278705 100644 --- a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java +++ b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java @@ -241,7 +241,7 @@ public IndexWarmer.TerminationHandle warmReader(final IndexShard indexShard, fin if (docMapper != null) { if (docMapper.hasNestedObjects()) { hasNested = true; - for (ObjectMapper objectMapper : docMapper.objectMappers().values()) { + for (ObjectMapper objectMapper : docMapper.mappers().objectMappers().values()) { if (objectMapper.nested().isNested()) { ObjectMapper parentObjectMapper = objectMapper.getParentObjectMapper(mapperService); if (parentObjectMapper != null && parentObjectMapper.nested().isNested()) { diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 7ce3fd8ba581a..ae3d36d2b8c27 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -200,7 +200,7 @@ private GetResult innerGetLoadFromStoredFields(String id, String[] storedFields, for (String field : storedFields) { Mapper fieldMapper = docMapper.mappers().getMapper(field); if (fieldMapper == null) { - if (docMapper.objectMappers().get(field) != null) { + if (docMapper.mappers().objectMappers().get(field) != null) { // Only fail if we know it is a object field, missing paths / fields shouldn't fail. throw new IllegalArgumentException("field [" + field + "] isn't a leaf field"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 2bcb15326f9a3..87f0d1b4d3244 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -198,14 +198,6 @@ public MappingLookup mappers() { return this.fieldMappers; } - public FieldTypeLookup fieldTypes() { - return mappers().fieldTypes(); - } - - public Map objectMappers() { - return mappers().objectMappers(); - } - public ParsedDocument parse(SourceToParse source) throws MapperParsingException { return documentParser.parseDocument(source, mapping.metadataMappers, this); } @@ -230,7 +222,7 @@ public ParsedDocument createNoopTombstoneDoc(String index, String reason) throws */ public ObjectMapper findNestedObjectMapper(int nestedDocId, SearchContext sc, LeafReaderContext context) throws IOException { ObjectMapper nestedObjectMapper = null; - for (ObjectMapper objectMapper : objectMappers().values()) { + for (ObjectMapper objectMapper : mappers().objectMappers().values()) { if (!objectMapper.nested().isNested()) { continue; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index bd5f0cd1f840c..99ac0f6f94c8e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -322,7 +322,7 @@ private static ObjectMapper createExistingMapperUpdate(List parent // only prefix with parent mapper if the parent mapper isn't the root (which has a fake name) updateParentName = lastParent.name() + '.' + nameParts[i]; } - ObjectMapper updateParent = docMapper.objectMappers().get(updateParentName); + ObjectMapper updateParent = docMapper.mappers().objectMappers().get(updateParentName); assert updateParent != null : updateParentName + " doesn't exist"; return createUpdate(updateParent, nameParts, i + 1, newMapper); } @@ -822,7 +822,7 @@ private static Tuple getDynamicParentMapper(ParseContext "Could not dynamically add mapping for field [{}]. Existing mapping for [{}] must be of type object but found [{}].", null, String.join(".", paths), currentPath, existingFieldMapper.typeName()); } - mapper = context.docMapper().objectMappers().get(currentPath); + mapper = context.docMapper().mappers().objectMappers().get(currentPath); if (mapper == null) { // One mapping is missing, check if we are allowed to create a dynamic one. ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context); @@ -867,7 +867,7 @@ private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper parentMapper, break; } String parentName = parentMapper.name().substring(0, lastDotNdx); - parentMapper = context.docMapper().objectMappers().get(parentName); + parentMapper = context.docMapper().mappers().objectMappers().get(parentName); if (parentMapper == null) { // If parentMapper is ever null, it means the parent of the current mapper was dynamically created. // But in order to be created dynamically, the dynamic setting of that parent was necessarily true diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index febc940ad40ac..e60222cdd87f4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -88,7 +88,7 @@ public enum MergeReason { * if a shard was moved to a different node or for administrative * purposes. */ - MAPPING_RECOVERY; + MAPPING_RECOVERY } public static final String SINGLE_MAPPING_NAME = "_doc"; @@ -411,7 +411,7 @@ public MappedFieldType fieldType(String fullName) { if (fullName.equals(TypeFieldType.NAME)) { return new TypeFieldType(this.mapper == null ? "_doc" : this.mapper.type()); } - return this.mapper == null ? null : this.mapper.fieldTypes().get(fullName); + return this.mapper == null ? null : this.mapper.mappers().fieldTypes().get(fullName); } /** @@ -423,7 +423,7 @@ public Set simpleMatchToFullName(String pattern) { // no wildcards return Collections.singleton(pattern); } - return this.mapper == null ? Collections.emptySet() : this.mapper.fieldTypes().simpleMatchToFullName(pattern); + return this.mapper == null ? Collections.emptySet() : this.mapper.mappers().fieldTypes().simpleMatchToFullName(pattern); } /** @@ -431,18 +431,18 @@ public Set simpleMatchToFullName(String pattern) { * the 'source path' for a multi-field is the path to its parent field. */ public Set sourcePath(String fullName) { - return this.mapper == null ? Collections.emptySet() : this.mapper.fieldTypes().sourcePaths(fullName); + return this.mapper == null ? Collections.emptySet() : this.mapper.mappers().fieldTypes().sourcePaths(fullName); } /** * Returns all mapped field types. */ public Iterable fieldTypes() { - return this.mapper == null ? Collections.emptySet() : this.mapper.fieldTypes(); + return this.mapper == null ? Collections.emptySet() : this.mapper.mappers().fieldTypes(); } public ObjectMapper getObjectMapper(String name) { - return this.mapper == null ? null : this.mapper.objectMappers().get(name); + return this.mapper == null ? null : this.mapper.mappers().objectMappers().get(name); } public Analyzer indexAnalyzer() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java index 49b344087e0b7..ad55512b862b3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java @@ -28,7 +28,7 @@ public void testFieldNameWithDots() throws Exception { })); assertNotNull(docMapper.mappers().getMapper("foo.bar")); assertNotNull(docMapper.mappers().getMapper("foo.baz")); - assertNotNull(docMapper.objectMappers().get("foo")); + assertNotNull(docMapper.mappers().objectMappers().get("foo")); } public void testFieldNameWithDeepDots() throws Exception { @@ -46,7 +46,7 @@ public void testFieldNameWithDeepDots() throws Exception { })); assertNotNull(docMapper.mappers().getMapper("foo.bar")); assertNotNull(docMapper.mappers().getMapper("foo.baz.deep.field")); - assertNotNull(docMapper.objectMappers().get("foo")); + assertNotNull(docMapper.mappers().objectMappers().get("foo")); } public void testFieldNameWithDotsConflict() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 2da8d388eee8b..eeb4ae58e7d25 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -306,7 +306,7 @@ public void testConfigureSimilarity() throws IOException { MapperService mapperService = createMapperService( fieldMapping(b -> b.field("type", "keyword").field("similarity", "boolean")) ); - MappedFieldType ft = mapperService.documentMapper().fieldTypes().get("field"); + MappedFieldType ft = mapperService.documentMapper().mappers().fieldTypes().get("field"); assertEquals("boolean", ft.getTextSearchInfo().getSimilarity().name()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 4c81fe0b1b898..2c0151f39d4cc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -93,7 +93,7 @@ public void testSingleNested() throws Exception { .parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); - ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1"); + ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); assertThat(nested1Mapper.nested().isNested(), equalTo(true)); ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", BytesReference @@ -145,11 +145,11 @@ public void testMultiNested() throws Exception { .parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); - ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1"); + ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); assertThat(nested1Mapper.nested().isNested(), equalTo(true)); assertThat(nested1Mapper.nested().isIncludeInParent(), equalTo(false)); assertThat(nested1Mapper.nested().isIncludeInRoot(), equalTo(false)); - ObjectMapper nested2Mapper = docMapper.objectMappers().get("nested1.nested2"); + ObjectMapper nested2Mapper = docMapper.mappers().objectMappers().get("nested1.nested2"); assertThat(nested2Mapper.nested().isNested(), equalTo(true)); assertThat(nested2Mapper.nested().isIncludeInParent(), equalTo(false)); assertThat(nested2Mapper.nested().isIncludeInRoot(), equalTo(false)); @@ -206,11 +206,11 @@ public void testMultiObjectAndNested1() throws Exception { .parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); - ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1"); + ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); assertThat(nested1Mapper.nested().isNested(), equalTo(true)); assertThat(nested1Mapper.nested().isIncludeInParent(), equalTo(false)); assertThat(nested1Mapper.nested().isIncludeInRoot(), equalTo(false)); - ObjectMapper nested2Mapper = docMapper.objectMappers().get("nested1.nested2"); + ObjectMapper nested2Mapper = docMapper.mappers().objectMappers().get("nested1.nested2"); assertThat(nested2Mapper.nested().isNested(), equalTo(true)); assertThat(nested2Mapper.nested().isIncludeInParent(), equalTo(true)); assertThat(nested2Mapper.nested().isIncludeInRoot(), equalTo(false)); @@ -269,11 +269,11 @@ public void testMultiObjectAndNested2() throws Exception { .parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); - ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1"); + ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); assertThat(nested1Mapper.nested().isNested(), equalTo(true)); assertThat(nested1Mapper.nested().isIncludeInParent(), equalTo(true)); assertThat(nested1Mapper.nested().isIncludeInRoot(), equalTo(false)); - ObjectMapper nested2Mapper = docMapper.objectMappers().get("nested1.nested2"); + ObjectMapper nested2Mapper = docMapper.mappers().objectMappers().get("nested1.nested2"); assertThat(nested2Mapper.nested().isNested(), equalTo(true)); assertThat(nested2Mapper.nested().isIncludeInParent(), equalTo(true)); assertThat(nested2Mapper.nested().isIncludeInRoot(), equalTo(false)); @@ -330,11 +330,11 @@ public void testMultiRootAndNested1() throws Exception { .parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); - ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1"); + ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); assertThat(nested1Mapper.nested().isNested(), equalTo(true)); assertThat(nested1Mapper.nested().isIncludeInParent(), equalTo(false)); assertThat(nested1Mapper.nested().isIncludeInRoot(), equalTo(false)); - ObjectMapper nested2Mapper = docMapper.objectMappers().get("nested1.nested2"); + ObjectMapper nested2Mapper = docMapper.mappers().objectMappers().get("nested1.nested2"); assertThat(nested2Mapper.nested().isNested(), equalTo(true)); assertThat(nested2Mapper.nested().isIncludeInParent(), equalTo(false)); assertThat(nested2Mapper.nested().isIncludeInRoot(), equalTo(true)); @@ -524,7 +524,7 @@ public void testNestedArrayStrict() throws Exception { .parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); - ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1"); + ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); assertThat(nested1Mapper.nested().isNested(), equalTo(true)); assertThat(nested1Mapper.dynamic(), equalTo(Dynamic.STRICT)); @@ -778,7 +778,7 @@ public void testReorderParent() throws IOException { .mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); - ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1"); + ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); assertThat(nested1Mapper.nested().isNested(), equalTo(true)); ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index bf2ccc1ca6c42..11ba0d2da9b39 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -204,7 +204,7 @@ public void testMergeEnabledForIndexTemplates() throws IOException { .endObject().endObject()); mapper = mapperService.merge("type", new CompressedXContent(update), MergeReason.INDEX_TEMPLATE); - ObjectMapper objectMapper = mapper.objectMappers().get("object"); + ObjectMapper objectMapper = mapper.mappers().objectMappers().get("object"); assertNotNull(objectMapper); assertFalse(objectMapper.isEnabled()); @@ -218,7 +218,7 @@ public void testMergeEnabledForIndexTemplates() throws IOException { .endObject().endObject()); mapper = mapperService.merge("type", new CompressedXContent(update), MergeReason.INDEX_TEMPLATE); - objectMapper = mapper.objectMappers().get("object"); + objectMapper = mapper.mappers().objectMappers().get("object"); assertNotNull(objectMapper); assertTrue(objectMapper.isEnabled()); } From c3d1981021cb59698bf6893be92d31af78bd2bee Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 20 Oct 2020 16:59:53 +0200 Subject: [PATCH 44/60] make sure AggregationTest creates reduced aggregations. (#63931) The test calls InternalAggregationTestCase#createTestInstanceForXContent() instead of InternalAggregationTestCase#createTestInstance() (The method needs to become public) --- .../elasticsearch/search/aggregations/AggregationsTests.java | 2 +- .../aggregations/metrics/InternalScriptedMetricTests.java | 2 +- .../org/elasticsearch/test/InternalAggregationTestCase.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index bdf80d8aa944b..5aa7e4b165e46 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -296,7 +296,7 @@ private static InternalAggregations createTestInstance(final int minNumAggs, fin singleBucketAggTestCase.subAggregationsSupplier = () -> InternalAggregations.EMPTY; } } - aggs.add(testCase.createTestInstance()); + aggs.add(testCase.createTestInstanceForXContent()); } return InternalAggregations.from(aggs); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java index f94a881f28bbc..5201c3646d91b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java @@ -160,7 +160,7 @@ protected void assertReduced(InternalScriptedMetric reduced, List Date: Tue, 20 Oct 2020 11:28:29 -0400 Subject: [PATCH 45/60] [ML] adding new flag exclude_generated that removes generated fields in GET config APIs (#63899) When exporting and cloning ml configurations in a cluster it can be frustrating to remove all the fields that were generated by the plugin. Especially as the number of these fields change from version to version. This flag, exclude_generated, allows the GET config APIs to return configurations with these generated fields removed. APIs supporting this flag: - GET _ml/anomaly_detection/ - GET _ml/datafeeds/ - GET _ml/data_frame/analytics/ The following fields are not returned in the objects: - any field that is not user settable (e.g. version, create_time) - any field that is a calculated default value (e.g. datafeed chunking_config) - any field that is automatically set via another Elastic stack process (e.g. anomaly job custom_settings.created_by) relates to #63055 --- .../client/MLRequestConverters.java | 16 ++--- .../ml/GetDataFrameAnalyticsRequest.java | 18 +++--- .../client/ml/GetDatafeedRequest.java | 18 +++--- .../client/ml/GetJobRequest.java | 18 +++--- .../client/ml/GetTrainedModelsRequest.java | 18 +++--- .../MlClientDocumentationIT.java | 8 +-- .../apis/get-datafeed.asciidoc | 8 +-- .../anomaly-detection/apis/get-job.asciidoc | 10 ++-- .../apis/get-dfanalytics.asciidoc | 60 +++++++++---------- .../apis/get-trained-models.asciidoc | 4 +- docs/reference/ml/ml-shared.asciidoc | 4 +- .../core/ml/datafeed/DatafeedConfig.java | 9 ++- .../dataframe/DataFrameAnalyticsConfig.java | 6 +- .../core/ml/inference/TrainedModelConfig.java | 6 +- .../xpack/core/ml/job/config/Job.java | 6 +- .../xpack/core/ml/utils/ToXContentParams.java | 2 +- .../ml/integration/MlBasicMultiNodeIT.java | 31 ++++++---- .../xpack/ml/integration/TrainedModelIT.java | 3 +- .../datafeeds/RestGetDatafeedsAction.java | 4 +- .../RestGetDataFrameAnalyticsAction.java | 4 +- .../inference/RestGetTrainedModelsAction.java | 4 +- .../xpack/ml/rest/job/RestGetJobsAction.java | 4 +- .../api/ml.get_data_frame_analytics.json | 2 +- .../rest-api-spec/api/ml.get_datafeeds.json | 2 +- .../rest-api-spec/api/ml.get_jobs.json | 2 +- .../api/ml.get_trained_models.json | 2 +- .../test/ml/data_frame_analytics_crud.yml | 3 +- .../rest-api-spec/test/ml/datafeeds_crud.yml | 4 +- .../rest-api-spec/test/ml/inference_crud.yml | 6 +- .../rest-api-spec/test/ml/jobs_get.yml | 3 +- 30 files changed, 146 insertions(+), 139 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 872a69b1b95c3..71e9580b48eb5 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -123,8 +123,8 @@ static Request getJob(GetJobRequest getJobRequest) { if (getJobRequest.getAllowNoMatch() != null) { params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch())); } - if (getJobRequest.getForExport() != null) { - params.putParam(GetJobRequest.FOR_EXPORT, Boolean.toString(getJobRequest.getForExport())); + if (getJobRequest.getExcludeGenerated() != null) { + params.putParam(GetJobRequest.EXCLUDE_GENERATED, Boolean.toString(getJobRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -273,8 +273,8 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) { params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getDatafeedRequest.getAllowNoMatch())); } - if (getDatafeedRequest.getForExport() != null) { - params.putParam(GetDatafeedRequest.FOR_EXPORT, Boolean.toString(getDatafeedRequest.getForExport())); + if (getDatafeedRequest.getExcludeGenerated() != null) { + params.putParam(GetDatafeedRequest.EXCLUDE_GENERATED, Boolean.toString(getDatafeedRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -653,8 +653,8 @@ static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) { if (getRequest.getAllowNoMatch() != null) { params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch())); } - if (getRequest.getForExport() != null) { - params.putParam(GetDataFrameAnalyticsRequest.FOR_EXPORT, Boolean.toString(getRequest.getForExport())); + if (getRequest.getExcludeGenerated() != null) { + params.putParam(GetDataFrameAnalyticsRequest.EXCLUDE_GENERATED, Boolean.toString(getRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -795,8 +795,8 @@ static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) if (getTrainedModelsRequest.getTags() != null) { params.putParam(GetTrainedModelsRequest.TAGS, Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getTags())); } - if (getTrainedModelsRequest.getForExport() != null) { - params.putParam(GetTrainedModelsRequest.FOR_EXPORT, Boolean.toString(getTrainedModelsRequest.getForExport())); + if (getTrainedModelsRequest.getExcludeGenerated() != null) { + params.putParam(GetTrainedModelsRequest.EXCLUDE_GENERATED, Boolean.toString(getTrainedModelsRequest.getExcludeGenerated())); } Request request = new Request(HttpGet.METHOD_NAME, endpoint); request.addParameters(params.asMap()); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java index bddc8b4c3e1bc..724c0b70d78d7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java @@ -32,12 +32,12 @@ public class GetDataFrameAnalyticsRequest implements Validatable { public static final String ALLOW_NO_MATCH = "allow_no_match"; - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private final List ids; private Boolean allowNoMatch; private PageParams pageParams; - private Boolean forExport; + private Boolean excludeGenerated; /** * Helper method to create a request that will get ALL Data Frame Analytics @@ -65,14 +65,14 @@ public Boolean getAllowNoMatch() { * This is useful when getting the configuration and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed + * @param excludeGenerated Boolean value indicating if certain fields should be removed */ - public void setForExport(boolean forExport) { - this.forExport = forExport; + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } /** @@ -111,12 +111,12 @@ public boolean equals(Object o) { GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o; return Objects.equals(ids, other.ids) && Objects.equals(allowNoMatch, other.allowNoMatch) - && Objects.equals(forExport, other.forExport) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, forExport, pageParams); + return Objects.hash(ids, allowNoMatch, excludeGenerated, pageParams); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java index 38e0b86ffc04f..57b55e8e13b88 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java @@ -41,12 +41,12 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject { public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private static final String ALL_DATAFEEDS = "_all"; private final List datafeedIds; private Boolean allowNoMatch; - private Boolean forExport; + private Boolean excludeGenerated; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -108,19 +108,19 @@ public Boolean getAllowNoMatch() { * This is useful when getting the configuration and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed + * @param excludeGenerated Boolean value indicating if certain fields should be removed */ - public void setForExport(boolean forExport) { - this.forExport = forExport; + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } @Override public int hashCode() { - return Objects.hash(datafeedIds, forExport, allowNoMatch); + return Objects.hash(datafeedIds, excludeGenerated, allowNoMatch); } @Override @@ -136,7 +136,7 @@ public boolean equals(Object other) { GetDatafeedRequest that = (GetDatafeedRequest) other; return Objects.equals(datafeedIds, that.datafeedIds) && Objects.equals(allowNoMatch, that.allowNoMatch) && - Objects.equals(forExport, that.forExport); + Objects.equals(excludeGenerated, that.excludeGenerated); } @Override diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java index 6ad09de62fdab..7c6909678e886 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java @@ -42,12 +42,12 @@ public class GetJobRequest implements Validatable, ToXContentObject { public static final ParseField JOB_IDS = new ParseField("job_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private static final String ALL_JOBS = "_all"; private final List jobIds; private Boolean allowNoMatch; - private Boolean forExport; + private Boolean excludeGenerated; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -108,19 +108,19 @@ public Boolean getAllowNoMatch() { * This is useful when getting the configuration and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed + * @param excludeGenerated Boolean value indicating if certain fields should be removed */ - public void setForExport(boolean forExport) { - this.forExport = forExport; + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } @Override public int hashCode() { - return Objects.hash(jobIds, forExport, allowNoMatch); + return Objects.hash(jobIds, excludeGenerated, allowNoMatch); } @Override @@ -135,7 +135,7 @@ public boolean equals(Object other) { GetJobRequest that = (GetJobRequest) other; return Objects.equals(jobIds, that.jobIds) && - Objects.equals(forExport, that.forExport) && + Objects.equals(excludeGenerated, that.excludeGenerated) && Objects.equals(allowNoMatch, that.allowNoMatch); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java index 48f9563f78185..f9be3032dfb18 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java @@ -39,7 +39,7 @@ public class GetTrainedModelsRequest implements Validatable { private static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance"; private static final String FEATURE_IMPORTANCE_BASELINE = "feature_importance_baseline"; public static final String ALLOW_NO_MATCH = "allow_no_match"; - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; public static final String TAGS = "tags"; public static final String INCLUDE = "include"; @@ -48,7 +48,7 @@ public class GetTrainedModelsRequest implements Validatable { private Boolean allowNoMatch; private Set includes = new HashSet<>(); private Boolean decompressDefinition; - private Boolean forExport; + private Boolean excludeGenerated; private PageParams pageParams; private List tags; @@ -163,8 +163,8 @@ public GetTrainedModelsRequest setTags(String... tags) { return setTags(Arrays.asList(tags)); } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } /** @@ -173,10 +173,10 @@ public Boolean getForExport() { * This is useful when getting the model and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed from the mode on GET + * @param excludeGenerated Boolean value indicating if certain fields should be removed from the mode on GET */ - public GetTrainedModelsRequest setForExport(Boolean forExport) { - this.forExport = forExport; + public GetTrainedModelsRequest setExcludeGenerated(Boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; return this; } @@ -198,12 +198,12 @@ public boolean equals(Object o) { && Objects.equals(allowNoMatch, other.allowNoMatch) && Objects.equals(decompressDefinition, other.decompressDefinition) && Objects.equals(includes, other.includes) - && Objects.equals(forExport, other.forExport) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, forExport); + return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, excludeGenerated); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 4e753365c1e4d..38080c056dbf8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -343,7 +343,7 @@ public void testGetJob() throws Exception { // tag::get-job-request GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1> request.setAllowNoMatch(true); // <2> - request.setForExport(false); // <3> + request.setExcludeGenerated(false); // <3> // end::get-job-request // tag::get-job-execute @@ -840,7 +840,7 @@ public void testGetDatafeed() throws Exception { // tag::get-datafeed-request GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1> request.setAllowNoMatch(true); // <2> - request.setForExport(false); // <3> + request.setExcludeGenerated(false); // <3> // end::get-datafeed-request // tag::get-datafeed-execute @@ -2866,7 +2866,7 @@ public void testGetDataFrameAnalytics() throws Exception { { // tag::get-data-frame-analytics-request GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1> - request.setForExport(false); // <2> + request.setExcludeGenerated(false); // <2> // end::get-data-frame-analytics-request // tag::get-data-frame-analytics-execute @@ -3728,7 +3728,7 @@ public void testGetTrainedModels() throws Exception { .setDecompressDefinition(false) // <6> .setAllowNoMatch(true) // <7> .setTags("regression") // <8> - .setForExport(false); // <9> + .setExcludeGenerated(false); // <9> // end::get-trained-models-request request.setTags((List)null); diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index ed6442c66ec68..545d74374d929 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -19,7 +19,7 @@ Retrieves configuration information for {dfeeds}. `GET _ml/datafeeds/` + -`GET _ml/datafeeds/_all` +`GET _ml/datafeeds/_all` [[ml-get-datafeed-prereqs]] == {api-prereq-title} @@ -36,7 +36,7 @@ comma-separated list of {dfeeds} or a wildcard expression. You can get information for all {dfeeds} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. +IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. [[ml-get-datafeed-path-parms]] == {api-path-parms-title} @@ -57,9 +57,9 @@ all {dfeeds}. (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [[ml-get-datafeed-results]] == {api-response-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 668c95c9891a2..0b5b865b2dc9c 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -34,7 +34,7 @@ using a group name, a comma-separated list of jobs, or a wildcard expression. You can get information for all {anomaly-jobs} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 jobs. +IMPORTANT: This API returns a maximum of 10,000 jobs. [[ml-get-job-path-parms]] == {api-path-parms-title} @@ -50,9 +50,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [[ml-get-job-results]] == {api-response-body-title} @@ -63,7 +63,7 @@ properties, see <>. `create_time`:: (string) The time the job was created. For example, `1491007356077`. This property is informational; you cannot change its value. - + `finished_time`:: (string) If the job closed or failed, this is the time the job finished, otherwise it is `null`. This property is informational; you cannot change its @@ -83,7 +83,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-id] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_match` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-job-example]] diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index a6adac16dbef5..cd3732339b844 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -27,18 +27,18 @@ experimental[] [[ml-get-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following +If the {es} {security-features} are enabled, you must have the following privileges: * cluster: `monitor_ml` - + For more information, see <> and {ml-docs-setup-privileges}. [[ml-get-dfanalytics-desc]] == {api-description-title} -You can get information for multiple {dfanalytics-jobs} in a single API request +You can get information for multiple {dfanalytics-jobs} in a single API request by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. @@ -46,12 +46,12 @@ by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. == {api-path-parms-title} ``:: -(Optional, string) +(Optional, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] + -- -You can get information for all {dfanalytics-jobs} by using _all, by specifying -`*` as the ``, or by omitting the +You can get information for all {dfanalytics-jobs} by using _all, by specifying +`*` as the ``, or by omitting the ``. -- @@ -60,20 +60,20 @@ You can get information for all {dfanalytics-jobs} by using _all, by specifying == {api-query-parms-title} `allow_no_match`:: -(Optional, boolean) +(Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match] `from`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] `size`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [role="child_attributes"] [[ml-get-dfanalytics-results]] @@ -81,7 +81,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] `data_frame_analytics`:: (array) -An array of {dfanalytics-job} resources, which are sorted by the `id` value in +An array of {dfanalytics-job} resources, which are sorted by the `id` value in ascending order. + .Properties of {dfanalytics-job} resources @@ -92,18 +92,18 @@ ascending order. //Begin analyzed_fields `analyzed_fields`::: -(object) Contains `includes` and/or `excludes` patterns that select which fields +(object) Contains `includes` and/or `excludes` patterns that select which fields are included in the analysis. + .Properties of `analyzed_fields` [%collapsible%open] ===== `excludes`::: -(Optional, array) An array of strings that defines the fields that are excluded +(Optional, array) An array of strings that defines the fields that are excluded from the analysis. - + `includes`::: -(Optional, array) An array of strings that defines the fields that are included +(Optional, array) An array of strings that defines the fields that are included in the analysis. ===== //End analyzed_fields @@ -115,11 +115,11 @@ in the analysis. [%collapsible%open] ===== `index`::: -(string) The _destination index_ that stores the results of the +(string) The _destination index_ that stores the results of the {dfanalytics-job}. `results_field`::: -(string) The name of the field that stores the results of the analysis. Defaults +(string) The name of the field that stores the results of the analysis. Defaults to `ml`. ===== //End dest @@ -131,36 +131,36 @@ to `ml`. (string) The `model_memory_limit` that has been set to the {dfanalytics-job}. `source`::: -(object) The configuration of how the analysis data is sourced. It has an +(object) The configuration of how the analysis data is sourced. It has an `index` parameter and optionally a `query` and a `_source`. + .Properties of `source` [%collapsible%open] ===== `index`::: -(array) Index or indices on which to perform the analysis. It can be a single +(array) Index or indices on which to perform the analysis. It can be a single index or index pattern as well as an array of indices or patterns. - + `query`::: -(object) The query that has been specified for the {dfanalytics-job}. The {es} -query domain-specific language (<>). This value corresponds to -the query object in an {es} search POST body. By default, this property has the +(object) The query that has been specified for the {dfanalytics-job}. The {es} +query domain-specific language (<>). This value corresponds to +the query object in an {es} search POST body. By default, this property has the following value: `{"match_all": {}}`. `_source`::: -(object) Contains the specified `includes` and/or `excludes` patterns that -select which fields are present in the destination. Fields that are excluded +(object) Contains the specified `includes` and/or `excludes` patterns that +select which fields are present in the destination. Fields that are excluded cannot be included in the analysis. + .Properties of `_source` [%collapsible%open] ====== `excludes`::: -(array) An array of strings that defines the fields that are excluded from the +(array) An array of strings that defines the fields that are excluded from the destination. - + `includes`::: -(array) An array of strings that defines the fields that are included in the +(array) An array of strings that defines the fields that are included in the destination. ====== //End of _source @@ -180,7 +180,7 @@ destination. [[ml-get-dfanalytics-example]] == {api-examples-title} -The following example gets configuration information for the `loganalytics` +The following example gets configuration information for the `loganalytics` {dfanalytics-job}: [source,console] diff --git a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc index 9fbfd09e33966..cf11b6aa7fb1d 100644 --- a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc @@ -65,9 +65,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-models] Specifies whether the included model definition should be returned as a JSON map (`true`) or in a custom compressed format (`false`). Defaults to `true`. -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] `from`:: (Optional, integer) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 685546d87bb5a..74bdcf04d58d2 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -678,11 +678,11 @@ The number of individual forecasts currently available for the job. A value of `1` or more indicates that forecasts exist. end::forecast-total[] -tag::for-export[] +tag::exclude-generated[] Indicates if certain fields should be removed from the configuration on retrieval. This allows the configuration to be in an acceptable format to be retrieved and then added to another cluster. Default is false. -end::for-export[] +end::exclude-generated[] tag::frequency[] The interval at which scheduled queries are made while the {dfeed} runs in real diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index e711ba2a71e64..b4ee4992148b7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -50,7 +50,7 @@ import java.util.Random; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; /** * Datafeed configuration options. Describes where to proactively pull input @@ -457,10 +457,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(ID.getPreferredName(), id); - // We don't include the job_id in export as we assume the PUT will be referring to a new job as well - builder.field(Job.ID.getPreferredName(), jobId); + builder.field(ID.getPreferredName(), id); + builder.field(Job.ID.getPreferredName(), jobId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { builder.field(CONFIG_TYPE.getPreferredName(), TYPE); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index 0468bc62dd675..ad1e39910ffb8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -34,7 +34,7 @@ import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING; import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @@ -227,8 +227,8 @@ public Integer getMaxNumThreads() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(ID.getPreferredName(), id); + builder.field(ID.getPreferredName(), id); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { if (createTime != null) { builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java index b25a28efad710..c6e70ef960ad9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java @@ -47,7 +47,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper.writeNamedObject; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class TrainedModelConfig implements ToXContentObject, Writeable { @@ -304,9 +304,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + builder.field(MODEL_ID.getPreferredName(), modelId); // If the model is to be exported for future import to another cluster, these fields are irrelevant. - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(MODEL_ID.getPreferredName(), modelId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { builder.field(CREATED_BY.getPreferredName(), createdBy); builder.field(VERSION.getPreferredName(), version.toString()); builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index a3055dde3e9e5..ba36b671cc648 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -42,7 +42,7 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; /** * This class represents a configured and created Job. The creation time is set @@ -515,8 +515,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { final String humanReadableSuffix = "_string"; - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(ID.getPreferredName(), jobId); + builder.field(ID.getPreferredName(), jobId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { builder.field(JOB_TYPE.getPreferredName(), jobType); if (jobVersion != null) { builder.field(JOB_VERSION.getPreferredName(), jobVersion); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java index b755c13ce18f6..f8cfd73eed876 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java @@ -24,7 +24,7 @@ public final class ToXContentParams { * * This helps to GET a configuration, copy it, and then PUT it directly without removing or changing any fields in between */ - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; /** * When serialising POJOs to X Content this indicates whether the calculated (i.e. not stored) fields diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index 2acb421c6e9d2..071b731bd9d59 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.yaml.snakeyaml.util.UriEncoder; +import javax.print.attribute.standard.JobStateReason; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; @@ -247,8 +248,9 @@ public void testExportAndPutJob() throws Exception { String jobId = "test-export-import-job"; createFarequoteJob(jobId); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?for_export=true")); + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("jobs")).get(0); + originalJobBody.remove("job_id"); XContentBuilder xContentBuilder = jsonBuilder().map(originalJobBody); Request request = new Request("PUT", BASE_PATH + "anomaly_detectors/" + jobId + "-import"); @@ -256,8 +258,9 @@ public void testExportAndPutJob() throws Exception { client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse).get("jobs")).get(0); + importedJobBody.remove("job_id"); assertThat(originalJobBody, equalTo(importedJobBody)); } @@ -270,8 +273,9 @@ public void testExportAndPutDatafeed() throws Exception { createDatafeed(datafeedId, jobId); Response dfResponse = client().performRequest( - new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?for_export=true")); + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?exclude_generated=true")); Map originalDfBody = (Map)((List) entityAsMap(dfResponse).get("datafeeds")).get(0); + originalDfBody.remove("datafeed_id"); //Delete this so we can PUT another datafeed for the same job client().performRequest(new Request("DELETE", BASE_PATH + "datafeeds/" + datafeedId)); @@ -284,8 +288,9 @@ public void testExportAndPutDatafeed() throws Exception { client().performRequest(request); Response importedDfResponse = client().performRequest( - new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?exclude_generated=true")); Map importedDfBody = (Map)((List) entityAsMap(importedDfResponse).get("datafeeds")).get(0); + importedDfBody.remove("datafeed_id"); assertThat(originalDfBody, equalTo(importedDfBody)); } @@ -325,8 +330,9 @@ public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Excepti client().performRequest(request); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); @@ -334,10 +340,11 @@ public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Excepti client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) .get("data_frame_analytics")) .get(0); + importedJobBody.remove("id"); assertThat(originalJobBody, equalTo(importedJobBody)); } @@ -378,8 +385,9 @@ public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { client().performRequest(request); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); @@ -387,10 +395,11 @@ public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) .get("data_frame_analytics")) .get(0); + importedJobBody.remove("id"); assertThat(originalJobBody, equalTo(importedJobBody)); } @@ -431,8 +440,9 @@ public void testExportAndPutDataFrameAnalytics_Classification() throws Exception client().performRequest(request); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); @@ -440,10 +450,11 @@ public void testExportAndPutDataFrameAnalytics_Classification() throws Exception client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) .get("data_frame_analytics")) .get(0); + importedJobBody.remove("id"); assertThat(originalJobBody, equalTo(importedJobBody)); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java index aa1c9658e76d2..0486ed61de3e3 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java @@ -204,11 +204,12 @@ public void testExportImportModel() throws IOException { getModel = client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "trained_models/" + modelId + - "?include=definition&decompress_definition=false&for_export=true")); + "?include=definition&decompress_definition=false&exclude_generated=true")); assertThat(getModel.getStatusLine().getStatusCode(), equalTo(200)); Map exportedModel = entityAsMap(getModel); Map modelDefinition = ((List>)exportedModel.get("trained_model_configs")).get(0); + modelDefinition.remove("model_id"); String importedModelId = "regression_model_to_import"; try (XContentBuilder builder = XContentFactory.jsonBuilder()) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java index 35ca39a4e05c0..1f6281e7f53a3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -21,7 +21,7 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetDatafeedsAction extends BaseRestHandler { @@ -57,6 +57,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Collections.singleton(FOR_EXPORT); + return Collections.singleton(EXCLUDE_GENERATED); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java index 140b00f0a0bdd..2e91bac05e312 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java @@ -21,7 +21,7 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetDataFrameAnalyticsAction extends BaseRestHandler { @@ -55,6 +55,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Collections.singleton(FOR_EXPORT); + return Collections.singleton(EXCLUDE_GENERATED); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java index bc563c823fa06..431df5401864e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java @@ -34,7 +34,7 @@ import static java.util.Arrays.asList; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction.Request.ALLOW_NO_MATCH; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetTrainedModelsAction extends BaseRestHandler { @@ -99,7 +99,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, FOR_EXPORT); + return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, EXCLUDE_GENERATED); } private static class RestToXContentListenerWithDefaultValues extends RestToXContentListener { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java index a90df97bddd44..bc4f7ffd2ce0c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -23,7 +23,7 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetJobsAction extends BaseRestHandler { @@ -59,6 +59,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Collections.singleton(FOR_EXPORT); + return Collections.singleton(EXCLUDE_GENERATED); } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json index 5bd3cd20b4137..04d1995aa4c6f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json @@ -44,7 +44,7 @@ "description":"specifies a max number of analytics to get", "default":100 }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json index b7a3760e56a3a..90dc0b72b1c72 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json @@ -39,7 +39,7 @@ "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", "deprecated":true }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json index ae3f0299747ad..fce317332c978 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json @@ -39,7 +39,7 @@ "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", "deprecated":true }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json index 779fe6069cb63..26728c46194a6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json @@ -69,7 +69,7 @@ "type":"list", "description":"A comma-separated list of tags that the model must have." }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index c7439e4774088..cb96f9aa5ca28 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -2188,7 +2188,7 @@ setup: - do: ml.get_data_frame_analytics: id: "simple-outlier-detection" - for_export: true + exclude_generated: true - match: { data_frame_analytics.0.source.index.0: "index-source" } - match: { data_frame_analytics.0.source.query: {"match_all" : {} } } - match: { data_frame_analytics.0.dest.index: "index-dest" } @@ -2201,4 +2201,3 @@ setup: }} - is_false: data_frame_analytics.0.create_time - is_false: data_frame_analytics.0.version - - is_false: data_frame_analytics.0.id diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml index 9b740c0c9a039..263b7af433696 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml @@ -519,10 +519,8 @@ setup: - do: ml.get_datafeeds: datafeed_id: test-for-export - for_export: true + exclude_generated: true - match: { datafeeds.0.indices.0: "index-foo"} - - is_false: datafeeds.0.datafeed_id - - is_false: datafeeds.0.job_id - is_false: datafeeds.0.create_time - is_false: datafeeds.0.query_delay - is_false: datafeeds.0.chunking_config diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml index 15c698f1d94c2..3b6dc6f5413ba 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml @@ -854,20 +854,20 @@ setup: } } --- -"Test for_export flag": +"Test exclude_generated flag": - do: ml.get_trained_models: model_id: "a-regression-model-1" - for_export: true + exclude_generated: true include: "definition" decompress_definition: false - match: { trained_model_configs.0.description: "empty model for tests" } + - match: { trained_model_configs.0.model_id: "a-regression-model-1" } - is_true: trained_model_configs.0.compressed_definition - is_true: trained_model_configs.0.input - is_true: trained_model_configs.0.inference_config - is_true: trained_model_configs.0.tags - - is_false: trained_model_configs.0.model_id - is_false: trained_model_configs.0.created_by - is_false: trained_model_configs.0.version - is_false: trained_model_configs.0.create_time diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml index b305e73069efc..d884ff7aecab2 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml @@ -92,9 +92,8 @@ setup: - do: ml.get_jobs: job_id: jobs-get-1 - for_export: true + exclude_generated: true - match: { jobs.0.description: "Job 1"} - - is_false: job_id - is_false: job_type - is_false: job_version - is_false: create_time From 2cd82f2b46c7e88e368df03102f85ac81a827c65 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 20 Oct 2020 11:38:40 -0400 Subject: [PATCH 46/60] [Transform] add new exclude_generated flag to GET transform (#63093) This adds a new flag `exclude_generated` for GET transform API. This flag is useful for when a transform needs to be cloned within a cluster or exported/imported between clusters. It removes certain fields that are not able to be set via the PUT api (e.g. version, create_time). relates https://github.com/elastic/elasticsearch/issues/63055 --- .../client/TransformRequestConverters.java | 4 ++ .../client/transform/GetTransformRequest.java | 13 +++++- .../TransformDocumentationIT.java | 1 + .../transform/get_transform.asciidoc | 3 ++ .../transform/apis/get-transform.asciidoc | 18 ++++++--- .../xpack/core/transform/TransformField.java | 1 + .../transform/transforms/SourceConfig.java | 7 +++- .../transform/transforms/TransformConfig.java | 40 +++++++++++-------- ...me_transform_deprecated.get_transform.json | 6 +++ .../api/transform.get_transform.json | 6 +++ .../test/transform/transforms_crud.yml | 31 ++++++++++++++ .../integration/TransformPivotRestIT.java | 33 +++++++++++++++ .../rest/action/RestGetTransformAction.java | 8 ++++ .../RestGetTransformActionDeprecated.java | 7 ++++ 14 files changed, 153 insertions(+), 25 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java index 4815353936b28..53daf463941ff 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java @@ -40,6 +40,7 @@ import static org.elasticsearch.client.RequestConverters.createEntity; import static org.elasticsearch.client.transform.DeleteTransformRequest.FORCE; import static org.elasticsearch.client.transform.GetTransformRequest.ALLOW_NO_MATCH; +import static org.elasticsearch.client.transform.GetTransformRequest.EXCLUDE_GENERATED; import static org.elasticsearch.client.transform.PutTransformRequest.DEFER_VALIDATION; import static org.elasticsearch.client.transform.StopTransformRequest.WAIT_FOR_CHECKPOINT; @@ -89,6 +90,9 @@ static Request getTransform(GetTransformRequest getRequest) { if (getRequest.getAllowNoMatch() != null) { request.addParameter(ALLOW_NO_MATCH, getRequest.getAllowNoMatch().toString()); } + if (getRequest.getExcludeGenerated() != null) { + request.addParameter(EXCLUDE_GENERATED, getRequest.getExcludeGenerated().toString()); + } return request; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java index f0238083f6af0..184222244732b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java @@ -30,6 +30,7 @@ public class GetTransformRequest implements Validatable { + public static final String EXCLUDE_GENERATED = "exclude_generated"; public static final String ALLOW_NO_MATCH = "allow_no_match"; /** * Helper method to create a request that will get ALL Transforms @@ -42,6 +43,7 @@ public static GetTransformRequest getAllTransformRequest() { private final List ids; private PageParams pageParams; private Boolean allowNoMatch; + private Boolean excludeGenerated; public GetTransformRequest(String... ids) { this.ids = Arrays.asList(ids); @@ -67,6 +69,14 @@ public void setAllowNoMatch(Boolean allowNoMatch) { this.allowNoMatch = allowNoMatch; } + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; + } + + public Boolean getExcludeGenerated() { + return excludeGenerated; + } + @Override public Optional validate() { if (ids == null || ids.isEmpty()) { @@ -80,7 +90,7 @@ public Optional validate() { @Override public int hashCode() { - return Objects.hash(ids, pageParams, allowNoMatch); + return Objects.hash(ids, pageParams, excludeGenerated, allowNoMatch); } @Override @@ -95,6 +105,7 @@ public boolean equals(Object obj) { GetTransformRequest other = (GetTransformRequest) obj; return Objects.equals(ids, other.ids) && Objects.equals(pageParams, other.pageParams) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(allowNoMatch, other.allowNoMatch); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java index c9c7d9d806b3f..766f3f2a292e2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java @@ -687,6 +687,7 @@ public void testGetDataFrameTransform() throws IOException, InterruptedException // tag::get-transform-request-options request.setPageParams(new PageParams(0, 100)); // <1> request.setAllowNoMatch(true); // <2> + request.setExcludeGenerated(false); // <3> // end::get-transform-request-options // tag::get-transform-execute diff --git a/docs/java-rest/high-level/transform/get_transform.asciidoc b/docs/java-rest/high-level/transform/get_transform.asciidoc index 64aa0f229c427..ca79ebf17e1ba 100644 --- a/docs/java-rest/high-level/transform/get_transform.asciidoc +++ b/docs/java-rest/high-level/transform/get_transform.asciidoc @@ -34,6 +34,9 @@ include-tagged::{doc-tests-file}[{api}-request-options] {transforms} to skip. `size` specifies the maximum number of {transforms} to get. Defaults to `0` and `100` respectively. <2> Whether to ignore if a wildcard expression matches no {transforms}. +<3> Optional boolean value for requesting the {transform} in a format that can +then be put into another cluster. Certain fields that can only be set when +the {transform} is created are removed. include::../execution.asciidoc[] diff --git a/docs/reference/transform/apis/get-transform.asciidoc b/docs/reference/transform/apis/get-transform.asciidoc index 939bc59413d4a..c9a80ab113569 100644 --- a/docs/reference/transform/apis/get-transform.asciidoc +++ b/docs/reference/transform/apis/get-transform.asciidoc @@ -26,12 +26,12 @@ Retrieves configuration information for {transforms}. [[get-transform-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following +If the {es} {security-features} are enabled, you must have the following privileges: -* `monitor_transform` +* `monitor_transform` -The built-in `transform_user` role has this privilege. +The built-in `transform_user` role has this privilege. For more information, see <> and <>. @@ -49,7 +49,7 @@ specifying `*` as the ``, or by omitting the ``. ``:: (Optional, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] - + [[get-transform-query-parms]] == {api-query-parms-title} @@ -65,6 +65,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from-transforms] (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] +`exclude_generated`:: +(Optional, boolean) +Excludes fields that were automatically added when creating the transform. +This allows the configuration to be in an acceptable format to be retrieved +and then added to another cluster. Default is false. + [[get-transform-response]] == {api-response-body-title} @@ -79,13 +85,13 @@ This property is informational; you cannot change its value. `version`:: (string) The version of {es} that existed on the node when the {transform} was created. - + [[get-transform-response-codes]] == {api-response-codes-title} `404` (Missing resources):: If `allow_no_match` is `false`, this code indicates that there are no - resources that match the request or only partial matches for the request. + resources that match the request or only partial matches for the request. [[get-transform-example]] == {api-examples-title} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index 46855b74ab210..cb52699c5b26d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -84,6 +84,7 @@ public final class TransformField { * API's) */ public static final String FOR_INTERNAL_STORAGE = "for_internal_storage"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; // internal document id public static String DOCUMENT_ID_FIELD = "_id"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java index b3478e89520c0..3e30482daef0c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.license.RemoteClusterLicenseChecker; +import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; import java.io.IOException; @@ -113,7 +114,11 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.array(INDEX.getPreferredName(), index); - builder.field(QUERY.getPreferredName(), queryConfig); + if (params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false) == false) { + builder.field(QUERY.getPreferredName(), queryConfig); + } else if(queryConfig.equals(QueryConfig.matchAll()) == false) { + builder.field(QUERY.getPreferredName(), queryConfig); + } builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 2f1995fd0afe0..d5b3e7ddb74a5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -342,9 +342,31 @@ public void writeTo(final StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false); + final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false); + assert (forInternalStorage && excludeGenerated) == false: + "unsupported behavior, exclude_generated is true and for_internal_storage is true"; builder.startObject(); builder.field(TransformField.ID.getPreferredName(), id); - builder.field(TransformField.SOURCE.getPreferredName(), source); + if (excludeGenerated == false) { + if (headers.isEmpty() == false && forInternalStorage) { + builder.field(HEADERS.getPreferredName(), headers); + } + if (transformVersion != null) { + builder.field(TransformField.VERSION.getPreferredName(), transformVersion); + } + if (createTime != null) { + builder.timeField( + TransformField.CREATE_TIME.getPreferredName(), + TransformField.CREATE_TIME.getPreferredName() + "_string", + createTime.toEpochMilli() + ); + } + if (forInternalStorage) { + builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME); + } + } + builder.field(TransformField.SOURCE.getPreferredName(), source, params); builder.field(TransformField.DESTINATION.getPreferredName(), dest); if (frequency != null) { builder.field(TransformField.FREQUENCY.getPreferredName(), frequency.getStringRep()); @@ -357,26 +379,10 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (pivotConfig != null) { builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig); } - if (params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) { - builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME); - } - if (headers.isEmpty() == false && params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } if (description != null) { builder.field(TransformField.DESCRIPTION.getPreferredName(), description); } builder.field(TransformField.SETTINGS.getPreferredName(), settings); - if (transformVersion != null) { - builder.field(TransformField.VERSION.getPreferredName(), transformVersion); - } - if (createTime != null) { - builder.timeField( - TransformField.CREATE_TIME.getPreferredName(), - TransformField.CREATE_TIME.getPreferredName() + "_string", - createTime.toEpochMilli() - ); - } builder.endObject(); return builder; } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json index 17f333861c5dc..22726a9596332 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json @@ -50,6 +50,12 @@ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no transforms. (This includes `_all` string or when no transforms have been specified)" + }, + "exclude_generated": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits generated fields. Allows transform configurations to be easily copied between clusters and within the same cluster" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json index 1ab97aa6a18db..69e87d9755477 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json @@ -42,6 +42,12 @@ "type":"boolean", "required":false, "description":"Whether to ignore if a wildcard expression matches no transforms. (This includes `_all` string or when no transforms have been specified)" + }, + "exclude_generated": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on transform PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml index 20b49c361b18f..5fd1a726602c5 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml @@ -670,3 +670,34 @@ setup: }, "description": "yaml test transform on airline-data" } + +--- +"Test transform for export": + - do: + transform.put_transform: + transform_id: "airline-transform" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-by-airline" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + }, + "description": "yaml test transform on airline-data" + } + - match: { acknowledged: true } + + - do: + transform.get_transform: + transform_id: "airline-transform" + exclude_generated: true + + - match: {transforms.0.source.index: ["airline-data"]} + - match: {transforms.0.dest.index: "airline-data-by-airline"} + - match: {transforms.0.pivot.group_by.airline.terms.field: "airline"} + - match: {transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime"} + - match: {transforms.0.description: "yaml test transform on airline-data"} + - match: {transforms.0.id: "airline-transform"} + - is_false: transforms.0.create_time + - is_false: transforms.0.version diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index e100fc6625743..b4dd50b0983fe 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -9,6 +9,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -1686,6 +1687,38 @@ public void testPivotWithFilter() throws Exception { assertEquals(3, actual.longValue()); } + @SuppressWarnings("unchecked") + public void testExportAndImport() throws Exception { + String transformId = "export-transform"; + String transformIndex = "export_reviews"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex); + + createPivotReviewsTransform(transformId, transformIndex, null, null, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS); + + Response response = adminClient().performRequest(new Request("GET", + getTransformEndpoint() + transformId + "?exclude_generated=true")); + Map storedConfig = ((List>) XContentMapValues.extractValue( + "transforms", + entityAsMap(response))) + .get(0); + storedConfig.remove("id"); + try (XContentBuilder builder = jsonBuilder()) { + builder.map(storedConfig); + Request putTransform = new Request("PUT", getTransformEndpoint() + transformId + "-import"); + putTransform.setJsonEntity(Strings.toString(builder)); + adminClient().performRequest(putTransform); + } + + response = adminClient().performRequest(new Request("GET", + getTransformEndpoint() + transformId + "-import" + "?exclude_generated=true")); + Map importConfig = ((List>) XContentMapValues.extractValue( + "transforms", + entityAsMap(response))) + .get(0); + importConfig.remove("id"); + assertThat(storedConfig, equalTo(importConfig)); + } + private void createDateNanoIndex(String indexName, int numDocs) throws IOException { // create mapping try (XContentBuilder builder = jsonBuilder()) { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformAction.java index f57e830debe7b..a6b599aafde36 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformAction.java @@ -14,10 +14,13 @@ import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.action.GetTransformAction; +import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.transform.TransformField.ALLOW_NO_MATCH; +import static org.elasticsearch.xpack.core.transform.TransformField.EXCLUDE_GENERATED; public class RestGetTransformAction extends BaseRestHandler { @@ -47,4 +50,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient public String getName() { return "transform_get_transform_action"; } + + @Override + protected Set responseParams() { + return Collections.singleton(EXCLUDE_GENERATED); + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/compat/RestGetTransformActionDeprecated.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/compat/RestGetTransformActionDeprecated.java index 211e197a4b736..dcf0a49416731 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/compat/RestGetTransformActionDeprecated.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/compat/RestGetTransformActionDeprecated.java @@ -18,9 +18,11 @@ import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.transform.TransformField.ALLOW_NO_MATCH; +import static org.elasticsearch.xpack.core.transform.TransformField.EXCLUDE_GENERATED; public class RestGetTransformActionDeprecated extends BaseRestHandler { @Override @@ -56,4 +58,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient public String getName() { return "data_frame_get_transforms_action"; } + + @Override + protected Set responseParams() { + return Collections.singleton(EXCLUDE_GENERATED); + } } From 8bf7976f50b4acf4ea64a994cc1bca14b8d482ab Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 20 Oct 2020 13:06:39 -0500 Subject: [PATCH 47/60] Gradle - compatible REST test plugin - adopt bwc artifact (#63629) Use newly exposed bwc:minor / checkout artifact as opposed to coding directly against the output of a bwc:minor checkout task. The copy test/api tasks have been updated to lazily compute the subset of the files from the configuration. This is to facilitate compatible testing which needs to filter singular bwc:minor checkout in core/x-pack/project set of files. By default (unless explicitly requested) the copy test/api tasks will continue to use the full fileTree of the configuration. related: #63173 --- .../gradle/test/rest/CopyRestApiTask.java | 30 ++++-- .../gradle/test/rest/CopyRestTestsTask.java | 30 ++++-- .../test/rest/YamlRestCompatTestPlugin.java | 101 +++++++----------- 3 files changed, 75 insertions(+), 86 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java index 08d46f97ddaf1..4e78154dec511 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java @@ -25,9 +25,10 @@ import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.ArchiveOperations; -import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.FileCollection; import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.FileTree; +import org.gradle.api.file.ProjectLayout; import org.gradle.api.plugins.JavaPluginConvention; import org.gradle.api.provider.ListProperty; import org.gradle.api.tasks.Input; @@ -47,6 +48,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.gradle.util.GradleUtils.getProjectPathFromTask; @@ -66,13 +68,19 @@ public class CopyRestApiTask extends DefaultTask { Configuration coreConfig; Configuration xpackConfig; Configuration additionalConfig; + Function coreConfigToFileTree = FileCollection::getAsFileTree; + Function xpackConfigToFileTree = FileCollection::getAsFileTree; + Function additionalConfigToFileTree = FileCollection::getAsFileTree; private final PatternFilterable corePatternSet; private final PatternFilterable xpackPatternSet; + private final ProjectLayout projectLayout; - public CopyRestApiTask() { + @Inject + public CopyRestApiTask(ProjectLayout projectLayout) { corePatternSet = getPatternSetFactory().create(); xpackPatternSet = getPatternSetFactory().create(); + this.projectLayout = projectLayout; } @Inject @@ -117,21 +125,21 @@ public FileTree getInputDir() { FileTree xpackFileTree = null; if (includeXpack.get().isEmpty() == false) { xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); - xpackFileTree = xpackConfig.getAsFileTree().matching(xpackPatternSet); + xpackFileTree = xpackConfigToFileTree.apply(xpackConfig).matching(xpackPatternSet); } boolean projectHasYamlRestTests = skipHasRestTestCheck || projectHasYamlRestTests(); if (includeCore.get().isEmpty() == false || projectHasYamlRestTests) { if (BuildParams.isInternal()) { corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); - coreFileTree = coreConfig.getAsFileTree().matching(corePatternSet); // directory on disk + coreFileTree = coreConfigToFileTree.apply(coreConfig).matching(corePatternSet); // directory on disk } else { coreFileTree = coreConfig.getAsFileTree(); // jar file } } - ConfigurableFileCollection fileCollection = additionalConfig == null - ? getProject().files(coreFileTree, xpackFileTree) - : getProject().files(coreFileTree, xpackFileTree, additionalConfig.getAsFileTree()); + FileCollection fileCollection = additionalConfig == null + ? projectLayout.files(coreFileTree, xpackFileTree) + : projectLayout.files(coreFileTree, xpackFileTree, additionalConfigToFileTree.apply(additionalConfig)); // if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input return projectHasYamlRestTests || includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false @@ -156,7 +164,7 @@ void copy() { if (BuildParams.isInternal()) { getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", projectPath); getFileSystemOperations().copy(c -> { - c.from(coreConfig.getAsFileTree()); + c.from(coreConfigToFileTree.apply(coreConfig)); c.into(getOutputDir()); c.include(corePatternSet.getIncludes()); }); @@ -167,7 +175,7 @@ void copy() { VersionProperties.getElasticsearch() ); getFileSystemOperations().copy(c -> { - c.from(getArchiveOperations().zipTree(coreConfig.getSingleFile())); + c.from(getArchiveOperations().zipTree(coreConfig.getSingleFile())); // jar file // this ends up as the same dir as outputDir c.into(Objects.requireNonNull(getSourceSet().orElseThrow().getOutput().getResourcesDir())); if (includeCore.get().isEmpty()) { @@ -183,7 +191,7 @@ void copy() { if (includeXpack.get().isEmpty() == false) { getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", projectPath); getFileSystemOperations().copy(c -> { - c.from(xpackConfig.getSingleFile()); + c.from(xpackConfigToFileTree.apply(xpackConfig)); c.into(getOutputDir()); c.include(xpackPatternSet.getIncludes()); }); @@ -192,7 +200,7 @@ void copy() { // copy any additional config if (additionalConfig != null) { getFileSystemOperations().copy(c -> { - c.from(additionalConfig.getAsFileTree()); + c.from(additionalConfigToFileTree.apply(additionalConfig)); c.into(getOutputDir()); }); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java index fb7cf7c6d2bc7..0552f9ff25ebd 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java @@ -25,9 +25,10 @@ import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.ArchiveOperations; -import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.FileCollection; import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.FileTree; +import org.gradle.api.file.ProjectLayout; import org.gradle.api.plugins.JavaPluginConvention; import org.gradle.api.provider.ListProperty; import org.gradle.api.tasks.Input; @@ -44,6 +45,7 @@ import java.io.File; import java.util.Objects; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.gradle.util.GradleUtils.getProjectPathFromTask; @@ -63,13 +65,19 @@ public class CopyRestTestsTask extends DefaultTask { Configuration coreConfig; Configuration xpackConfig; Configuration additionalConfig; + Function coreConfigToFileTree = FileCollection::getAsFileTree; + Function xpackConfigToFileTree = FileCollection::getAsFileTree; + Function additionalConfigToFileTree = FileCollection::getAsFileTree; private final PatternFilterable corePatternSet; private final PatternFilterable xpackPatternSet; + private final ProjectLayout projectLayout; - public CopyRestTestsTask() { + @Inject + public CopyRestTestsTask(ProjectLayout projectLayout) { corePatternSet = getPatternSetFactory().create(); xpackPatternSet = getPatternSetFactory().create(); + this.projectLayout = projectLayout; } @Inject @@ -109,19 +117,19 @@ public FileTree getInputDir() { FileTree xpackFileTree = null; if (includeXpack.get().isEmpty() == false) { xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); - xpackFileTree = xpackConfig.getAsFileTree().matching(xpackPatternSet); + xpackFileTree = xpackConfigToFileTree.apply(xpackConfig).matching(xpackPatternSet); } if (includeCore.get().isEmpty() == false) { if (BuildParams.isInternal()) { corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); - coreFileTree = coreConfig.getAsFileTree().matching(corePatternSet); // directory on disk + coreFileTree = coreConfigToFileTree.apply(coreConfig).matching(corePatternSet); // directory on disk } else { coreFileTree = coreConfig.getAsFileTree(); // jar file } } - ConfigurableFileCollection fileCollection = additionalConfig == null - ? getProject().files(coreFileTree, xpackFileTree) - : getProject().files(coreFileTree, xpackFileTree, additionalConfig.getAsFileTree()); + FileCollection fileCollection = additionalConfig == null + ? projectLayout.files(coreFileTree, xpackFileTree) + : projectLayout.files(coreFileTree, xpackFileTree, additionalConfigToFileTree.apply(additionalConfig)); // copy tests only if explicitly requested return includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false || additionalConfig != null @@ -147,7 +155,7 @@ void copy() { if (BuildParams.isInternal()) { getLogger().debug("Rest tests for project [{}] will be copied to the test resources.", projectPath); getFileSystemOperations().copy(c -> { - c.from(coreConfig.getAsFileTree()); + c.from(coreConfigToFileTree.apply(coreConfig)); c.into(getOutputDir()); c.include(corePatternSet.getIncludes()); }); @@ -158,7 +166,7 @@ void copy() { VersionProperties.getElasticsearch() ); getFileSystemOperations().copy(c -> { - c.from(getArchiveOperations().zipTree(coreConfig.getSingleFile())); + c.from(getArchiveOperations().zipTree(coreConfig.getSingleFile())); // jar file // this ends up as the same dir as outputDir c.into(Objects.requireNonNull(getSourceSet().orElseThrow().getOutput().getResourcesDir())); c.include( @@ -171,7 +179,7 @@ void copy() { if (includeXpack.get().isEmpty() == false) { getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", projectPath); getFileSystemOperations().copy(c -> { - c.from(xpackConfig.getAsFileTree()); + c.from(xpackConfigToFileTree.apply(xpackConfig)); c.into(getOutputDir()); c.include(xpackPatternSet.getIncludes()); }); @@ -179,7 +187,7 @@ void copy() { // copy any additional config if (additionalConfig != null) { getFileSystemOperations().copy(c -> { - c.from(additionalConfig.getAsFileTree()); + c.from(additionalConfigToFileTree.apply(additionalConfig)); c.into(getOutputDir()); }); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestCompatTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestCompatTestPlugin.java index be22f57ae3b05..2bef82660182b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestCompatTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestCompatTestPlugin.java @@ -20,7 +20,6 @@ package org.elasticsearch.gradle.test.rest; import org.elasticsearch.gradle.ElasticsearchJavaPlugin; -import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.test.RestIntegTestTask; import org.elasticsearch.gradle.test.RestTestBasePlugin; import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; @@ -30,6 +29,7 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.provider.Provider; import org.gradle.api.tasks.SourceSet; @@ -37,6 +37,7 @@ import java.io.File; import java.nio.file.Path; +import java.util.Map; import static org.elasticsearch.gradle.test.rest.RestTestUtil.createTestCluster; import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies; @@ -51,6 +52,9 @@ public class YamlRestCompatTestPlugin implements Plugin { public static final String SOURCE_SET_NAME = "yamlRestCompatTest"; private static final Path RELATIVE_API_PATH = Path.of("rest-api-spec/api"); private static final Path RELATIVE_TEST_PATH = Path.of("rest-api-spec/test"); + private static final Path RELATIVE_REST_API_RESOURCES = Path.of("rest-api-spec/src/main/resources"); + private static final Path RELATIVE_REST_XPACK_RESOURCES = Path.of("x-pack/plugin/src/test/resources"); + private static final Path RELATIVE_REST_PROJECT_RESOURCES = Path.of("src/yamlRestTest/resources"); @Override public void apply(Project project) { @@ -73,84 +77,53 @@ public void apply(Project project) { ElasticsearchCluster testCluster = createTestCluster(project, yamlCompatTestSourceSet); testCluster.setTestDistribution(TestDistribution.DEFAULT); - // TODO: once https://github.com/elastic/elasticsearch/pull/62473 lands refactor this to reference the checkoutDir as an artifact - int priorMajorVersion = VersionProperties.getElasticsearchVersion().getMajor() - 1; - final Path checkoutDir = project.findProject(":distribution:bwc:minor") - .getBuildDir() - .toPath() - .resolve("bwc") - .resolve("checkout-" + priorMajorVersion + ".x"); - // copy compatible rest specs - Configuration compatSpec = project.getConfigurations().create("compatSpec"); - Configuration xpackCompatSpec = project.getConfigurations().create("xpackCompatSpec"); - Configuration additionalCompatSpec = project.getConfigurations().create("additionalCompatSpec"); + Configuration bwcMinorConfig = project.getConfigurations().create("bwcMinor"); + Dependency bwcMinor = project.getDependencies().project(Map.of("path", ":distribution:bwc:minor", "configuration", "checkout")); + project.getDependencies().add(bwcMinorConfig.getName(), bwcMinor); + Provider copyCompatYamlSpecTask = project.getTasks() .register("copyRestApiCompatSpecsTask", CopyRestApiTask.class, task -> { + task.dependsOn(bwcMinorConfig); + task.coreConfig = bwcMinorConfig; + task.xpackConfig = bwcMinorConfig; + task.additionalConfig = bwcMinorConfig; task.includeCore.set(extension.restApi.getIncludeCore()); task.includeXpack.set(extension.restApi.getIncludeXpack()); task.sourceSetName = SOURCE_SET_NAME; task.skipHasRestTestCheck = true; - task.coreConfig = compatSpec; - project.getDependencies() - .add( - task.coreConfig.getName(), - project.files(checkoutDir.resolve("rest-api-spec/src/main/resources").resolve(RELATIVE_API_PATH)) - ); - task.xpackConfig = xpackCompatSpec; - project.getDependencies() - .add( - task.xpackConfig.getName(), - project.files(checkoutDir.resolve("x-pack/plugin/src/test/resources").resolve(RELATIVE_API_PATH)) - ); - task.additionalConfig = additionalCompatSpec; - // per project can define custom specifications - project.getDependencies() - .add( - task.additionalConfig.getName(), - project.files( - getCompatProjectPath(project, checkoutDir).resolve("src/yamlRestTest/resources").resolve(RELATIVE_API_PATH) - ) - ); - task.dependsOn(task.coreConfig); - task.dependsOn(task.xpackConfig); - task.dependsOn(task.additionalConfig); - task.dependsOn(":distribution:bwc:minor:checkoutBwcBranch"); + task.coreConfigToFileTree = config -> project.fileTree( + config.getSingleFile().toPath().resolve(RELATIVE_REST_API_RESOURCES).resolve(RELATIVE_API_PATH) + ); + task.xpackConfigToFileTree = config -> project.fileTree( + config.getSingleFile().toPath().resolve(RELATIVE_REST_XPACK_RESOURCES).resolve(RELATIVE_API_PATH) + ); + task.additionalConfigToFileTree = config -> project.fileTree( + getCompatProjectPath(project, config.getSingleFile().toPath()).resolve(RELATIVE_REST_PROJECT_RESOURCES) + .resolve(RELATIVE_API_PATH) + ); }); // copy compatible rest tests - Configuration compatTest = project.getConfigurations().create("compatTest"); - Configuration xpackCompatTest = project.getConfigurations().create("xpackCompatTest"); - Configuration additionalCompatTest = project.getConfigurations().create("additionalCompatTest"); Provider copyCompatYamlTestTask = project.getTasks() .register("copyRestApiCompatTestTask", CopyRestTestsTask.class, task -> { + task.dependsOn(bwcMinorConfig); + task.coreConfig = bwcMinorConfig; + task.xpackConfig = bwcMinorConfig; + task.additionalConfig = bwcMinorConfig; task.includeCore.set(extension.restTests.getIncludeCore()); task.includeXpack.set(extension.restTests.getIncludeXpack()); task.sourceSetName = SOURCE_SET_NAME; - task.coreConfig = compatTest; - project.getDependencies() - .add( - task.coreConfig.getName(), - project.files(checkoutDir.resolve("rest-api-spec/src/main/resources").resolve(RELATIVE_TEST_PATH)) - ); - task.xpackConfig = xpackCompatTest; - project.getDependencies() - .add( - task.xpackConfig.getName(), - project.files(checkoutDir.resolve("x-pack/plugin/src/test/resources").resolve(RELATIVE_TEST_PATH)) - ); - task.additionalConfig = additionalCompatTest; - project.getDependencies() - .add( - task.additionalConfig.getName(), - project.files( - getCompatProjectPath(project, checkoutDir).resolve("src/yamlRestTest/resources").resolve(RELATIVE_TEST_PATH) - ) - ); - task.dependsOn(task.coreConfig); - task.dependsOn(task.xpackConfig); - task.dependsOn(task.additionalConfig); - task.dependsOn(":distribution:bwc:minor:checkoutBwcBranch"); + task.coreConfigToFileTree = config -> project.fileTree( + config.getSingleFile().toPath().resolve(RELATIVE_REST_API_RESOURCES).resolve(RELATIVE_TEST_PATH) + ); + task.xpackConfigToFileTree = config -> project.fileTree( + config.getSingleFile().toPath().resolve(RELATIVE_REST_XPACK_RESOURCES).resolve(RELATIVE_TEST_PATH) + ); + task.additionalConfigToFileTree = config -> project.fileTree( + getCompatProjectPath(project, config.getSingleFile().toPath()).resolve(RELATIVE_REST_PROJECT_RESOURCES) + .resolve(RELATIVE_TEST_PATH) + ); task.dependsOn(copyCompatYamlSpecTask); }); From b4cc55c82232dbebd7ae90383d02c2c6fe5377bf Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 20 Oct 2020 20:11:29 +0200 Subject: [PATCH 48/60] Remove documentMapperParser method from MapperService (#63938) MapperService allows to retrieve its internal DocumentMapperParser instance. Such method is only used in tests, and always to parse mappings which is already exposed by MapperService through a specific parse method. This commit removes the getter for DocumentMapperParser from MapperService in favour of calling MapperService#parse --- .../RankFeatureMetaFieldMapperTests.java | 11 ++- .../PercolatorFieldMapperTests.java | 5 +- .../index/mapper/DocumentMapperParser.java | 2 +- .../index/mapper/MapperService.java | 5 -- .../fielddata/BinaryDVFieldDataTests.java | 2 +- .../index/mapper/CamelCaseFieldNameTests.java | 2 +- .../mapper/FieldNamesFieldMapperTests.java | 9 +-- .../index/mapper/IdFieldMapperTests.java | 3 +- .../index/mapper/IndexFieldMapperTests.java | 7 +- .../index/mapper/IpRangeFieldMapperTests.java | 9 +-- .../index/mapper/MultiFieldTests.java | 38 ++++------ .../index/mapper/NestedObjectMapperTests.java | 30 +++----- .../mapper/NullValueObjectMappingTests.java | 3 +- .../index/mapper/ObjectMapperTests.java | 17 +++-- .../index/mapper/PathMapperTests.java | 3 +- .../index/mapper/RootObjectMapperTests.java | 8 +-- .../index/mapper/RoutingFieldMapperTests.java | 6 +- .../index/mapper/SourceFieldMapperTests.java | 68 +++++++++--------- .../index/similarity/SimilarityTests.java | 2 +- .../CategoryContextMappingTests.java | 27 +++----- .../elasticsearch/index/MapperTestUtils.java | 9 ++- .../index/engine/EngineTestCase.java | 2 +- .../DataStreamTimestampFieldMapperTests.java | 7 +- .../mapper/CartesianFieldMapperTests.java | 18 ++--- ...GeoShapeWithDocValuesFieldMapperTests.java | 69 +++++++------------ .../index/mapper/PointFieldMapperTests.java | 36 ++++------ .../index/mapper/ShapeFieldMapperTests.java | 52 +++++--------- 27 files changed, 172 insertions(+), 278 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java index 36e55012da849..304d139f0c579 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; @@ -33,13 +32,11 @@ public class RankFeatureMetaFieldMapperTests extends ESSingleNodeTestCase { - IndexService indexService; - DocumentMapperParser parser; + MapperService mapperService; @Before public void setup() { - indexService = createIndex("test"); - parser = indexService.mapperService().documentMapperParser(); + mapperService = createIndex("test").mapperService(); } @Override @@ -52,7 +49,7 @@ public void testBasics() throws Exception { .startObject("properties").startObject("field").field("type", "rank_feature").endObject().endObject() .endObject().endObject()); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + DocumentMapper mapper = mapperService.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); assertNotNull(mapper.metadataMapper(RankFeatureMetaFieldMapper.class)); @@ -64,7 +61,7 @@ public void testBasics() throws Exception { */ public void testDocumentParsingFailsOnMetaField() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc").endObject().endObject()); - DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + DocumentMapper mapper = mapperService.parse("_doc", new CompressedXContent(mapping)); String rfMetaField = RankFeatureMetaFieldMapper.CONTENT_TYPE; BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field(rfMetaField, 0).endObject()); MapperParsingException e = expectThrows(MapperParsingException.class, () -> diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 27ae625d50b96..19f12d6130d00 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -59,7 +59,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParseContext; @@ -702,10 +701,8 @@ public void testEmptyName() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("").field("type", "percolator").endObject().endObject() .endObject().endObject()); - DocumentMapperParser parser = mapperService.documentMapperParser(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> parser.parse("type1", new CompressedXContent(mapping)) + () -> mapperService.parse("type1", new CompressedXContent(mapping)) ); assertThat(e.getMessage(), containsString("name cannot be empty string")); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index ae84088a1fee8..6fd9328a8686f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -61,7 +61,7 @@ public class DocumentMapperParser { } @SuppressWarnings("unchecked") - public DocumentMapper parse(@Nullable String type, CompressedXContent source) throws MapperParsingException { + DocumentMapper parse(@Nullable String type, CompressedXContent source) throws MapperParsingException { Map mapping = null; if (source != null) { mapping = XContentHelper.convertToMap(source.compressedReference(), true, XContentType.JSON).v2(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index e60222cdd87f4..c3e80c01a70f8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -147,11 +147,6 @@ public NamedAnalyzer getNamedAnalyzer(String analyzerName) { return this.indexAnalyzers.get(analyzerName); } - //TODO This is only used in tests, we may want to look into replacing those usages? - public DocumentMapperParser documentMapperParser() { - return this.documentMapperParser; - } - public Mapper.TypeParser.ParserContext parserContext() { return parserContextSupplier.get(); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java index 277a5f9e700c8..0a11b444f573e 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java @@ -50,7 +50,7 @@ public void testDocValue() throws Exception { .endObject() .endObject().endObject()); - final DocumentMapper mapper = mapperService.documentMapperParser().parse("test", new CompressedXContent(mapping)); + final DocumentMapper mapper = mapperService.parse("test", new CompressedXContent(mapping)); List bytesList1 = new ArrayList<>(2); bytesList1.add(randomBytes()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CamelCaseFieldNameTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CamelCaseFieldNameTests.java index 933bdd3a2144b..9f2e3309943fa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CamelCaseFieldNameTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CamelCaseFieldNameTests.java @@ -35,7 +35,7 @@ public void testCamelCaseFieldNameStaysAsIs() throws Exception { assertNotNull(documentMapper.mappers().getMapper("thisIsCamelCase")); assertNull(documentMapper.mappers().getMapper("this_is_camel_case")); - documentMapper = mapperService.documentMapperParser().parse("_doc", documentMapper.mappingSource()); + documentMapper = mapperService.parse("_doc", documentMapper.mappingSource()); assertNotNull(documentMapper.mappers().getMapper("thisIsCamelCase")); assertNull(documentMapper.mappers().getMapper("this_is_camel_case")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java index 839a6466386c6..d56f8f9d0cc53 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java @@ -72,8 +72,7 @@ public void testFieldType() throws Exception { .startObject("_field_names").endObject() .endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().hasDocValues()); @@ -85,8 +84,7 @@ public void testFieldType() throws Exception { public void testInjectIntoDocDuringParsing() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -108,7 +106,7 @@ public void testUsingEnabledSettingThrows() throws Exception { .startObject("field").field("type", "keyword").field("doc_values", false).endObject() .endObject().endObject().endObject()); MapperParsingException ex = expectThrows(MapperParsingException.class, - () -> createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping))); + () -> createIndex("test").mapperService().parse("type", new CompressedXContent(mapping))); assertEquals("The `enabled` setting for the `_field_names` field has been deprecated and removed. " + "Please remove it from your mappings and templates.", ex.getMessage()); @@ -127,7 +125,6 @@ public void testUsingEnabledBefore8() throws Exception { .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0)) .build()).mapperService() - .documentMapperParser() .parse("type", new CompressedXContent(mapping)); FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().isEnabled()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java index 2328d0eb76219..85a6af847950d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java @@ -51,8 +51,7 @@ protected Collection> getPlugins() { public void testIncludeInObjectNotAllowed() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); try { docMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java index cf0f2609dde40..9d87ba2052c10 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java @@ -44,8 +44,7 @@ protected Collection> getPlugins() { public void testDefaultDisabledIndexMapper() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -62,9 +61,9 @@ public void testIndexNotConfigurable() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_index").endObject() .endObject().endObject()); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); MapperParsingException e = expectThrows(MapperParsingException.class, - () -> parser.parse("type", new CompressedXContent(mapping))); + () -> mapperService.parse("type", new CompressedXContent(mapping))); assertEquals("_index is not configurable", e.getMessage()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java index 2f861ca6fc79c..7fd3bc9af8a32 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexService; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; @@ -38,13 +37,11 @@ public class IpRangeFieldMapperTests extends ESSingleNodeTestCase { - private IndexService indexService; - private DocumentMapperParser parser; + private MapperService mapperService; @Before public void setup() { - indexService = createIndex("test"); - parser = indexService.mapperService().documentMapperParser(); + mapperService = createIndex("test").mapperService(); } public void testStoreCidr() throws Exception { @@ -52,7 +49,7 @@ public void testStoreCidr() throws Exception { .startObject("properties").startObject("field").field("type", "ip_range") .field("store", true); mapping = mapping.endObject().endObject().endObject().endObject(); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(Strings.toString(mapping))); + DocumentMapper mapper = mapperService.parse("type", new CompressedXContent(Strings.toString(mapping))); assertEquals(Strings.toString(mapping), mapper.mappingSource().toString()); final Map cases = new HashMap<>(); cases.put("192.168.0.0/15", "192.169.255.255"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java index d56046f36e777..42284b0f61750 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java @@ -47,6 +47,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; @@ -139,8 +140,7 @@ public void testBuildThenParse() throws Exception { String builtMapping = builderDocMapper.mappingSource().string(); // reparse it - DocumentMapper docMapper = indexService.mapperService().documentMapperParser() - .parse("person", new CompressedXContent(builtMapping)); + DocumentMapper docMapper = indexService.mapperService().parse("person", new CompressedXContent(builtMapping)); BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/multifield/test-data.json")); @@ -182,8 +182,7 @@ public void testMultiFieldsInConsistentOrder() throws Exception { } builder = builder.endObject().endObject().endObject().endObject().endObject(); String mapping = Strings.toString(builder); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); Arrays.sort(multiFieldNames); Map sourceAsMap = @@ -205,13 +204,10 @@ public void testObjectFieldNotAllowed() throws Exception { .field("type", "text").startObject("fields").startObject("multi").field("type", "object") .endObject().endObject() .endObject().endObject().endObject().endObject()); - final DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - try { - parser.parse("type", new CompressedXContent(mapping)); - fail("expected mapping parse failure"); - } catch (MapperParsingException e) { - assertTrue(e.getMessage().contains("cannot be used in multi field")); - } + MapperService mapperService = createIndex("test").mapperService(); + MapperParsingException exception = expectThrows(MapperParsingException.class, + () -> mapperService.parse("type", new CompressedXContent(mapping))); + assertThat(exception.getMessage(), containsString("cannot be used in multi field")); } public void testNestedFieldNotAllowed() throws Exception { @@ -219,13 +215,10 @@ public void testNestedFieldNotAllowed() throws Exception { .field("type", "text").startObject("fields").startObject("multi").field("type", "nested") .endObject().endObject() .endObject().endObject().endObject().endObject()); - final DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - try { - parser.parse("type", new CompressedXContent(mapping)); - fail("expected mapping parse failure"); - } catch (MapperParsingException e) { - assertTrue(e.getMessage().contains("cannot be used in multi field")); - } + MapperService mapperService = createIndex("test").mapperService(); + MapperParsingException exception = expectThrows(MapperParsingException.class, + () -> mapperService.parse("type", new CompressedXContent(mapping))); + assertThat(exception.getMessage(), containsString("cannot be used in multi field")); } public void testMultiFieldWithDot() throws IOException { @@ -247,11 +240,8 @@ public void testMultiFieldWithDot() throws IOException { .endObject(); MapperService mapperService = createIndex("test").mapperService(); - try { - mapperService.documentMapperParser().parse("my_type", new CompressedXContent(Strings.toString(mapping))); - fail("this should throw an exception because one field contains a dot"); - } catch (MapperParsingException e) { - assertThat(e.getMessage(), equalTo("Field name [raw.foo] which is a multi field of [city] cannot contain '.'")); - } + MapperParsingException exception = expectThrows(MapperParsingException.class, + () -> mapperService.parse("my_type", new CompressedXContent(Strings.toString(mapping)))); + assertThat(exception.getMessage(), equalTo("Field name [raw.foo] which is a multi field of [city] cannot contain '.'")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 2c0151f39d4cc..62bc263d0886a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -60,8 +60,7 @@ public void testEmptyNested() throws Exception { .startObject("nested1").field("type", "nested").endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", BytesReference .bytes(XContentFactory.jsonBuilder() @@ -89,8 +88,7 @@ public void testSingleNested() throws Exception { .startObject("nested1").field("type", "nested").endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); @@ -141,8 +139,7 @@ public void testMultiNested() throws Exception { .endObject().endObject().endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); @@ -202,8 +199,7 @@ public void testMultiObjectAndNested1() throws Exception { .endObject().endObject().endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); @@ -265,8 +261,7 @@ public void testMultiObjectAndNested2() throws Exception { .endObject().endObject().endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); @@ -326,8 +321,7 @@ public void testMultiRootAndNested1() throws Exception { .endObject().endObject().endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); @@ -520,8 +514,7 @@ public void testNestedArrayStrict() throws Exception { .endObject().endObject().endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); @@ -620,7 +613,7 @@ public void testLimitNestedDocsDefaultSettings() throws Exception{ String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = mapperService.parse("type", new CompressedXContent(mapping)); long defaultMaxNoNestedDocs = MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.get(settings); // parsing a doc with No. nested objects > defaultMaxNoNestedDocs fails @@ -655,7 +648,7 @@ public void testLimitNestedDocs() throws Exception{ String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = mapperService.parse("type", new CompressedXContent(mapping)); // parsing a doc with 2 nested objects succeeds XContentBuilder docBuilder = XContentFactory.jsonBuilder(); @@ -707,7 +700,7 @@ public void testLimitNestedDocsMultipleNestedFields() throws Exception{ .startObject("nested1").field("type", "nested").endObject() .startObject("nested2").field("type", "nested").endObject() .endObject().endObject().endObject()); - DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = mapperService.parse("type", new CompressedXContent(mapping)); // parsing a doc with 2 nested objects succeeds XContentBuilder docBuilder = XContentFactory.jsonBuilder(); @@ -774,8 +767,7 @@ public void testReorderParent() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), VersionUtils.randomIndexCompatibleVersion(random())).build(); - DocumentMapper docMapper = createIndex("test", settings) - .mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test", settings).mapperService().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.hasNestedObjects(), equalTo(true)); ObjectMapper nested1Mapper = docMapper.mappers().objectMappers().get("nested1"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NullValueObjectMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NullValueObjectMappingTests.java index 6b58551bea958..a1b884b3eb191 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NullValueObjectMappingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NullValueObjectMappingTests.java @@ -36,8 +36,7 @@ public void testNullValueObject() throws IOException { .startObject("properties").startObject("obj1").field("type", "object").endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 11ba0d2da9b39..6132adb16d8d3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -40,8 +40,7 @@ public void testDifferentInnerObjectTokenFailure() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { defaultMapper.parse(new SourceToParse("test", "1", new BytesArray(" {\n" + " \"object\": {\n" + @@ -65,7 +64,7 @@ public void testEmptyArrayProperties() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startArray("properties").endArray() .endObject().endObject()); - createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); } public void testEmptyFieldsArrayMultiFields() throws Exception { @@ -82,7 +81,7 @@ public void testEmptyFieldsArrayMultiFields() throws Exception { .endObject() .endObject() .endObject()); - createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)); + createIndex("test").mapperService().parse("tweet", new CompressedXContent(mapping)); } public void testFieldsArrayMultiFieldsShouldThrowException() throws Exception { @@ -101,7 +100,7 @@ public void testFieldsArrayMultiFieldsShouldThrowException() throws Exception { .endObject() .endObject()); try { - createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)); + createIndex("test").mapperService().parse("tweet", new CompressedXContent(mapping)); fail("Expected MapperParsingException"); } catch(MapperParsingException e) { assertThat(e.getMessage(), containsString("expected map for property [fields]")); @@ -119,7 +118,7 @@ public void testEmptyFieldsArray() throws Exception { .endObject() .endObject() .endObject()); - createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)); + createIndex("test").mapperService().parse("tweet", new CompressedXContent(mapping)); } public void testFieldsWithFilledArrayShouldThrowException() throws Exception { @@ -135,7 +134,7 @@ public void testFieldsWithFilledArrayShouldThrowException() throws Exception { .endObject() .endObject()); try { - createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)); + createIndex("test").mapperService().parse("tweet", new CompressedXContent(mapping)); fail("Expected MapperParsingException"); } catch (MapperParsingException e) { assertThat(e.getMessage(), containsString("Expected map for property [fields]")); @@ -158,7 +157,7 @@ public void testFieldPropertiesArray() throws Exception { .endObject() .endObject() .endObject()); - createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)); + createIndex("test").mapperService().parse("tweet", new CompressedXContent(mapping)); } public void testMerge() throws IOException { @@ -343,7 +342,7 @@ public void testEmptyName() throws Exception { // Empty name not allowed in index created after 5.0 IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { - createIndex("test").mapperService().documentMapperParser().parse("", new CompressedXContent(mapping)); + createIndex("test").mapperService().parse("", new CompressedXContent(mapping)); }); assertThat(e.getMessage(), containsString("name cannot be empty string")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java index c96b8c44ee707..f902b2a51f2a0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java @@ -31,8 +31,7 @@ public class PathMapperTests extends ESSingleNodeTestCase { public void testPathMapping() throws IOException { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/path/test-mapping.json"); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("person", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("person", new CompressedXContent(mapping)); // test full name assertThat(docMapper.mappers().getMapper("first1"), nullValue()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index 42f78b1934eab..3e52680c74679 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -266,10 +266,10 @@ public void testIllegalFormatField() throws Exception { .endObject() .endObject()); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); for (String m : Arrays.asList(mapping, dynamicMapping)) { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> parser.parse("type", new CompressedXContent(m))); + () -> mapperService.parse("type", new CompressedXContent(m))); assertEquals("Invalid format: [[test_format]]: expected string value", e.getMessage()); } } @@ -283,9 +283,9 @@ public void testIllegalDynamicTemplates() throws Exception { .endObject() .endObject()); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); MapperParsingException e = expectThrows(MapperParsingException.class, - () -> parser.parse("type", new CompressedXContent(mapping))); + () -> mapperService.parse("type", new CompressedXContent(mapping))); assertEquals("Dynamic template syntax error. An array of named objects is expected.", e.getMessage()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index cf1404875382c..cd57c8fa7b176 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -34,8 +34,7 @@ public class RoutingFieldMapperTests extends ESSingleNodeTestCase { public void testRoutingMapper() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", BytesReference .bytes(XContentFactory.jsonBuilder() @@ -50,8 +49,7 @@ public void testRoutingMapper() throws Exception { public void testIncludeInObjectNotAllowed() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); try { docMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 1dfbc069bcd09..f6389df774906 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -37,6 +37,7 @@ import java.util.Map; import static org.elasticsearch.index.MapperTestUtils.assertConflicts; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class SourceFieldMapperTests extends ESSingleNodeTestCase { @@ -51,8 +52,8 @@ public void testNoFormat() throws Exception { .startObject("_source").endObject() .endObject().endObject()); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(mapping)); + MapperService mapperService = createIndex("test").mapperService(); + DocumentMapper documentMapper = mapperService.parse("type", new CompressedXContent(mapping)); ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject() .field("field", "value") @@ -61,7 +62,7 @@ public void testNoFormat() throws Exception { assertThat(XContentFactory.xContentType(doc.source().toBytesRef().bytes), equalTo(XContentType.JSON)); - documentMapper = parser.parse("type", new CompressedXContent(mapping)); + documentMapper = mapperService.parse("type", new CompressedXContent(mapping)); doc = documentMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.smileBuilder().startObject() .field("field", "value") @@ -76,8 +77,7 @@ public void testIncludes() throws Exception { .startObject("_source").array("includes", new String[]{"path1*"}).endObject() .endObject().endObject()); - DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper documentMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject() @@ -100,8 +100,7 @@ public void testExcludes() throws Exception { .startObject("_source").array("excludes", "path1*").endObject() .endObject().endObject()); - DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper documentMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject() @@ -120,91 +119,86 @@ public void testExcludes() throws Exception { } public void testEnabledNotUpdateable() throws Exception { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); // using default of true String mapping1 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject()); String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").field("enabled", false).endObject() .endObject().endObject()); - assertConflicts(mapping1, mapping2, parser, "Cannot update parameter [enabled] from [true] to [false]"); + assertConflicts(mapping1, mapping2, mapperService, "Cannot update parameter [enabled] from [true] to [false]"); // not changing is ok String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").field("enabled", true).endObject() .endObject().endObject()); - assertConflicts(mapping1, mapping3, parser); + assertConflicts(mapping1, mapping3, mapperService); } public void testIncludesNotUpdateable() throws Exception { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject()); String mapping1 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").array("includes", "foo.*").endObject() .endObject().endObject()); - assertConflicts(defaultMapping, mapping1, parser, "Cannot update parameter [includes] from [[]] to [[foo.*]]"); - assertConflicts(mapping1, defaultMapping, parser, "Cannot update parameter [includes] from [[foo.*]] to [[]]"); + assertConflicts(defaultMapping, mapping1, mapperService, "Cannot update parameter [includes] from [[]] to [[foo.*]]"); + assertConflicts(mapping1, defaultMapping, mapperService, "Cannot update parameter [includes] from [[foo.*]] to [[]]"); String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").array("includes", "foo.*", "bar.*").endObject() .endObject().endObject()); - assertConflicts(mapping1, mapping2, parser, "Cannot update parameter [includes] from [[foo.*]] to [[foo.*, bar.*]]"); + assertConflicts(mapping1, mapping2, mapperService, "Cannot update parameter [includes] from [[foo.*]] to [[foo.*, bar.*]]"); // not changing is ok - assertConflicts(mapping1, mapping1, parser); + assertConflicts(mapping1, mapping1, mapperService); } public void testExcludesNotUpdateable() throws Exception { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject()); String mapping1 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").array("excludes", "foo.*").endObject() .endObject().endObject()); - assertConflicts(defaultMapping, mapping1, parser, "Cannot update parameter [excludes] from [[]] to [[foo.*]]"); - assertConflicts(mapping1, defaultMapping, parser, "Cannot update parameter [excludes] from [[foo.*]] to [[]]"); + assertConflicts(defaultMapping, mapping1, mapperService, "Cannot update parameter [excludes] from [[]] to [[foo.*]]"); + assertConflicts(mapping1, defaultMapping, mapperService, "Cannot update parameter [excludes] from [[foo.*]] to [[]]"); String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").array("excludes", "foo.*", "bar.*").endObject() .endObject().endObject()); - assertConflicts(mapping1, mapping2, parser, "Cannot update parameter [excludes]"); + assertConflicts(mapping1, mapping2, mapperService, "Cannot update parameter [excludes]"); // not changing is ok - assertConflicts(mapping1, mapping1, parser); + assertConflicts(mapping1, mapping1, mapperService); } public void testComplete() throws Exception { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject()); - assertTrue(parser.parse("type", new CompressedXContent(mapping)).sourceMapper().isComplete()); + assertTrue(mapperService.parse("type", new CompressedXContent(mapping)).sourceMapper().isComplete()); mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").field("enabled", false).endObject() .endObject().endObject()); - assertFalse(parser.parse("type", new CompressedXContent(mapping)).sourceMapper().isComplete()); + assertFalse(mapperService.parse("type", new CompressedXContent(mapping)).sourceMapper().isComplete()); mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").array("includes", "foo.*").endObject() .endObject().endObject()); - assertFalse(parser.parse("type", new CompressedXContent(mapping)).sourceMapper().isComplete()); + assertFalse(mapperService.parse("type", new CompressedXContent(mapping)).sourceMapper().isComplete()); mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_source").array("excludes", "foo.*").endObject() .endObject().endObject()); - assertFalse(parser.parse("type", new CompressedXContent(mapping)).sourceMapper().isComplete()); + assertFalse(mapperService.parse("type", new CompressedXContent(mapping)).sourceMapper().isComplete()); } public void testSourceObjectContainsExtraTokens() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject()); - DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); - - try { - documentMapper.parse(new SourceToParse("test", "1", - new BytesArray("{}}"), XContentType.JSON)); // extra end object (invalid JSON) - fail("Expected parse exception"); - } catch (MapperParsingException e) { - assertNotNull(e.getRootCause()); - String message = e.getRootCause().getMessage(); - assertTrue(message, message.contains("Unexpected close marker '}'")); - } + DocumentMapper documentMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); + + MapperParsingException exception = expectThrows(MapperParsingException.class, + // extra end object (invalid JSON)) + () -> documentMapper.parse(new SourceToParse("test", "1", new BytesArray("{}}"), XContentType.JSON))); + assertNotNull(exception.getRootCause()); + assertThat(exception.getRootCause().getMessage(), containsString("Unexpected close marker '}'")); } } diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java index 8694bb283b7e3..f77bb2e66cb10 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java @@ -215,7 +215,7 @@ public void testResolveSimilaritiesFromMapping_Unknown() throws IOException { IndexService indexService = createIndex("foo"); try { - indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + indexService.mapperService().parse("type", new CompressedXContent(mapping)); fail("Expected MappingParsingException"); } catch (MapperParsingException e) { assertThat(e.getMessage(), equalTo("Unknown Similarity type [unknown_similarity] for field [field1]")); diff --git a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java index 4cbb0bc28d77c..cc0e68582ea95 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java @@ -74,8 +74,7 @@ public void testIndexingWithNoContexts() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("completion"); ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "1", BytesReference .bytes(jsonBuilder() @@ -113,8 +112,7 @@ public void testIndexingWithSimpleContexts() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("completion"); ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "1", BytesReference .bytes(jsonBuilder() @@ -147,8 +145,7 @@ public void testIndexingWithSimpleNumberContexts() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("completion"); ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "1", BytesReference .bytes(jsonBuilder() @@ -181,8 +178,7 @@ public void testIndexingWithSimpleBooleanContexts() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("completion"); ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "1", BytesReference .bytes(jsonBuilder() @@ -215,8 +211,7 @@ public void testIndexingWithSimpleNULLContexts() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); XContentBuilder builder = jsonBuilder() .startObject() .startArray("completion") @@ -249,8 +244,7 @@ public void testIndexingWithContextList() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("completion"); ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "1", BytesReference .bytes(jsonBuilder() @@ -281,8 +275,7 @@ public void testIndexingWithMixedTypeContextList() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("completion"); ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "1", BytesReference .bytes(jsonBuilder() @@ -313,8 +306,7 @@ public void testIndexingWithMixedTypeContextListHavingNULL() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); XContentBuilder builder = jsonBuilder() .startObject() .startObject("completion") @@ -348,8 +340,7 @@ public void testIndexingWithMultipleContexts() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("completion"); XContentBuilder builder = jsonBuilder() .startObject() diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 3d2ab6d3fcd00..9fc0cec26f822 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -27,7 +27,6 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.similarity.SimilarityService; @@ -78,14 +77,14 @@ public static MapperService newMapperService(NamedXContentRegistry xContentRegis public static void assertConflicts(String mapping1, String mapping2, - DocumentMapperParser parser, + MapperService mapperService, String... conflicts) throws IOException { - DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping1)); + DocumentMapper docMapper = mapperService.parse("type", new CompressedXContent(mapping1)); if (conflicts.length == 0) { - docMapper.merge(parser.parse("type", new CompressedXContent(mapping2)).mapping(), MergeReason.MAPPING_UPDATE); + docMapper.merge(mapperService.parse("type", new CompressedXContent(mapping2)).mapping(), MergeReason.MAPPING_UPDATE); } else { Exception e = expectThrows(IllegalArgumentException.class, - () -> docMapper.merge(parser.parse("type", new CompressedXContent(mapping2)).mapping(), MergeReason.MAPPING_UPDATE)); + () -> docMapper.merge(mapperService.parse("type", new CompressedXContent(mapping2)).mapping(), MergeReason.MAPPING_UPDATE)); for (String conflict : conflicts) { assertThat(e.getMessage(), containsString(conflict)); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 4a947181c5e70..130696bdbcea6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -342,7 +342,7 @@ public static CheckedBiFunction ne final String nestedMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("nested_field").field("type", "nested").endObject().endObject() .endObject().endObject()); - final DocumentMapper nestedMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(nestedMapping)); + final DocumentMapper nestedMapper = mapperService.parse("type", new CompressedXContent(nestedMapping)); return (docId, nestedFieldValues) -> { final XContentBuilder source = XContentFactory.jsonBuilder().startObject().field("field", "value"); if (nestedFieldValues > 0) { diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java index ffb270b10e74b..67d0c60ee5252 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; @@ -279,16 +278,16 @@ public void testValidateNotDisallowedAttribute() throws IOException { } public void testCannotUpdateTimestampField() throws IOException { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); String mapping1 = "{\"type\":{\"_data_stream_timestamp\":{\"enabled\":false}, \"properties\": {\"@timestamp\": {\"type\": \"date\"}}}}}"; String mapping2 = "{\"type\":{\"_data_stream_timestamp\":{\"enabled\":true}, \"properties\": {\"@timestamp2\": " + "{\"type\": \"date\"},\"@timestamp\": {\"type\": \"date\"}}}})"; - assertConflicts(mapping1, mapping2, parser, "Mapper for [_data_stream_timestamp]", "[enabled] from [false] to [true]"); + assertConflicts(mapping1, mapping2, mapperService, "Mapper for [_data_stream_timestamp]", "[enabled] from [false] to [true]"); mapping1 = "{\"type\":{\"properties\":{\"@timestamp\": {\"type\": \"date\"}}}}}"; mapping2 = "{\"type\":{\"_data_stream_timestamp\":{\"enabled\":true}, \"properties\": " + "{\"@timestamp2\": {\"type\": \"date\"},\"@timestamp\": {\"type\": \"date\"}}}})"; - assertConflicts(mapping1, mapping2, parser, "Mapper for [_data_stream_timestamp]", "[enabled] from [false] to [true]"); + assertConflicts(mapping1, mapping2, mapperService, "Mapper for [_data_stream_timestamp]", "[enabled] from [false] to [true]"); } } diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java index 45fbb5fd944d1..a9bc3751ca508 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java @@ -13,8 +13,8 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; @@ -47,8 +47,7 @@ protected abstract XContentBuilder createDefaultMapping(String fieldName, public void testWKT() throws IOException { String mapping = Strings.toString(createDefaultMapping(FIELD_NAME, randomBoolean(), randomBoolean())); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -63,9 +62,9 @@ public void testWKT() throws IOException { public void testEmptyName() throws IOException { String mapping = Strings.toString(createDefaultMapping("", randomBoolean(), randomBoolean())); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> parser.parse("type", new CompressedXContent(mapping)) + () -> mapperService.parse("type", new CompressedXContent(mapping)) ); assertThat(e.getMessage(), containsString("name cannot be empty string")); } @@ -73,8 +72,7 @@ public void testEmptyName() throws IOException { public void testInvalidPointValuesIgnored() throws IOException { String mapping = Strings.toString(createDefaultMapping(FIELD_NAME, true, randomBoolean())); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); assertThat(defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -144,8 +142,7 @@ public void testInvalidPointValuesIgnored() throws IOException { public void testZValue() throws IOException { String mapping = Strings.toString(createDefaultMapping(FIELD_NAME, false, true)); - DocumentMapper defaultMapper = createIndex("test1").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test1").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test1", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -157,8 +154,7 @@ public void testZValue() throws IOException { assertThat(doc.rootDoc().getField(FIELD_NAME), notNullValue()); mapping = Strings.toString(createDefaultMapping(FIELD_NAME, false, false)); - DocumentMapper defaultMapper2 = createIndex("test2").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper2 = createIndex("test2").mapperService().parse("type", new CompressedXContent(mapping)); MapperParsingException e = expectThrows(MapperParsingException.class, () -> defaultMapper2.parse(new SourceToParse("test2", "1", diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java index b9c61e6868428..c89782582e5c1 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java @@ -37,7 +37,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.FieldMapperTestCase; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; @@ -98,8 +97,7 @@ public void testDefaultConfiguration() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -117,7 +115,7 @@ public void testDefaultDocValueConfigurationOnPre7_8() throws IOException { .endObject().endObject()); Version oldVersion = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_7_0); - DocumentMapper defaultMapper = createIndex("test", settings(oldVersion).build()).mapperService().documentMapperParser() + DocumentMapper defaultMapper = createIndex("test", settings(oldVersion).build()).mapperService() .parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -137,8 +135,7 @@ public void testOrientationParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -155,8 +152,7 @@ public void testOrientationParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -177,8 +173,7 @@ public void testCoerceParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -193,8 +188,7 @@ public void testCoerceParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -215,8 +209,7 @@ public void testIgnoreZValue() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -231,8 +224,7 @@ public void testIgnoreZValue() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -251,8 +243,7 @@ public void testIgnoreMalformedParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -267,8 +258,7 @@ public void testIgnoreMalformedParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -288,8 +278,7 @@ public void testDocValues() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -304,8 +293,7 @@ public void testDocValues() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(GeoShapeWithDocValuesFieldMapper.class)); @@ -361,40 +349,36 @@ public void testEmptyName() throws Exception { .field("type", "geo_shape") .endObject().endObject() .endObject().endObject()); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> parser.parse("type1", new CompressedXContent(mapping)) + () -> mapperService.parse("type1", new CompressedXContent(mapping)) ); assertThat(e.getMessage(), containsString("name cannot be empty string")); } public void testSerializeDefaults() throws Exception { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "geo_shape") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((GeoShapeWithDocValuesFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"orientation\":\"" + - AbstractShapeGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); - assertTrue(serialized, serialized.contains("\"doc_values\":true")); - } + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((GeoShapeWithDocValuesFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + + AbstractShapeGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":true")); } public void testSerializeDocValues() throws IOException { boolean docValues = randomBoolean(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") .field("type", "geo_shape") .field("doc_values", docValues) .endObject().endObject() .endObject().endObject()); - DocumentMapper mapper = parser.parse("type1", new CompressedXContent(mapping)); + DocumentMapper mapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); String serialized = toXContentString((GeoShapeWithDocValuesFieldMapper) mapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"orientation\":\"" + AbstractShapeGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); @@ -411,8 +395,7 @@ public void testGeoShapeArrayParsing() throws Exception { .endObject() .endObject()); - DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser() - .parse("_doc", new CompressedXContent(mapping)); + DocumentMapper mapper = createIndex("test").mapperService().parse("_doc", new CompressedXContent(mapping)); BytesReference arrayedDoc = BytesReference.bytes(XContentFactory.jsonBuilder() .startObject() diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index 4e896ebdac557..5f3bd1bcc16d0 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -49,8 +49,7 @@ public void testValuesStored() throws Exception { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "point"); String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -66,8 +65,7 @@ public void testArrayValues() throws Exception { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "point").field("doc_values", false); String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -88,8 +86,7 @@ public void testLatLonInOneValue() throws Exception { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "point"); String mapping = Strings.toString(xContentBuilder.endObject().endObject().endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -105,8 +102,7 @@ public void testInOneValueStored() throws Exception { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "point"); String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -121,8 +117,7 @@ public void testLatLonInOneValueArray() throws Exception { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "point").field("doc_values", false); String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -143,8 +138,7 @@ public void testArray() throws Exception { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "point"); String mapping = Strings.toString(xContentBuilder.endObject().endObject().endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -161,8 +155,7 @@ public void testArrayDynamic() throws Exception { .startArray("dynamic_templates").startObject().startObject("point").field("match", "point*") .startObject("mapping").field("type", "point"); String mapping = Strings.toString(xContentBuilder.endObject().endObject().endObject().endArray().endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -178,8 +171,7 @@ public void testArrayStored() throws Exception { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "point"); String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -198,8 +190,7 @@ public void testArrayArrayStored() throws Exception { String mapping = Strings.toString(xContentBuilder.field("store", true) .field("doc_values", false).endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "1", BytesReference.bytes(XContentFactory.jsonBuilder() @@ -223,8 +214,7 @@ public void testNullValue() throws Exception { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(PointFieldMapper.class)); @@ -271,8 +261,7 @@ public void testIgnoreZValue() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(PointFieldMapper.class)); @@ -287,8 +276,7 @@ public void testIgnoreZValue() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(PointFieldMapper.class)); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index d5c1842a627ca..e2b96c5cef761 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; @@ -56,8 +55,7 @@ public void testDefaultConfiguration() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); @@ -77,8 +75,7 @@ public void testOrientationParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); @@ -95,8 +92,7 @@ public void testOrientationParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); @@ -117,8 +113,7 @@ public void testCoerceParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); @@ -133,8 +128,7 @@ public void testCoerceParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); @@ -154,8 +148,7 @@ public void testIgnoreZValue() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); @@ -170,8 +163,7 @@ public void testIgnoreZValue() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); @@ -190,8 +182,7 @@ public void testIgnoreMalformedParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); @@ -206,8 +197,7 @@ public void testIgnoreMalformedParsing() throws IOException { .endObject().endObject() .endObject().endObject()); - defaultMapper = createIndex("test2").mapperService().documentMapperParser() - .parse("type1", new CompressedXContent(mapping)); + defaultMapper = createIndex("test2").mapperService().parse("type1", new CompressedXContent(mapping)); fieldMapper = defaultMapper.mappers().getMapper("location"); assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); @@ -250,18 +240,15 @@ public void testShapeMapperMerge() throws Exception { } public void testSerializeDefaults() throws Exception { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("properties").startObject("location") - .field("type", "shape") - .endObject().endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping)); - String serialized = toXContentString((ShapeFieldMapper) defaultMapper.mappers().getMapper("location")); - assertTrue(serialized, serialized.contains("\"orientation\":\"" + - AbstractShapeGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); - } + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "shape") + .endObject().endObject() + .endObject().endObject()); + DocumentMapper defaultMapper = createIndex("test").mapperService().parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((ShapeFieldMapper) defaultMapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + + AbstractShapeGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); } public void testShapeArrayParsing() throws Exception { @@ -274,8 +261,7 @@ public void testShapeArrayParsing() throws Exception { .endObject() .endObject()); - DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser() - .parse("_doc", new CompressedXContent(mapping)); + DocumentMapper mapper = createIndex("test").mapperService().parse("_doc", new CompressedXContent(mapping)); BytesReference arrayedDoc = BytesReference.bytes(XContentFactory.jsonBuilder() .startObject() From 1545c0aa78ee8c5cf2ff87ae3e171f05a65ba4e9 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 20 Oct 2020 20:12:16 +0200 Subject: [PATCH 49/60] Minor FieldTypeLookup tweaks (#63944) Remove the default constructor, make the class final and adjust visibility of its methods: if the class is package private, it makes little sense to have public methods. --- .../elasticsearch/index/mapper/FieldTypeLookup.java | 13 ++++--------- .../index/mapper/FieldTypeLookupTests.java | 2 +- .../elasticsearch/test/AbstractBuilderTestCase.java | 5 ----- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index b142a6f45c21c..9b27ba77b27d1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.regex.Regex; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -33,7 +32,7 @@ /** * An immutable container for looking up {@link MappedFieldType}s by their name. */ -class FieldTypeLookup implements Iterable { +final class FieldTypeLookup implements Iterable { private final Map fullNameToFieldType = new HashMap<>(); private final Map aliasToConcreteName = new HashMap<>(); @@ -48,10 +47,6 @@ class FieldTypeLookup implements Iterable { private final Map> fieldToCopiedFields = new HashMap<>(); private final DynamicKeyFieldTypeLookup dynamicKeyLookup; - FieldTypeLookup() { - this(Collections.emptyList(), Collections.emptyList()); - } - FieldTypeLookup(Collection fieldMappers, Collection fieldAliasMappers) { Map dynamicKeyMappers = new HashMap<>(); @@ -87,7 +82,7 @@ class FieldTypeLookup implements Iterable { /** * Returns the mapped field type for the given field name. */ - public MappedFieldType get(String field) { + MappedFieldType get(String field) { String concreteField = aliasToConcreteName.getOrDefault(field, field); MappedFieldType fieldType = fullNameToFieldType.get(concreteField); if (fieldType != null) { @@ -102,7 +97,7 @@ public MappedFieldType get(String field) { /** * Returns a list of the full names of a simple match regex like pattern against full name and index name. */ - public Set simpleMatchToFullName(String pattern) { + Set simpleMatchToFullName(String pattern) { Set fields = new HashSet<>(); for (MappedFieldType fieldType : this) { if (Regex.simpleMatch(pattern, fieldType.name())) { @@ -129,7 +124,7 @@ public Set simpleMatchToFullName(String pattern) { * should be a concrete field and *not* an alias. * @return A set of paths in the _source that contain the field's values. */ - public Set sourcePaths(String field) { + Set sourcePaths(String field) { String resolvedField = field; int lastDotIndex = field.lastIndexOf('.'); if (lastDotIndex > 0) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index 6344691695d1e..2c1c00739256a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -33,7 +33,7 @@ public class FieldTypeLookupTests extends ESTestCase { public void testEmpty() { - FieldTypeLookup lookup = new FieldTypeLookup(); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.emptyList(), Collections.emptyList()); assertNull(lookup.get("foo")); Collection names = lookup.simpleMatchToFullName("foo"); assertNotNull(names); diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 4403c8a7a346a..a8ea696a69154 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -52,7 +52,6 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.ShardId; @@ -202,10 +201,6 @@ protected static String expectedFieldName(String builderFieldName) { return ALIAS_TO_CONCRETE_FIELD_NAME.getOrDefault(builderFieldName, builderFieldName); } - protected Iterable getMapping() { - return serviceHolder.mapperService.fieldTypes(); - } - @AfterClass public static void afterClass() throws Exception { IOUtils.close(serviceHolder); From 55172b6bc0a2c072f9b80a3fc595429ec35ea36d Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 20 Oct 2020 20:16:35 +0200 Subject: [PATCH 50/60] [Transform] add support for unsigned_long data type (#63940) add support for unsigned_long, which required a change in writing out integer results properly, because coerce is not supported for unsigned_long fixes #63871 --- .../integration/TransformPivotRestIT.java | 12 ++-- .../continuous/DateHistogramGroupByIT.java | 12 ++-- .../DateHistogramGroupByOtherTimeFieldIT.java | 24 ++++---- .../continuous/TermsGroupByIT.java | 11 ++-- .../continuous/TransformContinuousIT.java | 2 +- .../pivot/AggregationResultUtils.java | 3 +- .../transforms/pivot/SchemaUtil.java | 59 +++++++++++++++---- .../pivot/AggregationResultUtilsTests.java | 12 +++- .../AggregationSchemaAndResultTests.java | 16 +++-- .../transforms/pivot/SchemaUtilTests.java | 17 ++++++ 10 files changed, 119 insertions(+), 49 deletions(-) diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index b4dd50b0983fe..0b4a453fb1100 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -558,15 +558,15 @@ public void testContinuousPivotHistogram() throws Exception { Map searchResult = getAsMap(transformIndex + "/_search?q=every_2:0.0"); assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult)); - assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(19.0)); + assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(19)); searchResult = getAsMap(transformIndex + "/_search?q=every_2:2.0"); assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult)); - assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(27.0)); + assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(27)); searchResult = getAsMap(transformIndex + "/_search?q=every_2:4.0"); assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult)); - assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(27.0)); + assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(27)); final StringBuilder bulk = new StringBuilder(); @@ -610,15 +610,15 @@ public void testContinuousPivotHistogram() throws Exception { searchResult = getAsMap(transformIndex + "/_search?q=every_2:0.0"); assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult)); - assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(19.0)); + assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(19)); searchResult = getAsMap(transformIndex + "/_search?q=every_2:2.0"); assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult)); - assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(30.0)); + assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(30)); searchResult = getAsMap(transformIndex + "/_search?q=every_2:4.0"); assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult)); - assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(27.0)); + assertThat(((List) XContentMapValues.extractValue("hits.hits._source.user_dc", searchResult)).get(0), equalTo(27)); } diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java index f26279602dc78..4364355d2afa8 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java @@ -130,8 +130,8 @@ public void testIteration(int iteration) throws IOException { ); assertThat( "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration, - XContentMapValues.extractValue("count", source), - equalTo(Double.valueOf(bucket.getDocCount())) + ((Integer) XContentMapValues.extractValue("count", source)).longValue(), + equalTo(bucket.getDocCount()) ); // transform should only rewrite documents that require it @@ -146,9 +146,11 @@ public void testIteration(int iteration) throws IOException { // we use a fixed_interval of `1s`, the transform runs every `1s` so it the bucket might be recalculated at the next run // but // should NOT be recalculated for the 2nd/3rd/... run - Double.valueOf((Integer) XContentMapValues.extractValue(INGEST_RUN_FIELD, source)) - (Double) XContentMapValues - .extractValue(MAX_RUN_FIELD, source), - is(lessThanOrEqualTo(1.0)) + (Integer) XContentMapValues.extractValue(INGEST_RUN_FIELD, source) - (Integer) XContentMapValues.extractValue( + MAX_RUN_FIELD, + source + ), + is(lessThanOrEqualTo(1)) ); } } diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java index 938834887afc8..65deb06b88a37 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java @@ -138,8 +138,8 @@ private void assertResultsGroupByDateHistogram(int iteration, SearchResponse res ); assertThat( "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration, - XContentMapValues.extractValue("count", source), - equalTo(Double.valueOf(bucket.getDocCount())) + ((Integer) XContentMapValues.extractValue("count", source)).longValue(), + equalTo(bucket.getDocCount()) ); // transform should only rewrite documents that require it @@ -152,9 +152,11 @@ private void assertResultsGroupByDateHistogram(int iteration, SearchResponse res + iteration, // we use a fixed_interval of `1s`, the transform runs every `1s`, a bucket might be recalculated at the next run // but should NOT be recalculated for the 2nd/3rd/... run - Double.valueOf((Integer) XContentMapValues.extractValue(INGEST_RUN_FIELD, source)) - (Double) XContentMapValues - .extractValue(MAX_RUN_FIELD, source), - is(lessThanOrEqualTo(1.0)) + (Integer) XContentMapValues.extractValue(INGEST_RUN_FIELD, source) - (Integer) XContentMapValues.extractValue( + MAX_RUN_FIELD, + source + ), + is(lessThanOrEqualTo(1)) ); } @@ -196,8 +198,8 @@ private void assertResultsGroupByDateHistogramAndTerms(int iteration, SearchResp ); assertThat( "Doc count did not match, source: " + source + ", expected: " + bucket.get("count") + ", iteration: " + iteration, - XContentMapValues.extractValue("count", source), - equalTo(Double.valueOf(((Long) bucket.get("count")))) + ((Integer) XContentMapValues.extractValue("count", source)).longValue(), + equalTo(bucket.get("count")) ); assertThat( "Term did not match, source: " + source + ", expected: " + bucket.get("event") + ", iteration: " + iteration, @@ -215,9 +217,11 @@ private void assertResultsGroupByDateHistogramAndTerms(int iteration, SearchResp + iteration, // we use a fixed_interval of `1s`, the transform runs every `1s`, a bucket might be recalculated at the next run // but should NOT be recalculated for the 2nd/3rd/... run - Double.valueOf((Integer) XContentMapValues.extractValue(INGEST_RUN_FIELD, source)) - (Double) XContentMapValues - .extractValue(MAX_RUN_FIELD, source), - is(lessThanOrEqualTo(1.0)) + (Integer) XContentMapValues.extractValue(INGEST_RUN_FIELD, source) - (Integer) XContentMapValues.extractValue( + MAX_RUN_FIELD, + source + ), + is(lessThanOrEqualTo(1)) ); } assertFalse(sourceIterator.hasNext()); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java index dd17ebfbc89cc..841463234bd7f 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java @@ -60,7 +60,7 @@ public TransformConfig createConfig() { AggregatorFactories.Builder aggregations = new AggregatorFactories.Builder(); addCommonAggregations(aggregations); - aggregations.addAggregator(AggregationBuilders.max("metric.avg").field("metric")); + aggregations.addAggregator(AggregationBuilders.avg("metric.avg").field("metric")); pivotConfigBuilder.setAggregations(aggregations); transformConfigBuilder.setPivotConfig(pivotConfigBuilder.build()); @@ -83,7 +83,7 @@ public void testIteration(int iteration) throws IOException { // missing_bucket produces `null`, we can't use `null` in aggs, so we have to use a magic value, see gh#60043 terms.missing(MISSING_BUCKET_KEY); } - terms.subAggregation(AggregationBuilders.max("metric.avg").field("metric")); + terms.subAggregation(AggregationBuilders.avg("metric.avg").field("metric")); sourceBuilderSource.aggregation(terms); searchRequestSource.source(sourceBuilderSource); SearchResponse responseSource = search(searchRequestSource); @@ -129,8 +129,8 @@ public void testIteration(int iteration) throws IOException { ); assertThat( "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration, - XContentMapValues.extractValue("count", source), - equalTo(Double.valueOf(bucket.getDocCount())) + ((Integer) XContentMapValues.extractValue("count", source)).longValue(), + equalTo(bucket.getDocCount()) ); SingleValue avgAgg = (SingleValue) bucket.getAggregations().get("metric.avg"); @@ -154,8 +154,7 @@ public void testIteration(int iteration) throws IOException { + iteration + " full source: " + source, - // TODO: aggs return double for MAX_RUN_FIELD, although it is an integer - Double.valueOf((Integer) XContentMapValues.extractValue(INGEST_RUN_FIELD, source)), + XContentMapValues.extractValue(INGEST_RUN_FIELD, source), equalTo(XContentMapValues.extractValue(MAX_RUN_FIELD, source)) ); } diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java index 4f298a71db3fe..8b011ecdabe46 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java @@ -316,7 +316,7 @@ private void putIndex(String indexName, String dateType, boolean isDataStream) t .field("type", "keyword") .endObject() .startObject("metric") - .field("type", "integer") + .field("type", randomFrom("integer", "long", "unsigned_long")) .endObject() .startObject("location") .field("type", "geo_point") diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java index fbdffd1ec86ab..c5bde60717123 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java @@ -40,6 +40,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.xpack.transform.transforms.pivot.SchemaUtil.dropFloatingPointComponentIfTypeRequiresIt; import static org.elasticsearch.xpack.transform.transforms.pivot.SchemaUtil.isNumericType; public final class AggregationResultUtils { @@ -198,7 +199,7 @@ public Object value(Aggregation agg, Map fieldTypeMap, String lo // If the type is numeric or if the formatted string is the same as simply making the value a string, // gather the `value` type, otherwise utilize `getValueAsString` so we don't lose formatted outputs. if (isNumericType(fieldType) || aggregation.getValueAsString().equals(String.valueOf(aggregation.value()))) { - return aggregation.value(); + return dropFloatingPointComponentIfTypeRequiresIt(fieldType, aggregation.value()); } else { return aggregation.getValueAsString(); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java index 7bbf1d2faddff..0c839ba799564 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtil.java @@ -23,30 +23,59 @@ import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig; +import java.math.BigDecimal; import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; public final class SchemaUtil { private static final Logger logger = LogManager.getLogger(SchemaUtil.class); - // Full collection of numeric field type strings - private static final Set NUMERIC_FIELD_MAPPER_TYPES; + // Full collection of numeric field type strings and whether they are floating point or not + private static final Map NUMERIC_FIELD_MAPPER_TYPES; static { - Set types = Stream.of(NumberFieldMapper.NumberType.values()) - .map(NumberFieldMapper.NumberType::typeName) - .collect(Collectors.toSet()); - types.add("scaled_float"); // have to add manually since scaled_float is in a module + Map types = Stream.of(NumberFieldMapper.NumberType.values()) + .collect(Collectors.toMap(t -> t.typeName(), t -> t.numericType().isFloatingPoint())); + + // have to add manually since they are in a module + types.put("scaled_float", true); + types.put("unsigned_long", false); NUMERIC_FIELD_MAPPER_TYPES = types; } private SchemaUtil() {} public static boolean isNumericType(String type) { - return type != null && NUMERIC_FIELD_MAPPER_TYPES.contains(type); + return type != null && NUMERIC_FIELD_MAPPER_TYPES.containsKey(type); + } + + /** + * Convert a numeric value to a whole number if it's not a floating point number. + * + * Implementation decision: We do not care about the concrete type, but only if its floating point or not. + * Further checks (e.g. range) are done at indexing. + * + * If type is floating point but ends with `.0`, we still preserve `.0` in case + * the destination index uses dynamic mappings as well as being json friendly. + * + * @param type the type of the value according to the schema we know + * @param value the value as double (aggs return double for everything) + * @return value if its floating point, long if value is smaller than Long.MAX_VALUE, BigInteger otherwise + */ + public static Object dropFloatingPointComponentIfTypeRequiresIt(String type, double value) { + if (NUMERIC_FIELD_MAPPER_TYPES.getOrDefault(type, true) == false) { + assert value % 1 == 0; + if (value < Long.MAX_VALUE) { + return (long) value; + } + + // special case for unsigned long + return BigDecimal.valueOf(value).toBigInteger(); + } + + return value; } /** @@ -188,8 +217,11 @@ private static Map resolveMappings( } else if (destinationMapping != null) { targetMapping.put(targetFieldName, destinationMapping); } else { - logger.warn("Failed to deduce mapping for [{}], fall back to dynamic mapping. " + - "Create the destination index with complete mappings first to avoid deducing the mappings", targetFieldName); + logger.warn( + "Failed to deduce mapping for [{}], fall back to dynamic mapping. " + + "Create the destination index with complete mappings first to avoid deducing the mappings", + targetFieldName + ); } }); @@ -199,8 +231,11 @@ private static Map resolveMappings( if (destinationMapping != null) { targetMapping.put(targetFieldName, destinationMapping); } else { - logger.warn("Failed to deduce mapping for [{}], fall back to keyword. " + - "Create the destination index with complete mappings first to avoid deducing the mappings", targetFieldName); + logger.warn( + "Failed to deduce mapping for [{}], fall back to keyword. " + + "Create the destination index with complete mappings first to avoid deducing the mappings", + targetFieldName + ); targetMapping.put(targetFieldName, KeywordFieldMapper.CONTENT_TYPE); } }); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java index 0040d5b7c7119..49ff7ad0a35fd 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java @@ -654,6 +654,12 @@ public void testSingleValueAggExtractor() { AggregationResultUtils.getExtractor(agg).value(agg, Collections.singletonMap("metric", "string"), ""), equalTo("one_hundred") ); + + agg = createSingleMetricAgg("metric", 100.0, "one_hundred"); + assertThat( + AggregationResultUtils.getExtractor(agg).value(agg, Collections.singletonMap("metric", "unsigned_long"), ""), + equalTo(100L) + ); } private ScriptedMetric createScriptedMetric(Object returnValue) { @@ -836,7 +842,7 @@ public void testSingleBucketAggExtractor() { ); assertThat( AggregationResultUtils.getExtractor(agg).value(agg, asStringMap("sba2.sub1", "long", "sba2.sub2", "float"), ""), - equalTo(asMap("sub1", 100.0, "sub2", 33.33)) + equalTo(asMap("sub1", 100L, "sub2", 33.33)) ); agg = createSingleBucketAgg( @@ -848,7 +854,7 @@ public void testSingleBucketAggExtractor() { ); assertThat( AggregationResultUtils.getExtractor(agg).value(agg, asStringMap("sba3.sub1", "long", "sba3.sub2", "double"), ""), - equalTo(asMap("sub1", 100.0, "sub2", 33.33, "sub3", 42L)) + equalTo(asMap("sub1", 100L, "sub2", 33.33, "sub3", 42L)) ); agg = createSingleBucketAgg( @@ -861,7 +867,7 @@ public void testSingleBucketAggExtractor() { assertThat( AggregationResultUtils.getExtractor(agg) .value(agg, asStringMap("sba4.sub3.subsub1", "double", "sba4.sub2", "float", "sba4.sub1", "long"), ""), - equalTo(asMap("sub1", 100.0, "sub2", 33.33, "sub3", asMap("subsub1", 11.1))) + equalTo(asMap("sub1", 100L, "sub2", 33.33, "sub3", asMap("subsub1", 11.1))) ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java index a2e811993e522..613d049b48319 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java @@ -127,8 +127,11 @@ public void testBasic() throws InterruptedException { AggregationConfig aggregationConfig = new AggregationConfig(Collections.emptyMap(), aggs); GroupConfig groupConfig = GroupConfigTests.randomGroupConfig(); PivotConfig pivotConfig = new PivotConfig(groupConfig, aggregationConfig, null); - long numGroupsWithoutScripts = groupConfig.getGroups().values().stream() - .filter(singleGroupSource -> singleGroupSource.getScriptConfig() == null).count(); + long numGroupsWithoutScripts = groupConfig.getGroups() + .values() + .stream() + .filter(singleGroupSource -> singleGroupSource.getScriptConfig() == null) + .count(); this.>assertAsync( listener -> SchemaUtil.deduceMappings(client, pivotConfig, new String[] { "source-index" }, listener), @@ -191,8 +194,11 @@ public void testNested() throws InterruptedException { AggregationConfig aggregationConfig = new AggregationConfig(Collections.emptyMap(), aggs); GroupConfig groupConfig = GroupConfigTests.randomGroupConfig(); PivotConfig pivotConfig = new PivotConfig(groupConfig, aggregationConfig, null); - long numGroupsWithoutScripts = groupConfig.getGroups().values().stream() - .filter(singleGroupSource -> singleGroupSource.getScriptConfig() == null).count(); + long numGroupsWithoutScripts = groupConfig.getGroups() + .values() + .stream() + .filter(singleGroupSource -> singleGroupSource.getScriptConfig() == null) + .count(); this.>assertAsync( listener -> SchemaUtil.deduceMappings(client, pivotConfig, new String[] { "source-index" }, listener), @@ -219,7 +225,7 @@ public void testNested() throws InterruptedException { 23144, AggregationResultUtilsTests.createSingleMetricAgg("max_drinks_2", 45.0, "forty_five") ); - assertThat(AggregationResultUtils.getExtractor(agg).value(agg, mappings, ""), equalTo(asMap("max_drinks_2", 45.0))); + assertThat(AggregationResultUtils.getExtractor(agg).value(agg, mappings, ""), equalTo(asMap("max_drinks_2", 45L))); agg = AggregationResultUtilsTests.createSingleBucketAgg( "filter_3", diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java index 8dd8a5f816221..3db24778e4fa4 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java @@ -8,9 +8,12 @@ import org.elasticsearch.test.ESTestCase; +import java.math.BigInteger; import java.util.HashMap; import java.util.Map; +import static org.hamcrest.CoreMatchers.instanceOf; + public class SchemaUtilTests extends ESTestCase { public void testInsertNestedObjectMappings() { @@ -54,4 +57,18 @@ public void testInsertNestedObjectMappings() { assertFalse(fieldMappings.containsKey("")); } + public void testConvertToIntegerTypeIfNeeded() { + assertEquals(33L, SchemaUtil.dropFloatingPointComponentIfTypeRequiresIt("unsigned_long", 33.0)); + assertEquals(33L, SchemaUtil.dropFloatingPointComponentIfTypeRequiresIt("long", 33.0)); + assertEquals(33.0, SchemaUtil.dropFloatingPointComponentIfTypeRequiresIt("double", 33.0)); + assertEquals(33.0, SchemaUtil.dropFloatingPointComponentIfTypeRequiresIt("half_float", 33.0)); + assertEquals(33.0, SchemaUtil.dropFloatingPointComponentIfTypeRequiresIt("unknown", 33.0)); + assertEquals(33.0, SchemaUtil.dropFloatingPointComponentIfTypeRequiresIt(null, 33.0)); + + Object value = SchemaUtil.dropFloatingPointComponentIfTypeRequiresIt("unsigned_long", 1.8446744073709551615E19); + assertThat(value, instanceOf(BigInteger.class)); + + assertEquals(new BigInteger("18446744073709551615").doubleValue(), ((BigInteger) value).doubleValue(), 0.0); + } + } From 9a9b96b30b9fa89810a81c4e087898aba1842951 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 20 Oct 2020 15:19:57 -0400 Subject: [PATCH 51/60] Make agg test less confusing (#63952) We have an `@After` annotated method in `AggregatorTestCase` that cleans up releasibles. But it was `private`! Confusingly, it seemed to be working! I'm not sure why. This makes it public which is a little more sensible. --- .../elasticsearch/search/aggregations/AggregatorTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 743f1833a5131..77a4be0ff9c1e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -844,7 +844,7 @@ public IndexAnalyzers getIndexAnalyzers() { } @After - private void cleanupReleasables() { + public void cleanupReleasables() { Releasables.close(releasables); releasables.clear(); } From 7db889c79cd9462ae7fa611753d154f84cac08b7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 20 Oct 2020 15:46:53 -0400 Subject: [PATCH 52/60] Add a test for regex usage to runtime fields (#63951) Now that we've got regexes enabled by default (#63029) this adds a test to runtime fields just to make sure that it works with regexes. It does, but this adds a test to make sure it continues to work. --- .../test/runtime_fields/50_ip.yml | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml index 1208615b5515e..8e936e79f0266 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml @@ -18,18 +18,20 @@ setup: runtime_type: ip script: source: | - String m = doc["message"].value; - int end = m.indexOf(" "); - emit(m.substring(0, end)); + Matcher m = /([^ ]+) .+/.matcher(doc["message"].value); + if (m.matches()) { + emit(m.group(1)); + } # Test fetching from _source ip_from_source: type: runtime runtime_type: ip script: source: | - String m = params._source.message; - int end = m.indexOf(" "); - emit(m.substring(0, end)); + Matcher m = /([^ ]+) .+/.matcher(params._source.message); + if (m.matches()) { + emit(m.group(1)); + } # Test emitting many values ip_many: type: runtime @@ -71,9 +73,10 @@ setup: - match: {http_logs.mappings.properties.ip.runtime_type: ip } - match: http_logs.mappings.properties.ip.script.source: | - String m = doc["message"].value; - int end = m.indexOf(" "); - emit(m.substring(0, end)); + Matcher m = /([^ ]+) .+/.matcher(doc["message"].value); + if (m.matches()) { + emit(m.group(1)); + } - match: {http_logs.mappings.properties.ip.script.lang: painless } --- From 70f758e198ca0ffa52ae1d7779906ad5c510f778 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Wed, 21 Oct 2020 09:00:51 +0300 Subject: [PATCH 53/60] [ML] Extend default evaluation metrics to all available (#63939) This commit extends the set of default metrics for the data frame analytics evaluation API to all available metrics. The motivation is that if the user skips setting an explicit set of metrics, they get most of the evaluation offering. --- .../evaluation/classification/Classification.java | 2 +- .../ml/dataframe/evaluation/regression/Huber.java | 4 ++++ .../dataframe/evaluation/regression/Regression.java | 2 +- .../classification/ClassificationTests.java | 9 +++++++++ .../outlierdetection/OutlierDetectionTests.java | 12 ++++++++++++ .../evaluation/regression/RegressionTests.java | 9 +++++++++ .../ml/integration/ClassificationEvaluationIT.java | 9 ++++++++- .../xpack/ml/integration/RegressionEvaluationIT.java | 8 +++++++- .../rest-api-spec/test/ml/evaluate_data_frame.yml | 6 +++++- 9 files changed, 56 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Classification.java index 680cda5396779..adb257525bcfe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Classification.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/Classification.java @@ -92,7 +92,7 @@ public Classification(String actualField, } private static List defaultMetrics() { - return Arrays.asList(new MulticlassConfusionMatrix()); + return Arrays.asList(new Accuracy(), new MulticlassConfusionMatrix(), new Precision(), new Recall()); } public Classification(StreamInput in) throws IOException { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/Huber.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/Huber.java index 978ac0c74cded..4c6ee1af7f6bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/Huber.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/Huber.java @@ -80,6 +80,10 @@ public Huber(StreamInput in) throws IOException { this.delta = in.readDouble(); } + public Huber() { + this(DEFAULT_DELTA); + } + public Huber(@Nullable Double delta) { this.delta = delta != null ? delta : DEFAULT_DELTA; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/Regression.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/Regression.java index a90e6821255d7..a8bf53c781272 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/Regression.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/Regression.java @@ -76,7 +76,7 @@ public Regression(String actualField, String predictedField, @Nullable List defaultMetrics() { - return Arrays.asList(new MeanSquaredError(), new RSquared()); + return Arrays.asList(new MeanSquaredError(), new RSquared(), new Huber()); } public Regression(StreamInput in) throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java index c511ac42ee4ca..60c30b94a7fb7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java @@ -40,6 +40,7 @@ import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty; import static org.elasticsearch.test.hamcrest.OptionalMatchers.isPresent; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -110,6 +111,14 @@ public void testConstructor_GivenEmptyMetrics() { assertThat(e.getMessage(), equalTo("[classification] must have one or more metrics")); } + public void testConstructor_GivenDefaultMetrics() { + Classification classification = new Classification("actual", "predicted", null, null); + + List metrics = classification.getMetrics(); + + assertThat(metrics, containsInAnyOrder(new Accuracy(), new MulticlassConfusionMatrix(), new Precision(), new Recall())); + } + public void testGetFields() { Classification evaluation = new Classification("foo", "bar", "results", null); EvaluationFields fields = evaluation.getFields(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/OutlierDetectionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/OutlierDetectionTests.java index c0b72dbe1c234..d68daf218882f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/OutlierDetectionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/OutlierDetectionTests.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -89,6 +90,17 @@ public void testConstructor_GivenEmptyMetrics() { assertThat(e.getMessage(), equalTo("[outlier_detection] must have one or more metrics")); } + public void testConstructor_GivenDefaultMetrics() { + OutlierDetection outlierDetection = new OutlierDetection("actual", "predicted", null); + + List metrics = outlierDetection.getMetrics(); + + assertThat(metrics, containsInAnyOrder(new AucRoc(false), + new Precision(Arrays.asList(0.25, 0.5, 0.75)), + new Recall(Arrays.asList(0.25, 0.5, 0.75)), + new ConfusionMatrix(Arrays.asList(0.25, 0.5, 0.75)))); + } + public void testGetFields() { OutlierDetection evaluation = new OutlierDetection("foo", "bar", null); EvaluationFields fields = evaluation.getFields(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java index c8fc2d5d67d55..c6d060a3b84e4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -76,6 +77,14 @@ public void testConstructor_GivenEmptyMetrics() { assertThat(e.getMessage(), equalTo("[regression] must have one or more metrics")); } + public void testConstructor_GivenDefaultMetrics() { + Regression regression = new Regression("actual", "predicted", null); + + List metrics = regression.getMetrics(); + + assertThat(metrics, containsInAnyOrder(new Huber(), new MeanSquaredError(), new RSquared())); + } + public void testGetFields() { Regression evaluation = new Regression("foo", "bar", null); EvaluationFields fields = evaluation.getFields(); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java index 3deebf757c3a5..a02a3c4869b5c 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java @@ -32,6 +32,7 @@ import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -81,7 +82,13 @@ public void testEvaluate_DefaultMetrics() { assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Classification.NAME.getPreferredName())); assertThat( evaluateDataFrameResponse.getMetrics().stream().map(EvaluationMetricResult::getMetricName).collect(toList()), - contains(MulticlassConfusionMatrix.NAME.getPreferredName())); + containsInAnyOrder( + MulticlassConfusionMatrix.NAME.getPreferredName(), + Accuracy.NAME.getPreferredName(), + Precision.NAME.getPreferredName(), + Recall.NAME.getPreferredName() + ) + ); } public void testEvaluate_AllMetrics() { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionEvaluationIT.java index a947fe66c03cb..a849d244a0f82 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionEvaluationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionEvaluationIT.java @@ -24,6 +24,7 @@ import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -52,7 +53,12 @@ public void testEvaluate_DefaultMetrics() { assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(Regression.NAME.getPreferredName())); assertThat( evaluateDataFrameResponse.getMetrics().stream().map(EvaluationMetricResult::getMetricName).collect(toList()), - contains(MeanSquaredError.NAME.getPreferredName(), RSquared.NAME.getPreferredName())); + containsInAnyOrder( + MeanSquaredError.NAME.getPreferredName(), + RSquared.NAME.getPreferredName(), + Huber.NAME.getPreferredName() + ) + ); } public void testEvaluate_AllMetrics() { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml index 09cf11d266612..83fe922c02492 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml @@ -938,6 +938,10 @@ setup: } - is_true: classification.multiclass_confusion_matrix + - is_true: classification.accuracy + - is_true: classification.precision + - is_true: classification.recall + - is_false: classification.auc_roc --- "Test classification given missing actual_field": - do: @@ -1104,8 +1108,8 @@ setup: - match: { regression.mse.value: 28.67749840974834 } - match: { regression.r_squared.value: 0.8551031778603486 } + - match: { regression.huber.value: 1.9205280586939963 } - is_false: regression.msle.value - - is_false: regression.huber.value --- "Test regression given missing actual_field": - do: From 1376afd82871398bf0ad5c0f56824e6d3352cebc Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 21 Oct 2020 09:58:54 +0200 Subject: [PATCH 54/60] geo_point runtime field implementation (#63164) Run time field that emits geo points from lat/lon values. --- .../common/geo/GeoShapeUtils.java | 127 ++++++++ .../query/VectorGeoShapeQueryProcessor.java | 143 +-------- .../xpack/runtimefields/RuntimeFields.java | 2 + .../fielddata/GeoPointScriptDocValues.java | 49 ++++ .../fielddata/GeoPointScriptFieldData.java | 105 +++++++ .../mapper/GeoPointFieldScript.java | 66 +++++ .../mapper/GeoPointScriptFieldType.java | 89 ++++++ .../mapper/RuntimeFieldMapper.java | 15 + .../RuntimeFieldsPainlessExtension.java | 1 + .../AbstractGeoPointScriptFieldQuery.java | 31 ++ .../query/GeoPointScriptFieldExistsQuery.java | 31 ++ .../GeoPointScriptFieldGeoShapeQuery.java | 68 +++++ .../mapper/geo_point_whitelist.txt | 18 ++ .../AbstractScriptFieldTypeTestCase.java | 18 ++ .../mapper/GeoPointFieldScriptTests.java | 73 +++++ .../mapper/GeoPointScriptFieldTypeTests.java | 275 ++++++++++++++++++ .../mapper/RuntimeFieldMapperTests.java | 3 + ...tractGeoPointScriptFieldQueryTestCase.java | 22 ++ .../GeoPointScriptFieldExistsQueryTests.java | 41 +++ ...GeoPointScriptFieldGeoShapeQueryTests.java | 52 ++++ .../test/runtime_fields/100_geo_point.yml | 154 ++++++++++ 21 files changed, 1255 insertions(+), 128 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/GeoPointScriptDocValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/GeoPointScriptFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractGeoPointScriptFieldQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldExistsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldGeoShapeQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/mapper/geo_point_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldTypeTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractGeoPointScriptFieldQueryTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldExistsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldGeoShapeQueryTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/100_geo_point.yml diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoShapeUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoShapeUtils.java index 7e23385f2adf8..b3105a61e1a57 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoShapeUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoShapeUtils.java @@ -18,11 +18,24 @@ */ package org.elasticsearch.common.geo; +import org.apache.lucene.geo.LatLonGeometry; import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; + +import java.util.ArrayList; +import java.util.List; /** @@ -60,6 +73,120 @@ public static org.apache.lucene.geo.Circle toLuceneCircle(Circle circle) { return new org.apache.lucene.geo.Circle(circle.getLat(), circle.getLon(), circle.getRadiusMeters()); } + public static LatLonGeometry[] toLuceneGeometry( + String name, + QueryShardContext context, + Geometry geometry, + List> unsupportedGeometries + ) { + final List geometries = new ArrayList<>(); + geometry.visit(new GeometryVisitor<>() { + @Override + public Void visit(Circle circle) { + checkSupported(circle); + if (circle.isEmpty() == false) { + geometries.add(GeoShapeUtils.toLuceneCircle(circle)); + } + return null; + } + + @Override + public Void visit(GeometryCollection collection) { + checkSupported(collection); + if (collection.isEmpty() == false) { + for (Geometry shape : collection) { + shape.visit(this); + } + } + return null; + } + + @Override + public Void visit(org.elasticsearch.geometry.Line line) { + checkSupported(line); + if (line.isEmpty() == false) { + geometries.add(GeoShapeUtils.toLuceneLine(line)); + } + return null; + } + + @Override + public Void visit(LinearRing ring) { + throw new QueryShardException(context, "Field [" + name + "] found and unsupported shape LinearRing"); + } + + @Override + public Void visit(MultiLine multiLine) { + checkSupported(multiLine); + if (multiLine.isEmpty() == false) { + for (Line line : multiLine) { + visit(line); + } + } + return null; + } + + @Override + public Void visit(MultiPoint multiPoint) { + checkSupported(multiPoint); + if (multiPoint.isEmpty() == false) { + for (Point point : multiPoint) { + visit(point); + } + } + return null; + } + + @Override + public Void visit(MultiPolygon multiPolygon) { + checkSupported(multiPolygon); + if (multiPolygon.isEmpty() == false) { + for (Polygon polygon : multiPolygon) { + visit(polygon); + } + } + return null; + } + + @Override + public Void visit(Point point) { + checkSupported(point); + if (point.isEmpty() == false) { + geometries.add(toLucenePoint(point)); + } + return null; + + } + + @Override + public Void visit(org.elasticsearch.geometry.Polygon polygon) { + checkSupported(polygon); + if (polygon.isEmpty() == false) { + List collector = new ArrayList<>(); + GeoPolygonDecomposer.decomposePolygon(polygon, true, collector); + collector.forEach((p) -> geometries.add(toLucenePolygon(p))); + } + return null; + } + + @Override + public Void visit(Rectangle r) { + checkSupported(r); + if (r.isEmpty() == false) { + geometries.add(toLuceneRectangle(r)); + } + return null; + } + + private void checkSupported(Geometry geometry) { + if (unsupportedGeometries.contains(geometry.getClass())) { + throw new QueryShardException(context, "Field [" + name + "] found and unsupported shape [" + geometry.type() + "]"); + } + } + }); + return geometries.toArray(new LatLonGeometry[geometries.size()]); + } + private GeoShapeUtils() { } diff --git a/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java b/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java index eb519011faa32..24f58bf55fd63 100644 --- a/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java +++ b/server/src/main/java/org/elasticsearch/index/query/VectorGeoShapeQueryProcessor.java @@ -24,29 +24,25 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.Version; -import org.elasticsearch.common.geo.GeoLineDecomposer; -import org.elasticsearch.common.geo.GeoPolygonDecomposer; import org.elasticsearch.common.geo.GeoShapeUtils; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.GeometryCollection; -import org.elasticsearch.geometry.GeometryVisitor; import org.elasticsearch.geometry.Line; -import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.MultiLine; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class VectorGeoShapeQueryProcessor { + private static final List> WITHIN_UNSUPPORTED_GEOMETRIES = new ArrayList<>(); + static { + WITHIN_UNSUPPORTED_GEOMETRIES.add(Line.class); + WITHIN_UNSUPPORTED_GEOMETRIES.add(MultiLine.class); + } + public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context) { // CONTAINS queries are not supported by VECTOR strategy for indices created before version 7.5.0 (Lucene 8.3.0) if (relation == ShapeRelation.CONTAINS && context.indexVersionCreated().before(Version.V_7_5_0)) { @@ -58,125 +54,16 @@ public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relat } private Query getVectorQueryFromShape(Geometry queryShape, String fieldName, ShapeRelation relation, QueryShardContext context) { - final LuceneGeometryCollector visitor = new LuceneGeometryCollector(fieldName, context); - queryShape.visit(visitor); - final List geometries = visitor.geometries(); - if (geometries.size() == 0) { - return new MatchNoDocsQuery(); - } - return LatLonShape.newGeometryQuery(fieldName, relation.getLuceneRelation(), - geometries.toArray(new LatLonGeometry[geometries.size()])); - } - - private static class LuceneGeometryCollector implements GeometryVisitor { - private final List geometries = new ArrayList<>(); - private final String name; - private final QueryShardContext context; - - private LuceneGeometryCollector(String name, QueryShardContext context) { - this.name = name; - this.context = context; - } - - List geometries() { - return geometries; - } - - @Override - public Void visit(Circle circle) { - if (circle.isEmpty() == false) { - geometries.add(GeoShapeUtils.toLuceneCircle(circle)); - } - return null; - } - - @Override - public Void visit(GeometryCollection collection) { - for (Geometry shape : collection) { - shape.visit(this); - } - return null; - } - - @Override - public Void visit(org.elasticsearch.geometry.Line line) { - if (line.isEmpty() == false) { - List collector = new ArrayList<>(); - GeoLineDecomposer.decomposeLine(line, collector); - collectLines(collector); - } - return null; - } - - @Override - public Void visit(LinearRing ring) { - throw new QueryShardException(context, "Field [" + name + "] found and unsupported shape LinearRing"); + final LatLonGeometry[] luceneGeometries; + if (relation == ShapeRelation.WITHIN) { + luceneGeometries = GeoShapeUtils.toLuceneGeometry(fieldName, context, queryShape, WITHIN_UNSUPPORTED_GEOMETRIES); + } else { + luceneGeometries = GeoShapeUtils.toLuceneGeometry(fieldName, context, queryShape, Collections.emptyList()); } - - @Override - public Void visit(MultiLine multiLine) { - List collector = new ArrayList<>(); - GeoLineDecomposer.decomposeMultiLine(multiLine, collector); - collectLines(collector); - return null; - } - - @Override - public Void visit(MultiPoint multiPoint) { - for (Point point : multiPoint) { - visit(point); - } - return null; - } - - @Override - public Void visit(MultiPolygon multiPolygon) { - if (multiPolygon.isEmpty() == false) { - List collector = new ArrayList<>(); - GeoPolygonDecomposer.decomposeMultiPolygon(multiPolygon, true, collector); - collectPolygons(collector); - } - return null; - } - - @Override - public Void visit(Point point) { - if (point.isEmpty() == false) { - geometries.add(GeoShapeUtils.toLucenePoint(point)); - } - return null; - - } - - @Override - public Void visit(org.elasticsearch.geometry.Polygon polygon) { - if (polygon.isEmpty() == false) { - List collector = new ArrayList<>(); - GeoPolygonDecomposer.decomposePolygon(polygon, true, collector); - collectPolygons(collector); - } - return null; - } - - @Override - public Void visit(Rectangle r) { - if (r.isEmpty() == false) { - geometries.add(GeoShapeUtils.toLuceneRectangle(r)); - } - return null; - } - - private void collectLines(List geometryLines) { - for (Line line: geometryLines) { - geometries.add(GeoShapeUtils.toLuceneLine(line)); - } - } - - private void collectPolygons(List geometryPolygons) { - for (Polygon polygon : geometryPolygons) { - geometries.add(GeoShapeUtils.toLucenePolygon(polygon)); - } + if (luceneGeometries.length == 0) { + return new MatchNoDocsQuery(); } + return LatLonShape.newGeometryQuery(fieldName, relation.getLuceneRelation(), luceneGeometries); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 730d11a82b30d..62cb58fd94567 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.runtimefields.mapper.BooleanFieldScript; import org.elasticsearch.xpack.runtimefields.mapper.DateFieldScript; import org.elasticsearch.xpack.runtimefields.mapper.DoubleFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; import org.elasticsearch.xpack.runtimefields.mapper.IpFieldScript; import org.elasticsearch.xpack.runtimefields.mapper.LongFieldScript; import org.elasticsearch.xpack.runtimefields.mapper.RuntimeFieldMapper; @@ -36,6 +37,7 @@ public List> getContexts() { BooleanFieldScript.CONTEXT, DateFieldScript.CONTEXT, DoubleFieldScript.CONTEXT, + GeoPointFieldScript.CONTEXT, IpFieldScript.CONTEXT, LongFieldScript.CONTEXT, StringFieldScript.CONTEXT diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/GeoPointScriptDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/GeoPointScriptDocValues.java new file mode 100644 index 0000000000000..79a10625a4cc4 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/GeoPointScriptDocValues.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.geo.GeoEncodingUtils; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; + +import java.util.Arrays; + +public final class GeoPointScriptDocValues extends MultiGeoPointValues { + private final GeoPointFieldScript script; + private final GeoPoint point; + private int cursor; + + GeoPointScriptDocValues(GeoPointFieldScript script) { + this.script = script; + this.point = new GeoPoint(); + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + if (script.count() == 0) { + return false; + } + Arrays.sort(script.values(), 0, script.count()); + cursor = 0; + return true; + } + + @Override + public int docValueCount() { + return script.count(); + } + + @Override + public GeoPoint nextValue() { + final long value = script.values()[cursor++]; + final int lat = (int) (value >>> 32); + final int lon = (int) (value & 0xFFFFFFFF); + return point.reset(GeoEncodingUtils.decodeLatitude(lat), GeoEncodingUtils.decodeLongitude(lon)); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/GeoPointScriptFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/GeoPointScriptFieldData.java new file mode 100644 index 0000000000000..ef8ef730b73c2 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/GeoPointScriptFieldData.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.LeafGeoPointFieldData; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.plain.AbstractLeafGeoPointFieldData; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; + +public class GeoPointScriptFieldData implements IndexGeoPointFieldData { + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final GeoPointFieldScript.LeafFactory leafFactory; + + public Builder(String name, GeoPointFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public GeoPointScriptFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + return new GeoPointScriptFieldData(name, leafFactory); + } + } + + private final GeoPointFieldScript.LeafFactory leafFactory; + private final String name; + + private GeoPointScriptFieldData(String fieldName, GeoPointFieldScript.LeafFactory leafFactory) { + this.name = fieldName; + this.leafFactory = leafFactory; + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); + } + + @Override + public String getFieldName() { + return name; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.GEOPOINT; + } + + @Override + public LeafGeoPointFieldData load(LeafReaderContext context) { + GeoPointFieldScript script = leafFactory.newInstance(context); + return new AbstractLeafGeoPointFieldData() { + @Override + public MultiGeoPointValues getGeoPointValues() { + return new GeoPointScriptDocValues(script); + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() { + + } + }; + } + + @Override + public LeafGeoPointFieldData loadDirect(LeafReaderContext context) { + return load(context); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScript.java new file mode 100644 index 0000000000000..536b007cb8c1e --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScript.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; +import org.apache.lucene.document.LatLonDocValuesField; + +import java.util.List; +import java.util.Map; + +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; + +/** + * Script producing geo points. Similarly to what {@link LatLonDocValuesField} does, + * it encodes the points as a long value. + */ +public abstract class GeoPointFieldScript extends AbstractLongFieldScript { + public static final ScriptContext CONTEXT = newContext("geo_point_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "geo_point_whitelist.txt")); + } + + @SuppressWarnings("unused") + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(String fieldName, Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + GeoPointFieldScript newInstance(LeafReaderContext ctx); + } + + public GeoPointFieldScript(String fieldName, Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(fieldName, params, searchLookup, ctx); + } + + protected void emit(double lat, double lon) { + int latitudeEncoded = encodeLatitude(lat); + int longitudeEncoded = encodeLongitude(lon); + emit(Long.valueOf((((long) latitudeEncoded) << 32) | (longitudeEncoded & 0xFFFFFFFFL))); + } + + public static class Emit { + private final GeoPointFieldScript script; + + public Emit(GeoPointFieldScript script) { + this.script = script; + } + + public void emit(double lat, double lon) { + script.emit(lat, lon); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldType.java new file mode 100644 index 0000000000000..10d524b390e18 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldType.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.geo.LatLonGeometry; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.geo.GeoShapeUtils; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeQueryable; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xpack.runtimefields.fielddata.GeoPointScriptFieldData; +import org.elasticsearch.xpack.runtimefields.query.GeoPointScriptFieldExistsQuery; +import org.elasticsearch.xpack.runtimefields.query.GeoPointScriptFieldGeoShapeQuery; + +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +public final class GeoPointScriptFieldType extends AbstractScriptFieldType implements GeoShapeQueryable { + + private static final List> UNSUPPORTED_GEOMETRIES = new ArrayList<>(); + static { + UNSUPPORTED_GEOMETRIES.add(Line.class); + UNSUPPORTED_GEOMETRIES.add(MultiLine.class); + } + + GeoPointScriptFieldType(String name, Script script, GeoPointFieldScript.Factory scriptFactory, Map meta) { + super(name, script, scriptFactory::newFactory, meta); + } + + @Override + protected String runtimeType() { + return GeoPointFieldMapper.CONTENT_TYPE; + } + + @Override + protected Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ZoneId timeZone, + DateMathParser parser, + QueryShardContext context + ) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support range queries"); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new IllegalArgumentException( + "Geometry fields do not support exact searching, use dedicated geometry queries instead: [" + name() + "]" + ); + } + + @Override + public GeoPointScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + return new GeoPointScriptFieldData.Builder(name(), leafFactory(searchLookup.get())); + } + + @Override + public Query existsQuery(QueryShardContext context) { + checkAllowExpensiveQueries(context); + return new GeoPointScriptFieldExistsQuery(script, leafFactory(context), name()); + } + + @Override + public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context) { + if (shape == null) { + return new MatchNoDocsQuery(); + } + final LatLonGeometry[] geometries = GeoShapeUtils.toLuceneGeometry(fieldName, context, shape, UNSUPPORTED_GEOMETRIES); + return new GeoPointScriptFieldGeoShapeQuery(script, leafFactory(context), fieldName, geometries); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java index f4b757d505adb..630ca4df72f5e 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java @@ -11,6 +11,7 @@ import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; @@ -133,6 +134,20 @@ public static class Builder extends ParametrizedFieldMapper.Builder { builder.meta.getValue() ); }, + GeoPointFieldMapper.CONTENT_TYPE, + (builder, context) -> { + builder.formatAndLocaleNotSupported(); + GeoPointFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + GeoPointFieldScript.CONTEXT + ); + return new GeoPointScriptFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, NumberType.LONG.typeName(), (builder, context) -> { builder.formatAndLocaleNotSupported(); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldsPainlessExtension.java index dcd9ccfb505bd..509110ed4ba26 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldsPainlessExtension.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldsPainlessExtension.java @@ -20,6 +20,7 @@ public Map, List> getContextWhitelists() { Map.entry(BooleanFieldScript.CONTEXT, BooleanFieldScript.whitelist()), Map.entry(DateFieldScript.CONTEXT, DateFieldScript.whitelist()), Map.entry(DoubleFieldScript.CONTEXT, DoubleFieldScript.whitelist()), + Map.entry(GeoPointFieldScript.CONTEXT, GeoPointFieldScript.whitelist()), Map.entry(IpFieldScript.CONTEXT, IpFieldScript.whitelist()), Map.entry(LongFieldScript.CONTEXT, LongFieldScript.whitelist()), Map.entry(StringFieldScript.CONTEXT, StringFieldScript.whitelist()) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractGeoPointScriptFieldQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractGeoPointScriptFieldQuery.java new file mode 100644 index 0000000000000..067d1b07960dc --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractGeoPointScriptFieldQuery.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; + +/** + * Abstract base class for building queries based on {@link GeoPointFieldScript}. + */ +abstract class AbstractGeoPointScriptFieldQuery extends AbstractScriptFieldQuery { + + AbstractGeoPointScriptFieldQuery(Script script, GeoPointFieldScript.LeafFactory leafFactory, String fieldName) { + super(script, fieldName, leafFactory::newInstance); + } + + @Override + protected boolean matches(GeoPointFieldScript scriptContext, int docId) { + scriptContext.runForDoc(docId); + return matches(scriptContext.values(), scriptContext.count()); + } + + /** + * Does the value match this query? + */ + protected abstract boolean matches(long[] values, int count); +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldExistsQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldExistsQuery.java new file mode 100644 index 0000000000000..201f78573dea3 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldExistsQuery.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; + +public class GeoPointScriptFieldExistsQuery extends AbstractGeoPointScriptFieldQuery { + public GeoPointScriptFieldExistsQuery(Script script, GeoPointFieldScript.LeafFactory leafFactory, String fieldName) { + super(script, leafFactory, fieldName); + } + + @Override + protected boolean matches(long[] values, int count) { + return count > 0; + } + + @Override + public final String toString(String field) { + if (fieldName().contentEquals(field)) { + return getClass().getSimpleName(); + } + return fieldName() + ":" + getClass().getSimpleName(); + } + + // Superclass's equals and hashCode are great for this class +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldGeoShapeQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldGeoShapeQuery.java new file mode 100644 index 0000000000000..fdf82efb20ec8 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldGeoShapeQuery.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.geo.LatLonGeometry; +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; + +import java.util.Arrays; +import java.util.Objects; + +public class GeoPointScriptFieldGeoShapeQuery extends AbstractGeoPointScriptFieldQuery { + + // This class should be called Component2DPredicate in Lucene... + private final GeoEncodingUtils.PolygonPredicate predicate; + private final LatLonGeometry[] geometries; + + public GeoPointScriptFieldGeoShapeQuery( + Script script, + GeoPointFieldScript.LeafFactory leafFactory, + String fieldName, + LatLonGeometry... geometries + ) { + super(script, leafFactory, fieldName); + this.geometries = geometries; + predicate = GeoEncodingUtils.createComponentPredicate(LatLonGeometry.create(geometries)); + } + + @Override + protected boolean matches(long[] values, int count) { + for (int i = 0; i < count; i++) { + final long value = values[i]; + final int lat = (int) (value >>> 32); + final int lon = (int) (value & 0xFFFFFFFF); + if (predicate.test(lat, lon)) { + return true; + } + } + return false; + } + + @Override + public final String toString(String field) { + if (fieldName().contentEquals(field)) { + return getClass().getSimpleName(); + } + return fieldName() + ":" + getClass().getSimpleName(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + GeoPointScriptFieldGeoShapeQuery that = (GeoPointScriptFieldGeoShapeQuery) o; + return Arrays.equals(geometries, that.geometries); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), Arrays.hashCode(geometries)); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/mapper/geo_point_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/mapper/geo_point_whitelist.txt new file mode 100644 index 0000000000000..3f24d1a07cc4b --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/mapper/geo_point_whitelist.txt @@ -0,0 +1,18 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for ip-valued runtime fields + +# These two whitelists are required for painless to find the classes +class org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript @no_import { +} +class org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript$Factory @no_import { +} + +static_import { + # The `emit` callback to collect values for the field + void emit(org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript, double, double) bound_to org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript$Emit +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldTypeTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldTypeTestCase.java index 9e5d876bcf08a..6e173a7c72714 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldTypeTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldTypeTestCase.java @@ -66,6 +66,14 @@ protected static QueryShardContext mockContext(boolean allowExpensiveQueries) { return mockContext(allowExpensiveQueries, null); } + protected boolean supportsTermQueries() { + return true; + } + + protected boolean supportsRangeQueries() { + return true; + } + protected static QueryShardContext mockContext(boolean allowExpensiveQueries, MappedFieldType mappedFieldType) { MapperService mapperService = mock(MapperService.class); when(mapperService.fieldType(anyString())).thenReturn(mappedFieldType); @@ -102,42 +110,52 @@ public void testRangeQueryWithShapeRelationIsError() { } public void testRangeQueryIsExpensive() { + assumeTrue("Impl does not support range queries", supportsRangeQueries()); checkExpensiveQuery(this::randomRangeQuery); } public void testRangeQueryInLoop() { + assumeTrue("Impl does not support range queries", supportsRangeQueries()); checkLoop(this::randomRangeQuery); } public void testTermQueryIsExpensive() { + assumeTrue("Impl does not support term queries", supportsTermQueries()); checkExpensiveQuery(this::randomTermQuery); } public void testTermQueryInLoop() { + assumeTrue("Impl does not support term queries", supportsTermQueries()); checkLoop(this::randomTermQuery); } public void testTermsQueryIsExpensive() { + assumeTrue("Impl does not support term queries", supportsTermQueries()); checkExpensiveQuery(this::randomTermsQuery); } public void testTermsQueryInLoop() { + assumeTrue("Impl does not support term queries", supportsTermQueries()); checkLoop(this::randomTermsQuery); } public void testPhraseQueryIsError() { + assumeTrue("Impl does not support term queries", supportsTermQueries()); assertQueryOnlyOnText("phrase", () -> simpleMappedFieldType().phraseQuery(null, 1, false)); } public void testPhrasePrefixQueryIsError() { + assumeTrue("Impl does not support term queries", supportsTermQueries()); assertQueryOnlyOnText("phrase prefix", () -> simpleMappedFieldType().phrasePrefixQuery(null, 1, 1)); } public void testMultiPhraseQueryIsError() { + assumeTrue("Impl does not support term queries", supportsTermQueries()); assertQueryOnlyOnText("phrase", () -> simpleMappedFieldType().multiPhraseQuery(null, 1, false)); } public void testSpanPrefixQueryIsError() { + assumeTrue("Impl does not support term queries", supportsTermQueries()); assertQueryOnlyOnText("span prefix", () -> simpleMappedFieldType().spanPrefixQuery(null, null, null)); } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScriptTests.java new file mode 100644 index 0000000000000..f011b46940e26 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScriptTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + +public class GeoPointFieldScriptTests extends FieldScriptTestCase { + public static final GeoPointFieldScript.Factory DUMMY = (fieldName, params, lookup) -> ctx -> new GeoPointFieldScript( + fieldName, + params, + lookup, + ctx + ) { + @Override + public void execute() { + emit(0, 0); + } + }; + + @Override + protected ScriptContext context() { + return GeoPointFieldScript.CONTEXT; + } + + @Override + protected GeoPointFieldScript.Factory dummyScript() { + return DUMMY; + } + + public void testTooManyValues() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{}")))); + try (DirectoryReader reader = iw.getReader()) { + GeoPointFieldScript script = new GeoPointFieldScript( + "test", + Map.of(), + new SearchLookup(mock(MapperService.class), (ft, lookup) -> null), + reader.leaves().get(0) + ) { + @Override + public void execute() { + for (int i = 0; i <= AbstractFieldScript.MAX_VALUES; i++) { + emit(0, 0); + } + } + }; + Exception e = expectThrows(IllegalArgumentException.class, script::execute); + assertThat( + e.getMessage(), + equalTo("Runtime field [test] is emitting [101] values while the maximum number of values allowed is [100]") + ); + } + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldTypeTests.java new file mode 100644 index 0000000000000..bd65acd8ced3b --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldTypeTests.java @@ -0,0 +1,275 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.xpack.runtimefields.RuntimeFields; +import org.elasticsearch.xpack.runtimefields.fielddata.GeoPointScriptFieldData; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.equalTo; + +public class GeoPointScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTestCase { + + @Override + protected boolean supportsTermQueries() { + return false; + } + + @Override + protected boolean supportsRangeQueries() { + return false; + } + + @Override + public void testDocValues() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": {\"lat\": 45.0, \"lon\" : 45.0}}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": {\"lat\": 0.0, \"lon\" : 0.0}}")))); + List results = new ArrayList<>(); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + GeoPointScriptFieldType ft = build("fromLatLon", Map.of()); + GeoPointScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null); + searcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) { + MultiGeoPointValues dv = ifd.load(context).getGeoPointValues(); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) {} + + @Override + public void collect(int doc) throws IOException { + if (dv.advanceExact(doc)) { + for (int i = 0; i < dv.docValueCount(); i++) { + final GeoPoint point = dv.nextValue(); + results.add(new GeoPoint(point.lat(), point.lon())); + } + } + } + }; + } + }); + assertThat(results, equalTo(List.of(new GeoPoint(45.0, 45.0), new GeoPoint(0.0, 0.0)))); + } + } + } + + @Override + public void testSort() throws IOException { + GeoPointScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder("test", mockContext()::lookup).build(null, null); + Exception e = expectThrows(IllegalArgumentException.class, () -> ifd.sortField(null, MultiValueMode.MIN, null, false)); + assertThat(e.getMessage(), equalTo("can't sort on geo_point field without using specific sorting feature, like geo_distance")); + } + + @Override + public void testUsedInScript() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": {\"lat\": 45.0, \"lon\" : 45.0}}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": {\"lat\": 0.0, \"lon\" : 0.0}}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + QueryShardContext qsc = mockContext(true, simpleMappedFieldType()); + assertThat(searcher.count(new ScriptScoreQuery(new MatchAllDocsQuery(), new Script("test"), new ScoreScript.LeafFactory() { + @Override + public boolean needs_score() { + return false; + } + + @Override + public ScoreScript newInstance(LeafReaderContext ctx) { + return new ScoreScript(Map.of(), qsc.lookup(), ctx) { + @Override + public double execute(ExplanationHolder explanation) { + ScriptDocValues.GeoPoints points = (ScriptDocValues.GeoPoints) getDoc().get("test"); + return (int) points.get(0).lat() + 1; + } + }; + } + }, 2.5f, "test", 0, Version.CURRENT)), equalTo(1)); + } + } + } + + @Override + public void testExistsQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": {\"lat\": 45.0, \"lon\" : 45.0}}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": {\"lat\": 0.0, \"lon\" : 0.0}}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat(searcher.count(simpleMappedFieldType().existsQuery(mockContext())), equalTo(2)); + } + } + } + + @Override + public void testRangeQuery() throws IOException { + Exception e = expectThrows( + IllegalArgumentException.class, + () -> simpleMappedFieldType().rangeQuery("0.0", "45.0", false, false, null, null, null, mockContext()) + ); + assertThat(e.getMessage(), equalTo("Field [test] of type [runtime] does not support range queries")); + } + + @Override + protected Query randomRangeQuery(MappedFieldType ft, QueryShardContext ctx) { + throw new IllegalArgumentException("Unsupported"); + } + + @Override + public void testTermQuery() { + Exception e = expectThrows(IllegalArgumentException.class, () -> simpleMappedFieldType().termQuery("0.0,0.0", mockContext())); + assertThat( + e.getMessage(), + equalTo("Geometry fields do not support exact searching, use dedicated geometry queries instead: [test]") + ); + } + + @Override + protected Query randomTermQuery(MappedFieldType ft, QueryShardContext ctx) { + throw new IllegalArgumentException("Unsupported"); + } + + @Override + public void testTermsQuery() { + Exception e = expectThrows( + IllegalArgumentException.class, + () -> simpleMappedFieldType().termsQuery(List.of("0.0,0.0", "45.0,45.0"), mockContext()) + ); + + assertThat( + e.getMessage(), + equalTo("Geometry fields do not support exact searching, use dedicated geometry queries instead: [test]") + ); + + } + + @Override + protected Query randomTermsQuery(MappedFieldType ft, QueryShardContext ctx) { + return ft.termsQuery(randomList(100, () -> GeometryTestUtils.randomPoint()), mockContext()); + } + + @Override + protected GeoPointScriptFieldType simpleMappedFieldType() throws IOException { + return build("fromLatLon", Map.of()); + } + + @Override + protected MappedFieldType loopFieldType() throws IOException { + return build("loop", Map.of()); + } + + @Override + protected String runtimeType() { + return "geo_point"; + } + + private static GeoPointScriptFieldType build(String code, Map params) throws IOException { + return build(new Script(ScriptType.INLINE, "test", code, params)); + } + + private static GeoPointScriptFieldType build(Script script) throws IOException { + ScriptPlugin scriptPlugin = new ScriptPlugin() { + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "test"; + } + + @Override + public Set> getSupportedContexts() { + return Set.of(StringFieldScript.CONTEXT, GeoPointFieldScript.CONTEXT); + } + + @Override + public FactoryType compile( + String name, + String code, + ScriptContext context, + Map params + ) { + @SuppressWarnings("unchecked") + FactoryType factory = (FactoryType) factory(code); + return factory; + } + + private GeoPointFieldScript.Factory factory(String code) { + switch (code) { + case "fromLatLon": + return (fieldName, params, lookup) -> (ctx) -> new GeoPointFieldScript(fieldName, params, lookup, ctx) { + @Override + public void execute() { + Map foo = (Map) getSource().get("foo"); + emit(((Number) foo.get("lat")).doubleValue(), ((Number) foo.get("lon")).doubleValue()); + } + }; + case "loop": + return (fieldName, params, lookup) -> { + // Indicate that this script wants the field call "test", which *is* the name of this field + lookup.forkAndTrackFieldReferences("test"); + throw new IllegalStateException("shoud have thrown on the line above"); + }; + default: + throw new IllegalArgumentException("unsupported script [" + code + "]"); + } + } + }; + } + }; + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(scriptPlugin, new RuntimeFields())); + try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { + GeoPointFieldScript.Factory factory = scriptService.compile(script, GeoPointFieldScript.CONTEXT); + return new GeoPointScriptFieldType("test", script, factory, emptyMap()); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapperTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapperTests.java index b851c3f6e50d5..90432baa36027 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapperTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapperTests.java @@ -396,6 +396,9 @@ protected Object buildScriptFactory(ScriptContext context) { if (context == StringFieldScript.CONTEXT) { return StringFieldScriptTests.DUMMY; } + if (context == GeoPointFieldScript.CONTEXT) { + return GeoPointFieldScriptTests.DUMMY; + } throw new IllegalArgumentException("Unsupported context: " + context); }; diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractGeoPointScriptFieldQueryTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractGeoPointScriptFieldQueryTestCase.java new file mode 100644 index 0000000000000..d6cd0b8369e75 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractGeoPointScriptFieldQueryTestCase.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; + +import static org.mockito.Mockito.mock; + +public abstract class AbstractGeoPointScriptFieldQueryTestCase extends + AbstractScriptFieldQueryTestCase { + + protected final GeoPointFieldScript.LeafFactory leafFactory = mock(GeoPointFieldScript.LeafFactory.class); + + @Override + public final void testVisit() { + assertEmptyVisit(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldExistsQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldExistsQueryTests.java new file mode 100644 index 0000000000000..a1ad9b0948847 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldExistsQueryTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import static org.hamcrest.Matchers.equalTo; + +public class GeoPointScriptFieldExistsQueryTests extends AbstractGeoPointScriptFieldQueryTestCase { + @Override + protected GeoPointScriptFieldExistsQuery createTestInstance() { + return new GeoPointScriptFieldExistsQuery(randomScript(), leafFactory, randomAlphaOfLength(5)); + } + + @Override + protected GeoPointScriptFieldExistsQuery copy(GeoPointScriptFieldExistsQuery orig) { + return new GeoPointScriptFieldExistsQuery(orig.script(), leafFactory, orig.fieldName()); + } + + @Override + protected GeoPointScriptFieldExistsQuery mutate(GeoPointScriptFieldExistsQuery orig) { + if (randomBoolean()) { + new GeoPointScriptFieldExistsQuery(randomValueOtherThan(orig.script(), this::randomScript), leafFactory, orig.fieldName()); + } + return new GeoPointScriptFieldExistsQuery(orig.script(), leafFactory, orig.fieldName() + "modified"); + } + + @Override + public void testMatches() { + assertTrue(createTestInstance().matches(new long[] { 1L }, randomIntBetween(1, Integer.MAX_VALUE))); + assertFalse(createTestInstance().matches(new long[0], 0)); + assertFalse(createTestInstance().matches(new long[1], 0)); + } + + @Override + protected void assertToString(GeoPointScriptFieldExistsQuery query) { + assertThat(query.toString(query.fieldName()), equalTo("GeoPointScriptFieldExistsQuery")); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldGeoShapeQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldGeoShapeQueryTests.java new file mode 100644 index 0000000000000..c1183d8e7515f --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/GeoPointScriptFieldGeoShapeQueryTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.geo.Polygon; + +import static org.hamcrest.Matchers.equalTo; + +public class GeoPointScriptFieldGeoShapeQueryTests extends AbstractGeoPointScriptFieldQueryTestCase { + + private static final Polygon polygon1 = new Polygon(new double[] { -10, -10, 10, 10, -10 }, new double[] { -10, 10, 10, -10, -10 }); + private static final Polygon polygon2 = new Polygon(new double[] { -11, -10, 10, 10, -11 }, new double[] { -10, 10, 10, -10, -10 }); + + @Override + protected GeoPointScriptFieldGeoShapeQuery createTestInstance() { + return new GeoPointScriptFieldGeoShapeQuery(randomScript(), leafFactory, randomAlphaOfLength(5), polygon1); + } + + @Override + protected GeoPointScriptFieldGeoShapeQuery copy(GeoPointScriptFieldGeoShapeQuery orig) { + return new GeoPointScriptFieldGeoShapeQuery(orig.script(), leafFactory, orig.fieldName(), polygon1); + } + + @Override + protected GeoPointScriptFieldGeoShapeQuery mutate(GeoPointScriptFieldGeoShapeQuery orig) { + if (randomBoolean()) { + new GeoPointScriptFieldGeoShapeQuery( + randomValueOtherThan(orig.script(), this::randomScript), + leafFactory, + orig.fieldName(), + polygon2 + ); + } + return new GeoPointScriptFieldGeoShapeQuery(orig.script(), leafFactory, orig.fieldName() + "modified", polygon1); + } + + @Override + public void testMatches() { + assertTrue(createTestInstance().matches(new long[] { 1L }, randomIntBetween(1, Integer.MAX_VALUE))); + assertFalse(createTestInstance().matches(new long[0], 0)); + assertFalse(createTestInstance().matches(new long[1], 0)); + } + + @Override + protected void assertToString(GeoPointScriptFieldGeoShapeQuery query) { + assertThat(query.toString(query.fieldName()), equalTo("GeoPointScriptFieldGeoShapeQuery")); + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/100_geo_point.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/100_geo_point.yml new file mode 100644 index 0000000000000..6803b79c79ea7 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/100_geo_point.yml @@ -0,0 +1,154 @@ +--- +setup: + - do: + indices.create: + index: locations + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + location: + type: geo_point + location_from_doc_value: + type: runtime + runtime_type: geo_point + script: + source: | + emit(doc["location"].lat, doc["location"].lon); + location_from_source: + type: runtime + runtime_type: geo_point + script: + source: | + emit(params._source.location.lat, params._source.location.lon); + + + - do: + bulk: + index: locations + refresh: true + body: | + {"index":{}} + {"timestamp": "1998-04-30T14:30:17-05:00", "location" : {"lat": 13.5, "lon" : 34.89}} + {"index":{}} + {"timestamp": "1998-04-30T14:30:53-05:00", "location" : {"lat": -7.9, "lon" : 120.78}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:12-05:00", "location" : {"lat": 45.78, "lon" : -173.45}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:19-05:00", "location" : {"lat": 32.45, "lon" : 45.6}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:22-05:00", "location" : {"lat": -63.24, "lon" : 31.0}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:27-05:00", "location" : {"lat": 0.0, "lon" : 0.0}} + + +--- +"get mapping": + - do: + indices.get_mapping: + index: locations + - match: {locations.mappings.properties.location_from_source.type: runtime } + - match: {locations.mappings.properties.location_from_source.runtime_type: geo_point } + - match: + locations.mappings.properties.location_from_source.script.source: | + emit(params._source.location.lat, params._source.location.lon); + - match: {locations.mappings.properties.location_from_source.script.lang: painless } + + +--- +"fetch fields from source": + - do: + search: + index: locations + body: + sort: timestamp + fields: [location, location_from_doc_value, location_from_source] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.location.0.type: "Point" } + - match: {hits.hits.0.fields.location.0.coordinates: [34.89, 13.5] } + - match: {hits.hits.0.fields.location_from_doc_value: ["13.499999991618097, 34.889999935403466"] } + - match: {hits.hits.0.fields.location_from_source: ["13.499999991618097, 34.889999935403466"] } + +--- +"exists query": + - do: + search: + index: locations + body: + query: + exists: + field: location_from_source + - match: {hits.total.value: 6} + +--- +"geo bounding box query": + - do: + search: + index: locations + body: + query: + geo_shape: + location_from_source: + shape: + type: "envelope" + coordinates: [ [ -10, 10 ], [ 10, -10 ] ] + - match: {hits.total.value: 1} + +--- +"bounds agg": + - do: + search: + index: locations + body: + aggs: + bounds: + geo_bounds: + field: "location" + wrap_longitude: false + bounds_from_doc_value: + geo_bounds: + field: "location_from_doc_value" + wrap_longitude: false + bounds_from_source: + geo_bounds: + field: "location_from_source" + wrap_longitude: false + - match: {hits.total.value: 6} + - match: {aggregations.bounds.bounds.top_left.lat: 45.7799999602139 } + - match: {aggregations.bounds.bounds.top_left.lon: -173.4500000718981 } + - match: {aggregations.bounds.bounds.bottom_right.lat: -63.240000014193356 } + - match: {aggregations.bounds.bounds.bottom_right.lon: 120.77999993227422 } + - match: {aggregations.bounds_from_doc_value.bounds.top_left.lat: 45.7799999602139 } + - match: {aggregations.bounds_from_doc_value.bounds.top_left.lon: -173.4500000718981 } + - match: {aggregations.bounds_from_doc_value.bounds.bottom_right.lat: -63.240000014193356 } + - match: {aggregations.bounds_from_doc_value.bounds.bottom_right.lon: 120.77999993227422 } + - match: {aggregations.bounds_from_source.bounds.top_left.lat: 45.7799999602139 } + - match: {aggregations.bounds_from_source.bounds.top_left.lon: -173.4500000718981 } + - match: {aggregations.bounds_from_source.bounds.bottom_right.lat: -63.240000014193356 } + - match: {aggregations.bounds_from_source.bounds.bottom_right.lon: 120.77999993227422 } + +--- +"geo_distance sort": + - do: + search: + index: locations + body: + sort: + _geo_distance: + location_from_source: + lat: 0.0 + lon: 0.0 + - match: {hits.total.value: 6} + - match: {hits.hits.0._source.location.lat: 0.0 } + - match: {hits.hits.0._source.location.lon: 0.0 } + - match: {hits.hits.1._source.location.lat: 13.5 } + - match: {hits.hits.1._source.location.lon: 34.89 } + - match: {hits.hits.2._source.location.lat: 32.45 } + - match: {hits.hits.2._source.location.lon: 45.6 } + - match: {hits.hits.3._source.location.lat: -63.24 } + - match: {hits.hits.3._source.location.lon: 31.0 } + From dec6ea937fe0d2c6687f77b0eef2a7f4c39ed5ee Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 21 Oct 2020 11:04:58 +0200 Subject: [PATCH 55/60] Mute CoreValuesSourceTypeTests.testDatePrepareRoundingWithDocs (#63970) Relates to #63969 --- .../search/aggregations/support/CoreValuesSourceTypeTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java index 37b87463f0ee4..af4f50cf581f2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java @@ -57,6 +57,7 @@ public void testDatePrepareRoundingWithNothing() throws IOException { }); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63969") public void testDatePrepareRoundingWithDocs() throws IOException { long min = randomLongBetween(0, 1000000); long max = randomLongBetween(min + 1, 100000000000L); From 00b0bdff6a396656286abbc54525241cb89291f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 21 Oct 2020 11:16:37 +0200 Subject: [PATCH 56/60] Handle range query edge case (#63397) Currently when searching with an empty string as lower bound for a range query on text-based fields we return all documents when 'gte' is used (including the lower bound) but no documents when 'gt' is used. This might seem counterintuitive since every value should be greate than the empty string. The bug has been fixed in Lucene and this PR adds a test for assuring we observe the fixed behaviour on searches now. Closes #63386 --- .../search/simple/SimpleSearchIT.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 6a163543e3073..2ef7a961fde68 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -225,6 +225,51 @@ public void testSimpleDateRange() throws Exception { assertHitCount(searchResponse, 2L); } + public void testRangeQueryKeyword() throws Exception { + createIndex("test"); + + client().admin().indices().preparePutMapping("test").setSource("field", "type=keyword").get(); + + client().prepareIndex("test").setId("0").setSource("field", "").get(); + client().prepareIndex("test").setId("1").setSource("field", "A").get(); + client().prepareIndex("test").setId("2").setSource("field", "B").get(); + client().prepareIndex("test").setId("3").setSource("field", "C").get(); + ensureGreen(); + refresh(); + + SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("A").lte("B")).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gt("A").lte("B")).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 1L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("A").lt("B")).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 1L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte(null).lt("C")).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 3L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("B").lt(null)).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gt(null).lt(null)).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 4L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("").lt(null)).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 4L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gt("").lt(null)).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 3L); + } + public void testSimpleTerminateAfterCount() throws Exception { prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); ensureGreen(); From 473975c4d57f5795c61ed0fb53f8771c0517ddb6 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 21 Oct 2020 11:30:11 +0200 Subject: [PATCH 57/60] Mute DatafeedJobsIT#testDatafeedTimingStats_DatafeedRecreated (#63974) Relates to #63973 --- .../org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java index e75022c965059..7fe9cced1216a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java @@ -145,6 +145,7 @@ public void testLookbackOnlyDataStream() throws Exception { waitUntilJobIsClosed(job.getId()); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63973") public void testDatafeedTimingStats_DatafeedRecreated() throws Exception { client().admin().indices().prepareCreate("data") .setMapping("time", "type=date") From f45ea3068cd65b6f5b12d7b9de90e1bb19620fbc Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 21 Oct 2020 11:39:40 +0200 Subject: [PATCH 58/60] Mute FieldSortIT#testCastDate (#63972) Relates to #63719 --- .../java/org/elasticsearch/search/sort/FieldSortIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java index a3844acf0a8d0..53c0431f083bf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -1762,6 +1762,7 @@ public void testCastNumericType() throws Exception { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63719") public void testCastDate() throws Exception { assertAcked(prepareCreate("index_date") .setMapping("field", "type=date")); From 0482f5cb8f89c067293766f584c9ed5f20767ffa Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 21 Oct 2020 12:09:41 +0100 Subject: [PATCH 59/60] Only subject data streams to allow_auto_create, not auto_create_index --- .../indices/create/AutoCreateAction.java | 51 ++++++++++--------- .../indices/create/AutoCreateActionTests.java | 19 ++++--- .../datastreams/AutoCreateDataStreamIT.java | 50 ++++++++---------- 3 files changed, 58 insertions(+), 62 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index 00966f007c125..46334a1ff8155 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -30,9 +30,8 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; @@ -41,6 +40,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -116,25 +116,36 @@ protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { @Override public ClusterState execute(ClusterState currentState) throws Exception { - final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); - - // This will throw an exception if the index or data stream does not exist and creating it is prohibited. - final boolean shouldAutoCreate = autoCreateIndex.shouldAutoCreate(indexName, currentState); + final ComposableIndexTemplate template = resolveTemplate(request, currentState.metadata()); - if (shouldAutoCreate == false) { - // The index or data stream already exists. - return currentState; - } + if (template != null && template.getDataStreamTemplate() != null) { + // This expression only evaluates to true when the argument is non-null and false + if (Boolean.FALSE.equals(template.getAllowAutoCreate())) { + throw new IndexNotFoundException( + "composable template " + template.indexPatterns() + " forbids index auto creation" + ); + } - final DataStreamTemplate dataStreamTemplate = resolveAutoCreateDataStream(request, currentState.metadata()); - if (dataStreamTemplate != null) { CreateDataStreamClusterStateUpdateRequest createRequest = new CreateDataStreamClusterStateUpdateRequest( - request.index(), request.masterNodeTimeout(), request.timeout()); - ClusterState clusterState = metadataCreateDataStreamService.createDataStream(createRequest, currentState); + request.index(), + request.masterNodeTimeout(), + request.timeout() + ); + ClusterState clusterState = metadataCreateDataStreamService.createDataStream(createRequest, currentState); indexNameRef.set(clusterState.metadata().dataStreams().get(request.index()).getIndices().get(0).getName()); return clusterState; } else { + String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); indexNameRef.set(indexName); + + // This will throw an exception if the index does not exist and creating it is prohibited + final boolean shouldAutoCreate = autoCreateIndex.shouldAutoCreate(indexName, currentState); + + if (shouldAutoCreate == false) { + // The index already exists. + return currentState; + } + CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest(request.cause(), indexName, request.index()) .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()); @@ -150,16 +161,8 @@ protected ClusterBlockException checkBlock(CreateIndexRequest request, ClusterSt } } - static DataStreamTemplate resolveAutoCreateDataStream(CreateIndexRequest request, Metadata metadata) { + static ComposableIndexTemplate resolveTemplate(CreateIndexRequest request, Metadata metadata) { String v2Template = MetadataIndexTemplateService.findV2Template(metadata, request.index(), false); - if (v2Template != null) { - ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(v2Template); - if (composableIndexTemplate.getDataStreamTemplate() != null) { - return composableIndexTemplate.getDataStreamTemplate(); - } - } - - return null; + return v2Template != null ? metadata.templatesV2().get(v2Template) : null; } - } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java index 922ccc5df27c2..24cb4c37fb79c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java @@ -32,7 +32,7 @@ public class AutoCreateActionTests extends ESTestCase { - public void testResolveAutoCreateDataStreams() { + public void testResolveTemplates() { Metadata metadata; { Metadata.Builder mdBuilder = new Metadata.Builder(); @@ -44,23 +44,26 @@ public void testResolveAutoCreateDataStreams() { } CreateIndexRequest request = new CreateIndexRequest("logs-foobar"); - DataStreamTemplate result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + ComposableIndexTemplate result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, notNullValue()); - assertThat(result.getTimestampField(), equalTo("@timestamp")); + assertThat(result.getDataStreamTemplate(), notNullValue()); + assertThat(result.getDataStreamTemplate().getTimestampField(), equalTo("@timestamp")); request = new CreateIndexRequest("logs-barbaz"); - result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, notNullValue()); - assertThat(result.getTimestampField(), equalTo("@timestamp")); + assertThat(result.getDataStreamTemplate(), notNullValue()); + assertThat(result.getDataStreamTemplate().getTimestampField(), equalTo("@timestamp")); // An index that matches with a template without a data steam definition request = new CreateIndexRequest("legacy-logs-foobaz"); - result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); - assertThat(result, nullValue()); + result = AutoCreateAction.resolveTemplate(request, metadata); + assertThat(result, notNullValue()); + assertThat(result.getDataStreamTemplate(), nullValue()); // An index that doesn't match with an index template request = new CreateIndexRequest("my-index"); - result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, nullValue()); } diff --git a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java index 178d8d959f3e6..83dbf56a0a26b 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java @@ -40,30 +40,19 @@ public class AutoCreateDataStreamIT extends ESRestTestCase { /** * Check that setting {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} to false - * disable the automatic creation on data streams. + * does not affect the ability to auto-create data streams, which are not subject to the setting. */ - public void testCannotAutoCreateDataStreamWhenDisabled() throws IOException { + public void testCanAutoCreateDataStreamWhenAutoCreateIndexDisabled() throws IOException { configureAutoCreateIndex(false); - - // Attempt to add a document to a non-existing data stream. Auto-creating the data stream should fail owing to the setting above. - final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456"); - indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }"); - final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); - - assertThat( - Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), - containsString("no such index [recipe_kr] and [action.auto_create_index] is [false]") - ); + createTemplateWithAllowAutoCreate(null); + assertOK(this.indexDocument()); } /** - * Check that automatically creating an index is allowed, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} - * is false, when the index name matches a template and that template has allow_auto_create - * set to true. + * Check that automatically creating a data stream is allowed when the index name matches a template + * and that template has allow_auto_create set to true. */ - public void testCanAutoCreateDataStreamWhenAllowedByTemplate() throws IOException { - configureAutoCreateIndex(false); - + public void testCanAutoCreateDataStreamWhenExplicitlyAllowedByTemplate() throws IOException { createTemplateWithAllowAutoCreate(true); // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name @@ -73,12 +62,9 @@ public void testCanAutoCreateDataStreamWhenAllowedByTemplate() throws IOExceptio /** * Check that automatically creating a data stream is disallowed when the data stream name matches a template and that template has - * allow_auto_create explicitly to false, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} - * is set to true. + * allow_auto_create explicitly to false. */ public void testCannotAutoCreateDataStreamWhenDisallowedByTemplate() throws IOException { - configureAutoCreateIndex(true); - createTemplateWithAllowAutoCreate(false); // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name @@ -106,16 +92,20 @@ private void configureAutoCreateIndex(boolean value) throws IOException { } private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws IOException { - XContentBuilder builder = JsonXContent.contentBuilder() - .startObject() - .array("index_patterns", "recipe*") - .field("allow_auto_create", allowAutoCreate) - .startObject("data_stream") - .endObject() - .endObject(); + XContentBuilder b = JsonXContent.contentBuilder(); + b.startObject(); + { + b.array("index_patterns", "recipe*"); + if (allowAutoCreate != null) { + b.field("allow_auto_create", allowAutoCreate); + } + b.startObject("data_stream"); + b.endObject(); + } + b.endObject(); final Request createTemplateRequest = new Request("PUT", "_index_template/recipe_template"); - createTemplateRequest.setJsonEntity(Strings.toString(builder)); + createTemplateRequest.setJsonEntity(Strings.toString(b)); final Response createTemplateResponse = client().performRequest(createTemplateRequest); assertOK(createTemplateResponse); } From 6bd545661d510b7846627124e495ceb6ce65dbbd Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 22 Oct 2020 09:29:59 +0100 Subject: [PATCH 60/60] Tweak docs Mention data streams in the docs changes. Co-authored-by: Gordon Brown --- docs/reference/indices/put-component-template.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index cddc376394239..e58d370f60f29 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -118,7 +118,7 @@ This setting overrides the value of the <> cluster setting. If set to `true` in a template, then indices can be automatically created using that template even if auto-creation of indices is disabled via -`actions.auto_create_index`. If set to `false`, then indices matching the +`actions.auto_create_index`. If set to `false`, then indices or data streams matching the template must always be explicitly created, and may never be automatically created.