From cb1297bb701db1af1459fc770deda3d5ba7ac2cb Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 15 Jan 2020 11:55:14 +0100 Subject: [PATCH 1/2] Add analysis components and mapping types to the usage API. Knowing about used analysis components and mapping types would be incredibly useful in order to know which ones may be deprecated or should get more love. Some field types also act as a proxy to know about feature usage of some APIs like the `percolator` or `completion` fields types for percolation and the completion suggester, respectively. --- .../xpack/core/XPackClientPlugin.java | 4 +- .../elasticsearch/xpack/core/XPackField.java | 2 + .../elasticsearch/xpack/core/XPackPlugin.java | 3 + .../core/action/XPackUsageFeatureAction.java | 3 +- .../xpack/oss/IndexFeatureSetUsage.java | 131 ++++++++++++++++++ .../xpack/oss/IndexUsageTransportAction.java | 122 ++++++++++++++++ .../elasticsearch/xpack/oss/package-info.java | 10 ++ .../xpack/oss/IndexFeatureSetUsageTests.java | 98 +++++++++++++ .../oss/IndexUsageTransportActionTests.java | 95 +++++++++++++ .../rest-api-spec/test/xpack/15_basic.yml | 86 ++++++++++++ 10 files changed, 552 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/package-info.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsageTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexUsageTransportActionTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 82daf97cb0b54..3c7a0c7615945 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -236,6 +236,7 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction; import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction; import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction; +import org.elasticsearch.xpack.oss.IndexFeatureSetUsage; import java.util.ArrayList; import java.util.Arrays; @@ -561,7 +562,8 @@ public List getNamedWriteables() { // Spatial new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new), // data science - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new) + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new), + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX, IndexFeatureSetUsage::new) ).stream(), MlEvaluationNamedXContentProvider.getNamedWriteables().stream() ).collect(toList()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 8a74272429f87..3a836931b45a3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -53,6 +53,8 @@ public final class XPackField { public static final String ANALYTICS = "analytics"; /** Name constant for the enrich plugin. */ public static final String ENRICH = "enrich"; + /** Name constant for indices. */ + public static final String INDEX = "index"; private XPackField() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 9a2d5fd65eb6d..7a0307308b95a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -59,6 +59,7 @@ import org.elasticsearch.xpack.core.action.TransportXPackUsageAction; import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageResponse; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.rest.action.RestReloadAnalyzersAction; @@ -68,6 +69,7 @@ import org.elasticsearch.xpack.core.ssl.SSLConfigurationReloader; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.core.watcher.WatcherMetaData; +import org.elasticsearch.xpack.oss.IndexUsageTransportAction; import java.nio.file.Files; import java.nio.file.Path; @@ -255,6 +257,7 @@ public Collection createComponents(Client client, ClusterService cluster actions.add(new ActionHandler<>(XPackUsageAction.INSTANCE, getUsageAction())); actions.addAll(licensing.getActions()); actions.add(new ActionHandler<>(ReloadAnalyzerAction.INSTANCE, TransportReloadAnalyzersAction.class)); + actions.add(new ActionHandler<>(XPackUsageFeatureAction.INDEX, IndexUsageTransportAction.class)); return actions; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index fe43f9661488a..15e18ef38a4f9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -40,10 +40,11 @@ public class XPackUsageFeatureAction extends ActionType ALL = Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, - TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS + TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, INDEX ); private XPackUsageFeatureAction(String name) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java new file mode 100644 index 0000000000000..c4166ef4fd86f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.oss; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.Collections; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +public class IndexFeatureSetUsage extends XPackFeatureSet.Usage { + + private static Set sort(Set set) { + return Collections.unmodifiableSet(new TreeSet<>(set)); + } + + private final Set usedFieldTypes, usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers; + + public IndexFeatureSetUsage(Set usedFieldTypes, Set usedCharFilters, Set usedTokenizers, + Set usedTokenFilters, Set usedAnalyzers) { + super(XPackField.INDEX, true, true); + this.usedFieldTypes = sort(usedFieldTypes); + this.usedCharFilters = sort(usedCharFilters); + this.usedTokenizers = sort(usedTokenizers); + this.usedTokenFilters = sort(usedTokenFilters); + this.usedAnalyzers = sort(usedAnalyzers); + } + + public IndexFeatureSetUsage(StreamInput input) throws IOException { + super(input); + usedFieldTypes = input.readSet(StreamInput::readString); + usedCharFilters = input.readSet(StreamInput::readString); + usedTokenizers = input.readSet(StreamInput::readString); + usedTokenFilters = input.readSet(StreamInput::readString); + usedAnalyzers = input.readSet(StreamInput::readString); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeCollection(usedFieldTypes, StreamOutput::writeString); + out.writeCollection(usedCharFilters, StreamOutput::writeString); + out.writeCollection(usedTokenizers, StreamOutput::writeString); + out.writeCollection(usedTokenFilters, StreamOutput::writeString); + out.writeCollection(usedAnalyzers, StreamOutput::writeString); + } + + /** + * Return the set of used field types in the cluster. + */ + public Set getUsedFieldTypes() { + return usedFieldTypes; + } + + /** + * Return the set of used char filters in the cluster. + */ + public Set getUsedCharFilterTypes() { + return usedCharFilters; + } + + /** + * Return the set of used tokenizers in the cluster. + */ + public Set getUsedTokenizerTypes() { + return usedTokenizers; + } + + /** + * Return the set of used token filters in the cluster. + */ + public Set getUsedTokenFilterTypes() { + return usedTokenFilters; + } + + /** + * Return the set of used analyzers in the cluster. + */ + public Set getUsedAnalyzerTypes() { + return usedAnalyzers; + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + + builder.startObject("analysis"); + { + builder.field("char_filter_types", usedCharFilters); + builder.field("tokenizer_types", usedTokenizers); + builder.field("filter_types", usedTokenFilters); + builder.field("analyzer_types", usedAnalyzers); + } + builder.endObject(); + + builder.startObject("mappings"); + { + builder.field("field_types", usedFieldTypes); + } + builder.endObject(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IndexFeatureSetUsage that = (IndexFeatureSetUsage) o; + return available == that.available && enabled == that.enabled && + Objects.equals(usedFieldTypes, that.usedFieldTypes) && + Objects.equals(usedCharFilters, that.usedCharFilters) && + Objects.equals(usedTokenizers, that.usedTokenizers) && + Objects.equals(usedTokenFilters, that.usedTokenFilters) && + Objects.equals(usedAnalyzers, that.usedAnalyzers); + } + + @Override + public int hashCode() { + return Objects.hash(available, enabled, usedFieldTypes, usedCharFilters, usedTokenizers, usedTokenFilters, + usedAnalyzers); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java new file mode 100644 index 0000000000000..273c1d268fbb8 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.oss; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class IndexUsageTransportAction extends XPackUsageFeatureTransportAction { + + @Inject + public IndexUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(XPackUsageFeatureAction.INDEX.name(), transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver); + } + + @Override + protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state, + ActionListener listener) { + + final Set usedFieldTypes = new HashSet<>(); + final Set usedCharFilters = new HashSet<>(); + final Set usedTokenizers = new HashSet<>(); + final Set usedTokenFilters = new HashSet<>(); + final Set usedAnalyzers = new HashSet<>(); + + for (IndexMetaData indexMetaData : state.metaData()) { + MappingMetaData mappingMetaData = indexMetaData.mapping(); + if (mappingMetaData != null) { + populateFieldTypesFromObject(mappingMetaData.sourceAsMap(), usedFieldTypes); + } + + Settings indexSettings = indexMetaData.getSettings(); + + Map charFilterSettings = indexSettings.getGroups("index.analysis.char_filter"); + aggregateAnalysisTypes(charFilterSettings.values(), usedCharFilters); + + Map tokenizerSettings = indexSettings.getGroups("index.analysis.tokenizer"); + aggregateAnalysisTypes(tokenizerSettings.values(), usedTokenizers); + + Map tokenFilterSettings = indexSettings.getGroups("index.analysis.filter"); + aggregateAnalysisTypes(tokenFilterSettings.values(), usedTokenFilters); + + Map analyzerSettings = indexSettings.getGroups("index.analysis.analyzer"); + aggregateAnalysisTypes(analyzerSettings.values(), usedAnalyzers); + } + + listener.onResponse(new XPackUsageFeatureResponse( + new IndexFeatureSetUsage(usedFieldTypes, usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers))); + } + + static void populateFieldTypesFromObject(Map mapping, Set fieldTypes) { + Object properties = mapping.get("properties"); + if (properties != null && properties instanceof Map) { + @SuppressWarnings("unchecked") + Map propertiesAsMap = (Map) properties; + for (Object v : propertiesAsMap.values()) { + if (v != null && v instanceof Map) { + + @SuppressWarnings("unchecked") + Map fieldMapping = (Map) v; + populateFieldTypesFromField(fieldMapping, fieldTypes); + populateFieldTypesFromObject(fieldMapping, fieldTypes); + + // Multi fields + Object fieldsO = fieldMapping.get("fields"); + if (fieldsO != null && fieldsO instanceof Map) { + @SuppressWarnings("unchecked") + Map fields = (Map) fieldsO; + for (Object v2 : fields.values()) { + if (v2 instanceof Map) { + Map fieldMapping2 = (Map) v2; + populateFieldTypesFromField(fieldMapping2, fieldTypes); + } + } + } + } + } + } + } + + private static void populateFieldTypesFromField(Map mapping, Set fieldTypes) { + Object fieldType = mapping.get("type"); + if (fieldType != null) { + fieldTypes.add(fieldType.toString()); + } else if (mapping.containsKey("properties")) { + fieldTypes.add("object"); + } + } + + static void aggregateAnalysisTypes(Collection analysisComponents, Set usedTypes) { + for (Settings settings : analysisComponents) { + String type = settings.get("type"); + if (type != null) { + usedTypes.add(type); + } + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/package-info.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/package-info.java new file mode 100644 index 0000000000000..56582e0746737 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/package-info.java @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/** + * Package containing usage information for features that are exposed in OSS. + */ +package org.elasticsearch.xpack.oss; \ No newline at end of file diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsageTests.java new file mode 100644 index 0000000000000..be80280fd8b21 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsageTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.oss; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +public class IndexFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + @Override + protected Reader instanceReader() { + return IndexFeatureSetUsage::new; + } + + @Override + protected IndexFeatureSetUsage createTestInstance() { + Set fields = new HashSet<>(); + if (randomBoolean()) { + fields.add("keyword"); + } + if (randomBoolean()) { + fields.add("integer"); + } + + Set charFilters = new HashSet<>(); + if (randomBoolean()) { + charFilters.add("pattern_replace"); + } + + Set tokenizers = new HashSet<>(); + if (randomBoolean()) { + tokenizers.add("whitespace"); + } + + Set tokenFilters = new HashSet<>(); + if (randomBoolean()) { + tokenFilters.add("stop"); + } + + Set analyzers = new HashSet<>(); + if (randomBoolean()) { + tokenFilters.add("english"); + } + + return new IndexFeatureSetUsage(fields, charFilters, tokenizers, tokenFilters, analyzers); + } + + @Override + protected IndexFeatureSetUsage mutateInstance(IndexFeatureSetUsage instance) throws IOException { + switch (randomInt(4)) { + case 0: + Set fields = new HashSet<>(instance.getUsedFieldTypes()); + if (fields.add("keyword") == false) { + fields.remove("keyword"); + } + return new IndexFeatureSetUsage(fields, instance.getUsedCharFilterTypes(), instance.getUsedTokenizerTypes(), + instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes()); + case 1: + Set charFilters = new HashSet<>(instance.getUsedCharFilterTypes()); + if (charFilters.add("pattern_replace") == false) { + charFilters.remove("pattern_replace"); + } + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), charFilters, instance.getUsedTokenizerTypes(), + instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes()); + case 2: + Set tokenizers = new HashSet<>(instance.getUsedTokenizerTypes()); + if (tokenizers.add("whitespace") == false) { + tokenizers.remove("whitespace"); + } + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), tokenizers, + instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes()); + case 3: + Set tokenFilters = new HashSet<>(instance.getUsedTokenFilterTypes()); + if (tokenFilters.add("stop") == false) { + tokenFilters.remove("stop"); + } + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), instance.getUsedTokenizerTypes(), + tokenFilters, instance.getUsedAnalyzerTypes()); + case 4: + Set analyzers = new HashSet<>(instance.getUsedAnalyzerTypes()); + if (analyzers.add("english") == false) { + analyzers.remove("english"); + } + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), instance.getUsedTokenizerTypes(), + instance.getUsedTokenFilterTypes(), analyzers); + default: + throw new AssertionError(); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexUsageTransportActionTests.java new file mode 100644 index 0000000000000..6d82dfb43e25d --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexUsageTransportActionTests.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.oss; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class IndexUsageTransportActionTests extends ESTestCase { + + public void testCountTopLevelFields() { + Map mapping = new HashMap<>(); + Set fields = new HashSet<>(); + IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + assertEquals(Collections.emptySet(), fields); + + Map properties = new HashMap<>(); + mapping.put("properties", properties); + + Map keywordField = new HashMap<>(); + keywordField.put("type", "keyword"); + properties.put("foo", keywordField); + IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + assertEquals(Collections.singleton("keyword"), fields); + + Map IndexField = new HashMap<>(); + IndexField.put("type", "integer"); + properties.put("bar", IndexField); + fields = new HashSet<>(); + IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + assertEquals(new HashSet<>(Arrays.asList("keyword", "integer")), fields); + + properties.put("baz", IndexField); + fields = new HashSet<>(); + IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + assertEquals(new HashSet<>(Arrays.asList("keyword", "integer")), fields); + } + + public void testCountMultiFields() { + Map keywordField = new HashMap<>(); + keywordField.put("type", "keyword"); + + Map textField = new HashMap<>(); + textField.put("type", "text"); + + Map fields = new HashMap<>(); + fields.put("keyword", keywordField); + textField.put("fields", fields); + + Map properties = new HashMap<>(); + properties.put("foo", textField); + + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + + Set usedFields = new HashSet<>(); + IndexUsageTransportAction.populateFieldTypesFromObject(mapping, usedFields); + assertEquals(new HashSet<>(Arrays.asList("keyword", "text")), usedFields); + } + + public void testCountInnerFields() { + Map keywordField = new HashMap<>(); + keywordField.put("type", "keyword"); + + Map properties = new HashMap<>(); + properties.put("foo", keywordField); + + Map objectMapping = new HashMap<>(); + objectMapping.put("properties", properties); + + Map mapping = new HashMap<>(); + + properties = new HashMap<>(); + properties.put("obj", objectMapping); + mapping.put("properties", properties); + Set fields = new HashSet<>(); + IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + assertEquals(new HashSet<>(Arrays.asList("keyword", "object")), fields); + + properties.put("bar", keywordField); + fields = new HashSet<>(); + IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + assertEquals(new HashSet<>(Arrays.asList("keyword", "object")), fields); + } + +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml index d8b25a29531bc..2799121b0ded4 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml @@ -166,4 +166,90 @@ - is_true: features.monitoring.available - is_false: tagline +--- +"Usage stats for mappings": + - do: + xpack.usage: {} + + - match: { index.mappings.field_types: [] } + + - do: + indices.create: + index: test-index1 + body: + mappings: + properties: + foo: + type: keyword + + - do: + indices.create: + index: test-index2 + body: + mappings: + properties: + foo: + type: keyword + bar: + properties: + quux: + type: integer + + - do: + xpack.usage: {} + + - match: { index.mappings.field_types: [ "integer", "keyword", "object" ] } + +--- +"Usage stats for analysis": + - do: + xpack.usage: {} + + - match: { index.analysis.char_filter_types: [] } + - match: { index.analysis.tokenizer_types: [] } + - match: { index.analysis.filter_types: [] } + - match: { index.analysis.analyzer_types: [] } + + - do: + indices.create: + index: test-index1 + body: + settings: + analysis: + char_filter: + c: + type: mapping + mappings: [ "a => b" ] + tokenizer: + tok: + type: pattern + pattern: "," + filter: + st: + type: stop + stopwords: [ "a" ] + analyzer: + en: + type: standard + stopwords: "_english_" + + - do: + indices.create: + index: test-index2 + body: + mappings: + properties: + foo: + type: keyword + bar: + properties: + quux: + type: integer + + - do: + xpack.usage: {} + - match: { index.analysis.char_filter_types: [ "mapping" ] } + - match: { index.analysis.tokenizer_types: [ "pattern" ] } + - match: { index.analysis.filter_types: [ "stop" ] } + - match: { index.analysis.analyzer_types: [ "standard" ] } From 5c6278f621c8e808ac7c887c5cd3990dabd99c79 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 15 Jan 2020 16:18:21 +0100 Subject: [PATCH 2/2] Review --- .../xpack/oss/IndexFeatureSetUsage.java | 66 ++++++++++++-- .../xpack/oss/IndexUsageTransportAction.java | 61 +++++++++---- .../xpack/oss/IndexFeatureSetUsageTests.java | 88 +++++++++++++++++-- .../oss/IndexUsageTransportActionTests.java | 25 ++++-- .../rest-api-spec/test/xpack/15_basic.yml | 26 +++--- 5 files changed, 214 insertions(+), 52 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java index c4166ef4fd86f..cd779d09d52ac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java @@ -24,16 +24,24 @@ private static Set sort(Set set) { return Collections.unmodifiableSet(new TreeSet<>(set)); } - private final Set usedFieldTypes, usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers; - - public IndexFeatureSetUsage(Set usedFieldTypes, Set usedCharFilters, Set usedTokenizers, - Set usedTokenFilters, Set usedAnalyzers) { + private final Set usedFieldTypes; + private final Set usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers; + private final Set usedBuiltInCharFilters, usedBuiltInTokenizers, usedBuiltInTokenFilters, usedBuiltInAnalyzers; + + public IndexFeatureSetUsage(Set usedFieldTypes, + Set usedCharFilters, Set usedTokenizers, Set usedTokenFilters, Set usedAnalyzers, + Set usedBuiltInCharFilters, Set usedBuiltInTokenizers, Set usedBuiltInTokenFilters, + Set usedBuiltInAnalyzers) { super(XPackField.INDEX, true, true); this.usedFieldTypes = sort(usedFieldTypes); this.usedCharFilters = sort(usedCharFilters); this.usedTokenizers = sort(usedTokenizers); this.usedTokenFilters = sort(usedTokenFilters); this.usedAnalyzers = sort(usedAnalyzers); + this.usedBuiltInCharFilters = sort(usedBuiltInCharFilters); + this.usedBuiltInTokenizers = sort(usedBuiltInTokenizers); + this.usedBuiltInTokenFilters = sort(usedBuiltInTokenFilters); + this.usedBuiltInAnalyzers = sort(usedBuiltInAnalyzers); } public IndexFeatureSetUsage(StreamInput input) throws IOException { @@ -43,6 +51,10 @@ public IndexFeatureSetUsage(StreamInput input) throws IOException { usedTokenizers = input.readSet(StreamInput::readString); usedTokenFilters = input.readSet(StreamInput::readString); usedAnalyzers = input.readSet(StreamInput::readString); + usedBuiltInCharFilters = input.readSet(StreamInput::readString); + usedBuiltInTokenizers = input.readSet(StreamInput::readString); + usedBuiltInTokenFilters = input.readSet(StreamInput::readString); + usedBuiltInAnalyzers = input.readSet(StreamInput::readString); } @Override @@ -53,6 +65,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(usedTokenizers, StreamOutput::writeString); out.writeCollection(usedTokenFilters, StreamOutput::writeString); out.writeCollection(usedAnalyzers, StreamOutput::writeString); + out.writeCollection(usedBuiltInCharFilters, StreamOutput::writeString); + out.writeCollection(usedBuiltInTokenizers, StreamOutput::writeString); + out.writeCollection(usedBuiltInTokenFilters, StreamOutput::writeString); + out.writeCollection(usedBuiltInAnalyzers, StreamOutput::writeString); } /** @@ -90,6 +106,34 @@ public Set getUsedAnalyzerTypes() { return usedAnalyzers; } + /** + * Return the set of used built-in char filters in the cluster. + */ + public Set getUsedBuiltInCharFilters() { + return usedCharFilters; + } + + /** + * Return the set of used built-in tokenizers in the cluster. + */ + public Set getUsedBuiltInTokenizers() { + return usedTokenizers; + } + + /** + * Return the set of used built-in token filters in the cluster. + */ + public Set getUsedBuiltInTokenFilters() { + return usedTokenFilters; + } + + /** + * Return the set of used built-in analyzers in the cluster. + */ + public Set getUsedBuiltInAnalyzers() { + return usedAnalyzers; + } + @Override protected void innerXContent(XContentBuilder builder, Params params) throws IOException { super.innerXContent(builder, params); @@ -100,6 +144,11 @@ protected void innerXContent(XContentBuilder builder, Params params) throws IOEx builder.field("tokenizer_types", usedTokenizers); builder.field("filter_types", usedTokenFilters); builder.field("analyzer_types", usedAnalyzers); + + builder.field("built_in_char_filters", usedBuiltInCharFilters); + builder.field("built_in_tokenizers", usedBuiltInTokenizers); + builder.field("built_in_filters", usedBuiltInTokenFilters); + builder.field("built_in_analyzers", usedBuiltInAnalyzers); } builder.endObject(); @@ -120,12 +169,17 @@ public boolean equals(Object o) { Objects.equals(usedCharFilters, that.usedCharFilters) && Objects.equals(usedTokenizers, that.usedTokenizers) && Objects.equals(usedTokenFilters, that.usedTokenFilters) && - Objects.equals(usedAnalyzers, that.usedAnalyzers); + Objects.equals(usedAnalyzers, that.usedAnalyzers) && + Objects.equals(usedBuiltInCharFilters, that.usedBuiltInCharFilters) && + Objects.equals(usedBuiltInTokenizers, that.usedBuiltInTokenizers) && + Objects.equals(usedBuiltInTokenFilters, that.usedBuiltInTokenFilters) && + Objects.equals(usedBuiltInAnalyzers, that.usedBuiltInAnalyzers); } @Override public int hashCode() { return Objects.hash(available, enabled, usedFieldTypes, usedCharFilters, usedTokenizers, usedTokenFilters, - usedAnalyzers); + usedAnalyzers, usedBuiltInCharFilters, usedBuiltInTokenizers, usedBuiltInTokenFilters, + usedBuiltInAnalyzers); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java index 273c1d268fbb8..dd0e002c93e84 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java @@ -27,6 +27,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.function.Consumer; public class IndexUsageTransportAction extends XPackUsageFeatureTransportAction { @@ -46,33 +47,65 @@ protected void masterOperation(Task task, XPackUsageRequest request, ClusterStat final Set usedTokenizers = new HashSet<>(); final Set usedTokenFilters = new HashSet<>(); final Set usedAnalyzers = new HashSet<>(); + final Set usedBuiltInCharFilters = new HashSet<>(); + final Set usedBuiltInTokenizers = new HashSet<>(); + final Set usedBuiltInTokenFilters = new HashSet<>(); + final Set usedBuiltInAnalyzers = new HashSet<>(); for (IndexMetaData indexMetaData : state.metaData()) { MappingMetaData mappingMetaData = indexMetaData.mapping(); if (mappingMetaData != null) { - populateFieldTypesFromObject(mappingMetaData.sourceAsMap(), usedFieldTypes); + visitMapping(mappingMetaData.getSourceAsMap(), fieldMapping -> { + Object type = fieldMapping.get("type"); + if (type != null) { + usedFieldTypes.add(type.toString()); + } else if (fieldMapping.containsKey("properties")) { + usedFieldTypes.add("object"); + } + + for (String key : new String[] { "analyzer", "search_analyzer", "search_quote_analyzer" }) { + Object analyzer = fieldMapping.get(key); + if (analyzer != null) { + usedBuiltInAnalyzers.add(analyzer.toString()); + } + } + }); } Settings indexSettings = indexMetaData.getSettings(); + Map analyzerSettings = indexSettings.getGroups("index.analysis.analyzer"); + usedBuiltInAnalyzers.removeAll(analyzerSettings.keySet()); + for (Settings analyzerSetting : analyzerSettings.values()) { + usedAnalyzers.add(analyzerSetting.get("type", "custom")); + usedBuiltInCharFilters.addAll(analyzerSetting.getAsList("char_filter")); + String tokenizer = analyzerSetting.get("tokenizer"); + if (tokenizer != null) { + usedBuiltInTokenizers.add(tokenizer); + } + usedBuiltInTokenFilters.addAll(analyzerSetting.getAsList("filter")); + } + Map charFilterSettings = indexSettings.getGroups("index.analysis.char_filter"); + usedBuiltInCharFilters.removeAll(charFilterSettings.keySet()); aggregateAnalysisTypes(charFilterSettings.values(), usedCharFilters); Map tokenizerSettings = indexSettings.getGroups("index.analysis.tokenizer"); + usedBuiltInTokenizers.removeAll(tokenizerSettings.keySet()); aggregateAnalysisTypes(tokenizerSettings.values(), usedTokenizers); Map tokenFilterSettings = indexSettings.getGroups("index.analysis.filter"); + usedBuiltInTokenFilters.removeAll(tokenFilterSettings.keySet()); aggregateAnalysisTypes(tokenFilterSettings.values(), usedTokenFilters); - - Map analyzerSettings = indexSettings.getGroups("index.analysis.analyzer"); - aggregateAnalysisTypes(analyzerSettings.values(), usedAnalyzers); } listener.onResponse(new XPackUsageFeatureResponse( - new IndexFeatureSetUsage(usedFieldTypes, usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers))); + new IndexFeatureSetUsage(usedFieldTypes, + usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers, + usedBuiltInCharFilters, usedBuiltInTokenizers, usedBuiltInTokenFilters, usedBuiltInAnalyzers))); } - static void populateFieldTypesFromObject(Map mapping, Set fieldTypes) { + static void visitMapping(Map mapping, Consumer> fieldMappingConsumer) { Object properties = mapping.get("properties"); if (properties != null && properties instanceof Map) { @SuppressWarnings("unchecked") @@ -82,8 +115,8 @@ static void populateFieldTypesFromObject(Map mapping, Set fie @SuppressWarnings("unchecked") Map fieldMapping = (Map) v; - populateFieldTypesFromField(fieldMapping, fieldTypes); - populateFieldTypesFromObject(fieldMapping, fieldTypes); + fieldMappingConsumer.accept(fieldMapping); + visitMapping(fieldMapping, fieldMappingConsumer); // Multi fields Object fieldsO = fieldMapping.get("fields"); @@ -92,8 +125,9 @@ static void populateFieldTypesFromObject(Map mapping, Set fie Map fields = (Map) fieldsO; for (Object v2 : fields.values()) { if (v2 instanceof Map) { + @SuppressWarnings("unchecked") Map fieldMapping2 = (Map) v2; - populateFieldTypesFromField(fieldMapping2, fieldTypes); + fieldMappingConsumer.accept(fieldMapping2); } } } @@ -102,15 +136,6 @@ static void populateFieldTypesFromObject(Map mapping, Set fie } } - private static void populateFieldTypesFromField(Map mapping, Set fieldTypes) { - Object fieldType = mapping.get("type"); - if (fieldType != null) { - fieldTypes.add(fieldType.toString()); - } else if (mapping.containsKey("properties")) { - fieldTypes.add("object"); - } - } - static void aggregateAnalysisTypes(Collection analysisComponents, Set usedTypes) { for (Settings settings : analysisComponents) { String type = settings.get("type"); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsageTests.java index be80280fd8b21..078a17a7aa555 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsageTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsageTests.java @@ -50,47 +50,117 @@ protected IndexFeatureSetUsage createTestInstance() { tokenFilters.add("english"); } - return new IndexFeatureSetUsage(fields, charFilters, tokenizers, tokenFilters, analyzers); + Set builtInCharFilters = new HashSet<>(); + if (randomBoolean()) { + builtInCharFilters.add("html_strip"); + } + + Set builtInTokenizers = new HashSet<>(); + if (randomBoolean()) { + builtInTokenizers.add("keyword"); + } + + Set builtInTokenFilters = new HashSet<>(); + if (randomBoolean()) { + builtInTokenFilters.add("trim"); + } + + Set builtInAnalyzers = new HashSet<>(); + if (randomBoolean()) { + builtInAnalyzers.add("french"); + } + + return new IndexFeatureSetUsage(fields, + charFilters, tokenizers, tokenFilters, analyzers, + builtInCharFilters, builtInTokenizers, builtInTokenFilters, builtInAnalyzers); } @Override protected IndexFeatureSetUsage mutateInstance(IndexFeatureSetUsage instance) throws IOException { - switch (randomInt(4)) { + switch (randomInt(8)) { case 0: Set fields = new HashSet<>(instance.getUsedFieldTypes()); if (fields.add("keyword") == false) { fields.remove("keyword"); } return new IndexFeatureSetUsage(fields, instance.getUsedCharFilterTypes(), instance.getUsedTokenizerTypes(), - instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes()); + instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes(), instance.getUsedBuiltInCharFilters(), + instance.getUsedBuiltInTokenizers(), instance.getUsedBuiltInTokenFilters(), + instance.getUsedBuiltInAnalyzers()); case 1: Set charFilters = new HashSet<>(instance.getUsedCharFilterTypes()); if (charFilters.add("pattern_replace") == false) { charFilters.remove("pattern_replace"); } return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), charFilters, instance.getUsedTokenizerTypes(), - instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes()); + instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes(), instance.getUsedBuiltInCharFilters(), + instance.getUsedBuiltInTokenizers(), instance.getUsedBuiltInTokenFilters(), + instance.getUsedBuiltInAnalyzers()); case 2: Set tokenizers = new HashSet<>(instance.getUsedTokenizerTypes()); if (tokenizers.add("whitespace") == false) { tokenizers.remove("whitespace"); } return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), tokenizers, - instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes()); + instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes(), instance.getUsedBuiltInCharFilters(), + instance.getUsedBuiltInTokenizers(), instance.getUsedBuiltInTokenFilters(), + instance.getUsedBuiltInAnalyzers()); case 3: Set tokenFilters = new HashSet<>(instance.getUsedTokenFilterTypes()); if (tokenFilters.add("stop") == false) { tokenFilters.remove("stop"); } - return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), instance.getUsedTokenizerTypes(), - tokenFilters, instance.getUsedAnalyzerTypes()); + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), + instance.getUsedTokenizerTypes(), + tokenFilters, instance.getUsedAnalyzerTypes(), instance.getUsedBuiltInCharFilters(), + instance.getUsedBuiltInTokenizers(), instance.getUsedBuiltInTokenFilters(), + instance.getUsedBuiltInAnalyzers()); case 4: Set analyzers = new HashSet<>(instance.getUsedAnalyzerTypes()); if (analyzers.add("english") == false) { analyzers.remove("english"); } - return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), instance.getUsedTokenizerTypes(), - instance.getUsedTokenFilterTypes(), analyzers); + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), + instance.getUsedTokenizerTypes(), instance.getUsedTokenFilterTypes(), analyzers, + instance.getUsedBuiltInCharFilters(), instance.getUsedBuiltInTokenizers(), instance.getUsedBuiltInTokenFilters(), + instance.getUsedBuiltInAnalyzers()); + case 5: + Set builtInCharFilters = new HashSet<>(); + if (builtInCharFilters.add("html_strip") == false) { + builtInCharFilters.remove("html_strip"); + } + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), + instance.getUsedTokenizerTypes(), + instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes(), builtInCharFilters, + instance.getUsedBuiltInTokenizers(), instance.getUsedBuiltInTokenFilters(), + instance.getUsedBuiltInAnalyzers()); + case 6: + Set builtInTokenizers = new HashSet<>(); + if (builtInTokenizers.add("keyword") == false) { + builtInTokenizers.remove("keyword"); + } + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), + instance.getUsedTokenizerTypes(), instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes(), + instance.getUsedBuiltInCharFilters(), builtInTokenizers, instance.getUsedBuiltInTokenFilters(), + instance.getUsedBuiltInAnalyzers()); + case 7: + Set builtInTokenFilters = new HashSet<>(); + if (builtInTokenFilters.add("trim") == false) { + builtInTokenFilters.remove("trim"); + } + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), + instance.getUsedTokenizerTypes(), instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes(), + instance.getUsedBuiltInCharFilters(), instance.getUsedBuiltInTokenizers(), builtInTokenFilters, + instance.getUsedBuiltInAnalyzers()); + case 8: + Set builtInAnalyzers = new HashSet<>(); + if (builtInAnalyzers.add("french") == false) { + builtInAnalyzers.remove("french"); + } + return new IndexFeatureSetUsage(instance.getUsedFieldTypes(), instance.getUsedCharFilterTypes(), + instance.getUsedTokenizerTypes(), instance.getUsedTokenFilterTypes(), instance.getUsedAnalyzerTypes(), + instance.getUsedBuiltInCharFilters(), instance.getUsedBuiltInTokenizers(), instance.getUsedBuiltInTokenFilters(), + builtInAnalyzers); default: throw new AssertionError(); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexUsageTransportActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexUsageTransportActionTests.java index 6d82dfb43e25d..605e8cc0e1534 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexUsageTransportActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/oss/IndexUsageTransportActionTests.java @@ -17,10 +17,21 @@ public class IndexUsageTransportActionTests extends ESTestCase { + private static void collectTypes(Map mapping, Set types) { + IndexUsageTransportAction.visitMapping(mapping, + m -> { + if (m.containsKey("type")) { + types.add(m.get("type").toString()); + } else { + types.add("object"); + } + }); + } + public void testCountTopLevelFields() { Map mapping = new HashMap<>(); Set fields = new HashSet<>(); - IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + collectTypes(mapping, fields); assertEquals(Collections.emptySet(), fields); Map properties = new HashMap<>(); @@ -29,19 +40,19 @@ public void testCountTopLevelFields() { Map keywordField = new HashMap<>(); keywordField.put("type", "keyword"); properties.put("foo", keywordField); - IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + collectTypes(mapping, fields); assertEquals(Collections.singleton("keyword"), fields); Map IndexField = new HashMap<>(); IndexField.put("type", "integer"); properties.put("bar", IndexField); fields = new HashSet<>(); - IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + collectTypes(mapping, fields); assertEquals(new HashSet<>(Arrays.asList("keyword", "integer")), fields); properties.put("baz", IndexField); fields = new HashSet<>(); - IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + collectTypes(mapping, fields); assertEquals(new HashSet<>(Arrays.asList("keyword", "integer")), fields); } @@ -63,7 +74,7 @@ public void testCountMultiFields() { mapping.put("properties", properties); Set usedFields = new HashSet<>(); - IndexUsageTransportAction.populateFieldTypesFromObject(mapping, usedFields); + collectTypes(mapping, usedFields); assertEquals(new HashSet<>(Arrays.asList("keyword", "text")), usedFields); } @@ -83,12 +94,12 @@ public void testCountInnerFields() { properties.put("obj", objectMapping); mapping.put("properties", properties); Set fields = new HashSet<>(); - IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + collectTypes(mapping, fields); assertEquals(new HashSet<>(Arrays.asList("keyword", "object")), fields); properties.put("bar", keywordField); fields = new HashSet<>(); - IndexUsageTransportAction.populateFieldTypesFromObject(mapping, fields); + collectTypes(mapping, fields); assertEquals(new HashSet<>(Arrays.asList("keyword", "object")), fields); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml index 2799121b0ded4..4757ee00360e9 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml @@ -232,19 +232,17 @@ en: type: standard stopwords: "_english_" - - - do: - indices.create: - index: test-index2 - body: + cust: + char_filter: [ "html_strip" ] + tokenizer: "keyword" + filter: [ "trim" ] mappings: properties: - foo: - type: keyword - bar: - properties: - quux: - type: integer + message: + type: "text" + analyzer: french + search_analyzer: spanish + search_quote_analyzer: german - do: xpack.usage: {} @@ -252,4 +250,8 @@ - match: { index.analysis.char_filter_types: [ "mapping" ] } - match: { index.analysis.tokenizer_types: [ "pattern" ] } - match: { index.analysis.filter_types: [ "stop" ] } - - match: { index.analysis.analyzer_types: [ "standard" ] } + - match: { index.analysis.analyzer_types: [ "custom", "standard" ] } + - match: { index.analysis.built_in_char_filters: [ "html_strip" ] } + - match: { index.analysis.built_in_tokenizers: [ "keyword" ] } + - match: { index.analysis.built_in_filters: [ "trim" ] } + - match: { index.analysis.built_in_analyzers: [ "french", "german", "spanish" ] }