Skip to content

Commit af59ad7

Browse files
jpountzjpountz
authored
Add analysis components and mapping types to the usage API. (#51031)
Knowing about used analysis components and mapping types would be incredibly useful in order to know which ones may be deprecated or should get more love. Some field types also act as a proxy to know about feature usage of some APIs like the `percolator` or `completion` fields types for percolation and the completion suggester, respectively.
1 parent da80e32 commit af59ad7

File tree

10 files changed

+714
-2
lines changed

10 files changed

+714
-2
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,7 @@
236236
import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
237237
import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction;
238238
import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
239+
import org.elasticsearch.xpack.oss.IndexFeatureSetUsage;
239240

240241
import java.util.ArrayList;
241242
import java.util.Arrays;
@@ -561,7 +562,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
561562
// Spatial
562563
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new),
563564
// data science
564-
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new)
565+
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new),
566+
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX, IndexFeatureSetUsage::new)
565567
).stream(),
566568
MlEvaluationNamedXContentProvider.getNamedWriteables().stream()
567569
).collect(toList());

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

+2
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ public final class XPackField {
5353
public static final String ANALYTICS = "analytics";
5454
/** Name constant for the enrich plugin. */
5555
public static final String ENRICH = "enrich";
56+
/** Name constant for indices. */
57+
public static final String INDEX = "index";
5658

5759
private XPackField() {}
5860

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java

+3
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
6060
import org.elasticsearch.xpack.core.action.XPackInfoAction;
6161
import org.elasticsearch.xpack.core.action.XPackUsageAction;
62+
import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
6263
import org.elasticsearch.xpack.core.action.XPackUsageResponse;
6364
import org.elasticsearch.xpack.core.ml.MlMetadata;
6465
import org.elasticsearch.xpack.core.rest.action.RestReloadAnalyzersAction;
@@ -68,6 +69,7 @@
6869
import org.elasticsearch.xpack.core.ssl.SSLConfigurationReloader;
6970
import org.elasticsearch.xpack.core.ssl.SSLService;
7071
import org.elasticsearch.xpack.core.watcher.WatcherMetaData;
72+
import org.elasticsearch.xpack.oss.IndexUsageTransportAction;
7173

7274
import java.nio.file.Files;
7375
import java.nio.file.Path;
@@ -255,6 +257,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
255257
actions.add(new ActionHandler<>(XPackUsageAction.INSTANCE, getUsageAction()));
256258
actions.addAll(licensing.getActions());
257259
actions.add(new ActionHandler<>(ReloadAnalyzerAction.INSTANCE, TransportReloadAnalyzersAction.class));
260+
actions.add(new ActionHandler<>(XPackUsageFeatureAction.INDEX, IndexUsageTransportAction.class));
258261
return actions;
259262
}
260263

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,11 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
4040
public static final XPackUsageFeatureAction FROZEN_INDICES = new XPackUsageFeatureAction(XPackField.FROZEN_INDICES);
4141
public static final XPackUsageFeatureAction SPATIAL = new XPackUsageFeatureAction(XPackField.SPATIAL);
4242
public static final XPackUsageFeatureAction ANALYTICS = new XPackUsageFeatureAction(XPackField.ANALYTICS);
43+
public static final XPackUsageFeatureAction INDEX = new XPackUsageFeatureAction(XPackField.INDEX);
4344

4445
public static final List<XPackUsageFeatureAction> ALL = Arrays.asList(
4546
SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
46-
TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS
47+
TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, INDEX
4748
);
4849

4950
private XPackUsageFeatureAction(String name) {

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexFeatureSetUsage.java

+185
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.oss;
8+
9+
import org.elasticsearch.common.io.stream.StreamInput;
10+
import org.elasticsearch.common.io.stream.StreamOutput;
11+
import org.elasticsearch.common.xcontent.XContentBuilder;
12+
import org.elasticsearch.xpack.core.XPackFeatureSet;
13+
import org.elasticsearch.xpack.core.XPackField;
14+
15+
import java.io.IOException;
16+
import java.util.Collections;
17+
import java.util.Objects;
18+
import java.util.Set;
19+
import java.util.TreeSet;
20+
21+
public class IndexFeatureSetUsage extends XPackFeatureSet.Usage {
22+
23+
private static Set<String> sort(Set<String> set) {
24+
return Collections.unmodifiableSet(new TreeSet<>(set));
25+
}
26+
27+
private final Set<String> usedFieldTypes;
28+
private final Set<String> usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers;
29+
private final Set<String> usedBuiltInCharFilters, usedBuiltInTokenizers, usedBuiltInTokenFilters, usedBuiltInAnalyzers;
30+
31+
public IndexFeatureSetUsage(Set<String> usedFieldTypes,
32+
Set<String> usedCharFilters, Set<String> usedTokenizers, Set<String> usedTokenFilters, Set<String> usedAnalyzers,
33+
Set<String> usedBuiltInCharFilters, Set<String> usedBuiltInTokenizers, Set<String> usedBuiltInTokenFilters,
34+
Set<String> usedBuiltInAnalyzers) {
35+
super(XPackField.INDEX, true, true);
36+
this.usedFieldTypes = sort(usedFieldTypes);
37+
this.usedCharFilters = sort(usedCharFilters);
38+
this.usedTokenizers = sort(usedTokenizers);
39+
this.usedTokenFilters = sort(usedTokenFilters);
40+
this.usedAnalyzers = sort(usedAnalyzers);
41+
this.usedBuiltInCharFilters = sort(usedBuiltInCharFilters);
42+
this.usedBuiltInTokenizers = sort(usedBuiltInTokenizers);
43+
this.usedBuiltInTokenFilters = sort(usedBuiltInTokenFilters);
44+
this.usedBuiltInAnalyzers = sort(usedBuiltInAnalyzers);
45+
}
46+
47+
public IndexFeatureSetUsage(StreamInput input) throws IOException {
48+
super(input);
49+
usedFieldTypes = input.readSet(StreamInput::readString);
50+
usedCharFilters = input.readSet(StreamInput::readString);
51+
usedTokenizers = input.readSet(StreamInput::readString);
52+
usedTokenFilters = input.readSet(StreamInput::readString);
53+
usedAnalyzers = input.readSet(StreamInput::readString);
54+
usedBuiltInCharFilters = input.readSet(StreamInput::readString);
55+
usedBuiltInTokenizers = input.readSet(StreamInput::readString);
56+
usedBuiltInTokenFilters = input.readSet(StreamInput::readString);
57+
usedBuiltInAnalyzers = input.readSet(StreamInput::readString);
58+
}
59+
60+
@Override
61+
public void writeTo(StreamOutput out) throws IOException {
62+
super.writeTo(out);
63+
out.writeCollection(usedFieldTypes, StreamOutput::writeString);
64+
out.writeCollection(usedCharFilters, StreamOutput::writeString);
65+
out.writeCollection(usedTokenizers, StreamOutput::writeString);
66+
out.writeCollection(usedTokenFilters, StreamOutput::writeString);
67+
out.writeCollection(usedAnalyzers, StreamOutput::writeString);
68+
out.writeCollection(usedBuiltInCharFilters, StreamOutput::writeString);
69+
out.writeCollection(usedBuiltInTokenizers, StreamOutput::writeString);
70+
out.writeCollection(usedBuiltInTokenFilters, StreamOutput::writeString);
71+
out.writeCollection(usedBuiltInAnalyzers, StreamOutput::writeString);
72+
}
73+
74+
/**
75+
* Return the set of used field types in the cluster.
76+
*/
77+
public Set<String> getUsedFieldTypes() {
78+
return usedFieldTypes;
79+
}
80+
81+
/**
82+
* Return the set of used char filters in the cluster.
83+
*/
84+
public Set<String> getUsedCharFilterTypes() {
85+
return usedCharFilters;
86+
}
87+
88+
/**
89+
* Return the set of used tokenizers in the cluster.
90+
*/
91+
public Set<String> getUsedTokenizerTypes() {
92+
return usedTokenizers;
93+
}
94+
95+
/**
96+
* Return the set of used token filters in the cluster.
97+
*/
98+
public Set<String> getUsedTokenFilterTypes() {
99+
return usedTokenFilters;
100+
}
101+
102+
/**
103+
* Return the set of used analyzers in the cluster.
104+
*/
105+
public Set<String> getUsedAnalyzerTypes() {
106+
return usedAnalyzers;
107+
}
108+
109+
/**
110+
* Return the set of used built-in char filters in the cluster.
111+
*/
112+
public Set<String> getUsedBuiltInCharFilters() {
113+
return usedCharFilters;
114+
}
115+
116+
/**
117+
* Return the set of used built-in tokenizers in the cluster.
118+
*/
119+
public Set<String> getUsedBuiltInTokenizers() {
120+
return usedTokenizers;
121+
}
122+
123+
/**
124+
* Return the set of used built-in token filters in the cluster.
125+
*/
126+
public Set<String> getUsedBuiltInTokenFilters() {
127+
return usedTokenFilters;
128+
}
129+
130+
/**
131+
* Return the set of used built-in analyzers in the cluster.
132+
*/
133+
public Set<String> getUsedBuiltInAnalyzers() {
134+
return usedAnalyzers;
135+
}
136+
137+
@Override
138+
protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
139+
super.innerXContent(builder, params);
140+
141+
builder.startObject("analysis");
142+
{
143+
builder.field("char_filter_types", usedCharFilters);
144+
builder.field("tokenizer_types", usedTokenizers);
145+
builder.field("filter_types", usedTokenFilters);
146+
builder.field("analyzer_types", usedAnalyzers);
147+
148+
builder.field("built_in_char_filters", usedBuiltInCharFilters);
149+
builder.field("built_in_tokenizers", usedBuiltInTokenizers);
150+
builder.field("built_in_filters", usedBuiltInTokenFilters);
151+
builder.field("built_in_analyzers", usedBuiltInAnalyzers);
152+
}
153+
builder.endObject();
154+
155+
builder.startObject("mappings");
156+
{
157+
builder.field("field_types", usedFieldTypes);
158+
}
159+
builder.endObject();
160+
}
161+
162+
@Override
163+
public boolean equals(Object o) {
164+
if (this == o) return true;
165+
if (o == null || getClass() != o.getClass()) return false;
166+
IndexFeatureSetUsage that = (IndexFeatureSetUsage) o;
167+
return available == that.available && enabled == that.enabled &&
168+
Objects.equals(usedFieldTypes, that.usedFieldTypes) &&
169+
Objects.equals(usedCharFilters, that.usedCharFilters) &&
170+
Objects.equals(usedTokenizers, that.usedTokenizers) &&
171+
Objects.equals(usedTokenFilters, that.usedTokenFilters) &&
172+
Objects.equals(usedAnalyzers, that.usedAnalyzers) &&
173+
Objects.equals(usedBuiltInCharFilters, that.usedBuiltInCharFilters) &&
174+
Objects.equals(usedBuiltInTokenizers, that.usedBuiltInTokenizers) &&
175+
Objects.equals(usedBuiltInTokenFilters, that.usedBuiltInTokenFilters) &&
176+
Objects.equals(usedBuiltInAnalyzers, that.usedBuiltInAnalyzers);
177+
}
178+
179+
@Override
180+
public int hashCode() {
181+
return Objects.hash(available, enabled, usedFieldTypes, usedCharFilters, usedTokenizers, usedTokenFilters,
182+
usedAnalyzers, usedBuiltInCharFilters, usedBuiltInTokenizers, usedBuiltInTokenFilters,
183+
usedBuiltInAnalyzers);
184+
}
185+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/oss/IndexUsageTransportAction.java

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.oss;
8+
9+
import org.elasticsearch.action.ActionListener;
10+
import org.elasticsearch.action.support.ActionFilters;
11+
import org.elasticsearch.cluster.ClusterState;
12+
import org.elasticsearch.cluster.metadata.IndexMetaData;
13+
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
14+
import org.elasticsearch.cluster.metadata.MappingMetaData;
15+
import org.elasticsearch.cluster.service.ClusterService;
16+
import org.elasticsearch.common.inject.Inject;
17+
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.protocol.xpack.XPackUsageRequest;
19+
import org.elasticsearch.tasks.Task;
20+
import org.elasticsearch.threadpool.ThreadPool;
21+
import org.elasticsearch.transport.TransportService;
22+
import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
23+
import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
24+
import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
25+
26+
import java.util.Collection;
27+
import java.util.HashSet;
28+
import java.util.Map;
29+
import java.util.Set;
30+
import java.util.function.Consumer;
31+
32+
public class IndexUsageTransportAction extends XPackUsageFeatureTransportAction {
33+
34+
@Inject
35+
public IndexUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
36+
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
37+
super(XPackUsageFeatureAction.INDEX.name(), transportService, clusterService, threadPool, actionFilters,
38+
indexNameExpressionResolver);
39+
}
40+
41+
@Override
42+
protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
43+
ActionListener<XPackUsageFeatureResponse> listener) {
44+
45+
final Set<String> usedFieldTypes = new HashSet<>();
46+
final Set<String> usedCharFilters = new HashSet<>();
47+
final Set<String> usedTokenizers = new HashSet<>();
48+
final Set<String> usedTokenFilters = new HashSet<>();
49+
final Set<String> usedAnalyzers = new HashSet<>();
50+
final Set<String> usedBuiltInCharFilters = new HashSet<>();
51+
final Set<String> usedBuiltInTokenizers = new HashSet<>();
52+
final Set<String> usedBuiltInTokenFilters = new HashSet<>();
53+
final Set<String> usedBuiltInAnalyzers = new HashSet<>();
54+
55+
for (IndexMetaData indexMetaData : state.metaData()) {
56+
MappingMetaData mappingMetaData = indexMetaData.mapping();
57+
if (mappingMetaData != null) {
58+
visitMapping(mappingMetaData.getSourceAsMap(), fieldMapping -> {
59+
Object type = fieldMapping.get("type");
60+
if (type != null) {
61+
usedFieldTypes.add(type.toString());
62+
} else if (fieldMapping.containsKey("properties")) {
63+
usedFieldTypes.add("object");
64+
}
65+
66+
for (String key : new String[] { "analyzer", "search_analyzer", "search_quote_analyzer" }) {
67+
Object analyzer = fieldMapping.get(key);
68+
if (analyzer != null) {
69+
usedBuiltInAnalyzers.add(analyzer.toString());
70+
}
71+
}
72+
});
73+
}
74+
75+
Settings indexSettings = indexMetaData.getSettings();
76+
77+
Map<String, Settings> analyzerSettings = indexSettings.getGroups("index.analysis.analyzer");
78+
usedBuiltInAnalyzers.removeAll(analyzerSettings.keySet());
79+
for (Settings analyzerSetting : analyzerSettings.values()) {
80+
usedAnalyzers.add(analyzerSetting.get("type", "custom"));
81+
usedBuiltInCharFilters.addAll(analyzerSetting.getAsList("char_filter"));
82+
String tokenizer = analyzerSetting.get("tokenizer");
83+
if (tokenizer != null) {
84+
usedBuiltInTokenizers.add(tokenizer);
85+
}
86+
usedBuiltInTokenFilters.addAll(analyzerSetting.getAsList("filter"));
87+
}
88+
89+
Map<String, Settings> charFilterSettings = indexSettings.getGroups("index.analysis.char_filter");
90+
usedBuiltInCharFilters.removeAll(charFilterSettings.keySet());
91+
aggregateAnalysisTypes(charFilterSettings.values(), usedCharFilters);
92+
93+
Map<String, Settings> tokenizerSettings = indexSettings.getGroups("index.analysis.tokenizer");
94+
usedBuiltInTokenizers.removeAll(tokenizerSettings.keySet());
95+
aggregateAnalysisTypes(tokenizerSettings.values(), usedTokenizers);
96+
97+
Map<String, Settings> tokenFilterSettings = indexSettings.getGroups("index.analysis.filter");
98+
usedBuiltInTokenFilters.removeAll(tokenFilterSettings.keySet());
99+
aggregateAnalysisTypes(tokenFilterSettings.values(), usedTokenFilters);
100+
}
101+
102+
listener.onResponse(new XPackUsageFeatureResponse(
103+
new IndexFeatureSetUsage(usedFieldTypes,
104+
usedCharFilters, usedTokenizers, usedTokenFilters, usedAnalyzers,
105+
usedBuiltInCharFilters, usedBuiltInTokenizers, usedBuiltInTokenFilters, usedBuiltInAnalyzers)));
106+
}
107+
108+
static void visitMapping(Map<String, ?> mapping, Consumer<Map<String, ?>> fieldMappingConsumer) {
109+
Object properties = mapping.get("properties");
110+
if (properties != null && properties instanceof Map) {
111+
@SuppressWarnings("unchecked")
112+
Map<String, ?> propertiesAsMap = (Map<String, ?>) properties;
113+
for (Object v : propertiesAsMap.values()) {
114+
if (v != null && v instanceof Map) {
115+
116+
@SuppressWarnings("unchecked")
117+
Map<String, ?> fieldMapping = (Map<String, ?>) v;
118+
fieldMappingConsumer.accept(fieldMapping);
119+
visitMapping(fieldMapping, fieldMappingConsumer);
120+
121+
// Multi fields
122+
Object fieldsO = fieldMapping.get("fields");
123+
if (fieldsO != null && fieldsO instanceof Map) {
124+
@SuppressWarnings("unchecked")
125+
Map<String, ?> fields = (Map<String, ?>) fieldsO;
126+
for (Object v2 : fields.values()) {
127+
if (v2 instanceof Map) {
128+
@SuppressWarnings("unchecked")
129+
Map<String, ?> fieldMapping2 = (Map<String, ?>) v2;
130+
fieldMappingConsumer.accept(fieldMapping2);
131+
}
132+
}
133+
}
134+
}
135+
}
136+
}
137+
}
138+
139+
static void aggregateAnalysisTypes(Collection<Settings> analysisComponents, Set<String> usedTypes) {
140+
for (Settings settings : analysisComponents) {
141+
String type = settings.get("type");
142+
if (type != null) {
143+
usedTypes.add(type);
144+
}
145+
}
146+
}
147+
}

0 commit comments

Comments
 (0)