Skip to content

Commit 0718361

Browse files
committed
Add telemetry for data tiers
This commit adds telemetry for our data tier formalization. This telemetry helps determine the topology of the cluster with regard to the content, data, hot, warm, & cold tiers/roles. An example of the telemetry looks like: ``` GET /_xpack/usage?human { ... "data_tiers" : { "available" : true, "enabled" : true, "data_warm" : { ... }, "data" : { ... }, "data_cold" : { ... }, "data_content" : { "node_count" : 1, "index_count" : 6, "total_shard_count" : 6, "primary_shard_count" : 6, "doc_count" : 71, "total_size" : "59.6kb", "total_size_bytes" : 61110, "primary_size" : "59.6kb", "primary_size_bytes" : 61110, "primary_shard_size_avg" : "9.9kb", "primary_shard_size_avg_bytes" : 10185, "primary_shard_size_median_bytes" : "8kb", "primary_shard_size_median_bytes" : 8254, "primary_shard_size_mad_bytes" : "7.2kb", "primary_shard_size_mad_bytes" : 7391 }, "data_hot" : { ... } } } ``` The fields are as follows: - node_count :: number of nodes with this tier/role - index_count :: number of indices on this tier - total_shard_count :: total number of shards for all nodes in this tier - primary_shard_count :: number of primary shards for all nodes in this tier - doc_count :: number of documents for all nodes in this tier - total_size_bytes :: total number of bytes for all shards for all nodes in this tier - primary_size_bytes :: number of bytes for all primary shards on all nodes in this tier - primary_shard_size_avg_bytes :: average shard size for primary shard in this tier - primary_shard_size_median_bytes :: median shard size for primary shard in this tier - primary_shard_size_mad_bytes :: [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) of shard size for primary shard in this tier Relates to elastic#60848
1 parent ff59d88 commit 0718361

File tree

12 files changed

+582
-5
lines changed

12 files changed

+582
-5
lines changed

x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,21 +16,24 @@
1616
import org.elasticsearch.plugins.Plugin;
1717
import org.elasticsearch.test.ESIntegTestCase;
1818
import org.elasticsearch.xpack.core.DataTier;
19-
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
19+
import org.elasticsearch.xpack.core.DataTiersFeatureSetUsage;
20+
import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
21+
import org.elasticsearch.xpack.core.action.XPackUsageResponse;
2022

2123
import java.util.Arrays;
2224
import java.util.Collection;
2325
import java.util.Collections;
2426

2527
import static org.hamcrest.Matchers.equalTo;
28+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
2629

2730
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
2831
public class DataTierIT extends ESIntegTestCase {
2932
private static final String index = "myindex";
3033

3134
@Override
3235
protected Collection<Class<? extends Plugin>> nodePlugins() {
33-
return Collections.singleton(LocalStateCompositeXPackPlugin.class);
36+
return Collections.singleton(DataTierTelemetryPlugin.class);
3437
}
3538

3639
public void testDefaultIndexAllocateToContent() {
@@ -194,6 +197,61 @@ public void testTemplateOverridesDefaults() {
194197
ensureYellow(index);
195198
}
196199

200+
public void testDataTierTelemetry() {
201+
startContentOnlyNode();
202+
startContentOnlyNode();
203+
startHotOnlyNode();
204+
205+
client().admin().indices().prepareCreate(index)
206+
.setSettings(Settings.builder()
207+
.put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, "data_hot")
208+
.put("index.number_of_shards", 2)
209+
.put("index.number_of_replicas", 0))
210+
.setWaitForActiveShards(0)
211+
.get();
212+
213+
client().admin().indices().prepareCreate(index + "2")
214+
.setSettings(Settings.builder()
215+
.put("index.number_of_shards", 1)
216+
.put("index.number_of_replicas", 1))
217+
.setWaitForActiveShards(0)
218+
.get();
219+
220+
ensureGreen();
221+
client().prepareIndex(index).setSource("foo", "bar").get();
222+
client().prepareIndex(index + "2").setSource("foo", "bar").get();
223+
client().prepareIndex(index + "2").setSource("foo", "bar").get();
224+
refresh(index, index + "2");
225+
226+
DataTiersFeatureSetUsage usage = getUsage();
227+
// We can't guarantee that internal indices aren't created, so some of these are >= checks
228+
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).nodeCount, equalTo(2));
229+
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).indexCount, greaterThanOrEqualTo(1));
230+
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).totalShardCount, greaterThanOrEqualTo(2));
231+
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryShardCount, greaterThanOrEqualTo(1));
232+
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).docCount, greaterThanOrEqualTo(2L));
233+
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryByteCount, greaterThanOrEqualTo(1L));
234+
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryByteCountMedian, greaterThanOrEqualTo(1L));
235+
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryShardBytesMAD, greaterThanOrEqualTo(0L));
236+
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).nodeCount, equalTo(1));
237+
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).indexCount, greaterThanOrEqualTo(1));
238+
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).totalShardCount, greaterThanOrEqualTo(2));
239+
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryShardCount, greaterThanOrEqualTo(2));
240+
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).docCount, greaterThanOrEqualTo(1L));
241+
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryByteCount, greaterThanOrEqualTo(1L));
242+
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryByteCountMedian, greaterThanOrEqualTo(1L));
243+
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryShardBytesMAD, greaterThanOrEqualTo(0L));
244+
}
245+
246+
private DataTiersFeatureSetUsage getUsage() {
247+
XPackUsageResponse usages = new XPackUsageRequestBuilder(client()).execute().actionGet();
248+
return usages.getUsages().stream()
249+
.filter(u -> u instanceof DataTiersFeatureSetUsage)
250+
.findFirst()
251+
.map(u -> (DataTiersFeatureSetUsage) u)
252+
.orElseThrow();
253+
}
254+
197255
public void startDataNode() {
198256
Settings nodeSettings = Settings.builder()
199257
.putList("node.roles", Arrays.asList("master", "data", "ingest"))
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.cluster.routing.allocation;
8+
9+
import org.elasticsearch.action.support.ActionFilters;
10+
import org.elasticsearch.action.support.TransportAction;
11+
import org.elasticsearch.client.node.NodeClient;
12+
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
13+
import org.elasticsearch.cluster.service.ClusterService;
14+
import org.elasticsearch.common.inject.Inject;
15+
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.license.LicenseService;
17+
import org.elasticsearch.protocol.xpack.XPackInfoRequest;
18+
import org.elasticsearch.protocol.xpack.XPackInfoResponse;
19+
import org.elasticsearch.protocol.xpack.XPackUsageRequest;
20+
import org.elasticsearch.threadpool.ThreadPool;
21+
import org.elasticsearch.transport.TransportService;
22+
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
23+
import org.elasticsearch.xpack.core.action.TransportXPackInfoAction;
24+
import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
25+
import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
26+
import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
27+
import org.elasticsearch.xpack.core.action.XPackUsageResponse;
28+
29+
import java.nio.file.Path;
30+
import java.util.Collections;
31+
import java.util.List;
32+
33+
/**
34+
* This plugin extends {@link LocalStateCompositeXPackPlugin} to only make the data tier telemetry
35+
* available. This allows telemetry to be retrieved in integration tests where it would otherwise
36+
* throw errors trying to retrieve all of the different telemetry types.
37+
*/
38+
public class DataTierTelemetryPlugin extends LocalStateCompositeXPackPlugin {
39+
40+
public static class DataTiersTransportXPackUsageAction extends TransportXPackUsageAction {
41+
@Inject
42+
public DataTiersTransportXPackUsageAction(ThreadPool threadPool, TransportService transportService,
43+
ClusterService clusterService, ActionFilters actionFilters,
44+
IndexNameExpressionResolver indexNameExpressionResolver, NodeClient client) {
45+
super(threadPool, transportService, clusterService, actionFilters, indexNameExpressionResolver, client);
46+
}
47+
@Override
48+
protected List<XPackUsageFeatureAction> usageActions() {
49+
return Collections.singletonList(XPackUsageFeatureAction.DATA_TIERS);
50+
}
51+
}
52+
53+
public static class DataTiersTransportXPackInfoAction extends TransportXPackInfoAction {
54+
@Inject
55+
public DataTiersTransportXPackInfoAction(TransportService transportService, ActionFilters actionFilters,
56+
LicenseService licenseService, NodeClient client) {
57+
super(transportService, actionFilters, licenseService, client);
58+
}
59+
60+
@Override
61+
protected List<XPackInfoFeatureAction> infoActions() {
62+
return Collections.singletonList(XPackInfoFeatureAction.DATA_TIERS);
63+
}
64+
}
65+
66+
public DataTierTelemetryPlugin(final Settings settings, final Path configPath) {
67+
super(settings, configPath);
68+
}
69+
70+
@Override
71+
protected Class<? extends TransportAction<XPackUsageRequest, XPackUsageResponse>> getUsageAction() {
72+
return DataTiersTransportXPackUsageAction.class;
73+
}
74+
75+
@Override
76+
protected Class<? extends TransportAction<XPackInfoRequest, XPackInfoResponse>> getInfoAction() {
77+
return DataTiersTransportXPackInfoAction.class;
78+
}
79+
}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import org.elasticsearch.index.shard.IndexSettingProvider;
1717
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
1818

19+
import java.util.Arrays;
20+
import java.util.HashSet;
1921
import java.util.Set;
2022

2123
/**
@@ -34,6 +36,8 @@ public class DataTier {
3436
public static final String DATA_WARM = "data_warm";
3537
public static final String DATA_COLD = "data_cold";
3638

39+
public static final Set<String> ALL_DATA_TIERS = new HashSet<>(Arrays.asList(DATA_CONTENT, DATA_HOT, DATA_WARM, DATA_COLD));
40+
3741
/**
3842
* Returns true if the given tier name is a valid tier
3943
*/
Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.core;
8+
9+
import org.elasticsearch.Version;
10+
import org.elasticsearch.common.Strings;
11+
import org.elasticsearch.common.io.stream.StreamInput;
12+
import org.elasticsearch.common.io.stream.StreamOutput;
13+
import org.elasticsearch.common.io.stream.Writeable;
14+
import org.elasticsearch.common.unit.ByteSizeValue;
15+
import org.elasticsearch.common.xcontent.ToXContentObject;
16+
import org.elasticsearch.common.xcontent.XContentBuilder;
17+
18+
import java.io.IOException;
19+
import java.util.Collections;
20+
import java.util.Map;
21+
import java.util.Objects;
22+
23+
/**
24+
* {@link DataTiersFeatureSetUsage} represents the xpack usage for data tiers.
25+
* This includes things like the number of nodes per tier, indices, shards, etc.
26+
* See {@link TierSpecificStats} for the stats that are tracked on a per-tier
27+
* basis.
28+
*/
29+
public class DataTiersFeatureSetUsage extends XPackFeatureSet.Usage {
30+
private final Map<String, TierSpecificStats> tierStats;
31+
32+
public DataTiersFeatureSetUsage(StreamInput in) throws IOException {
33+
super(in);
34+
this.tierStats = in.readMap(StreamInput::readString, TierSpecificStats::new);
35+
}
36+
37+
public DataTiersFeatureSetUsage(Map<String, TierSpecificStats> tierStats) {
38+
super(XPackField.DATA_TIERS, true, true);
39+
this.tierStats = tierStats;
40+
}
41+
42+
@Override
43+
public Version getMinimalSupportedVersion() {
44+
return Version.V_7_10_0;
45+
}
46+
47+
public Map<String, TierSpecificStats> getTierStats() {
48+
return Collections.unmodifiableMap(tierStats);
49+
}
50+
51+
@Override
52+
public void writeTo(StreamOutput out) throws IOException {
53+
super.writeTo(out);
54+
out.writeMap(tierStats, StreamOutput::writeString, (o, v) -> v.writeTo(o));
55+
}
56+
57+
@Override
58+
protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
59+
super.innerXContent(builder, params);
60+
for (Map.Entry<String, TierSpecificStats> tierStats : tierStats.entrySet()) {
61+
builder.field(tierStats.getKey(), tierStats.getValue());
62+
}
63+
}
64+
65+
@Override
66+
public int hashCode() {
67+
return Objects.hash(tierStats);
68+
}
69+
70+
@Override
71+
public boolean equals(Object obj) {
72+
if (obj == null) {
73+
return false;
74+
}
75+
if (getClass() != obj.getClass()) {
76+
return false;
77+
}
78+
DataTiersFeatureSetUsage other = (DataTiersFeatureSetUsage) obj;
79+
return Objects.equals(available, other.available) &&
80+
Objects.equals(enabled, other.enabled) &&
81+
Objects.equals(tierStats, other.tierStats);
82+
}
83+
84+
@Override
85+
public String toString() {
86+
return Strings.toString(this);
87+
}
88+
89+
/**
90+
* {@link TierSpecificStats} represents statistics about nodes in a single
91+
* tier, for example, how many nodes there are, the index count, shard
92+
* count, etc.
93+
*/
94+
public static class TierSpecificStats implements Writeable, ToXContentObject {
95+
96+
public final int nodeCount;
97+
public final int indexCount;
98+
public final int totalShardCount;
99+
public final int primaryShardCount;
100+
public final long docCount;
101+
public final long totalByteCount;
102+
public final long primaryByteCount;
103+
public final long primaryByteCountMedian;
104+
public final long primaryShardBytesMAD;
105+
106+
public TierSpecificStats(StreamInput in) throws IOException {
107+
this.nodeCount = in.readVInt();
108+
this.indexCount = in.readVInt();
109+
this.totalShardCount = in.readVInt();
110+
this.primaryShardCount = in.readVInt();
111+
this.docCount = in.readVLong();
112+
this.totalByteCount = in.readVLong();
113+
this.primaryByteCount = in.readVLong();
114+
this.primaryByteCountMedian = in.readVLong();
115+
this.primaryShardBytesMAD = in.readVLong();
116+
}
117+
118+
public TierSpecificStats(int nodeCount, int indexCount, int totalShardCount, int primaryShardCount, long docCount,
119+
long totalByteCount, long primaryByteCount, long primaryByteCountMedian, long primaryShardBytesMAD) {
120+
this.nodeCount = nodeCount;
121+
this.indexCount = indexCount;
122+
this.totalShardCount = totalShardCount;
123+
this.primaryShardCount = primaryShardCount;
124+
this.docCount = docCount;
125+
this.totalByteCount = totalByteCount;
126+
this.primaryByteCount = primaryByteCount;
127+
this.primaryByteCountMedian = primaryByteCountMedian;
128+
this.primaryShardBytesMAD = primaryShardBytesMAD;
129+
}
130+
131+
@Override
132+
public void writeTo(StreamOutput out) throws IOException {
133+
out.writeVInt(this.nodeCount);
134+
out.writeVInt(this.indexCount);
135+
out.writeVInt(this.totalShardCount);
136+
out.writeVInt(this.primaryShardCount);
137+
out.writeVLong(this.docCount);
138+
out.writeVLong(this.totalByteCount);
139+
out.writeVLong(this.primaryByteCount);
140+
out.writeVLong(this.primaryByteCountMedian);
141+
out.writeVLong(this.primaryShardBytesMAD);
142+
}
143+
144+
@Override
145+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
146+
builder.startObject();
147+
builder.field("node_count", nodeCount);
148+
builder.field("index_count", indexCount);
149+
builder.field("total_shard_count", totalShardCount);
150+
builder.field("primary_shard_count", primaryShardCount);
151+
builder.field("doc_count", docCount);
152+
builder.humanReadableField("total_size_bytes", "total_size", new ByteSizeValue(totalByteCount));
153+
builder.humanReadableField("primary_size_bytes", "primary_size", new ByteSizeValue(primaryByteCount));
154+
builder.humanReadableField("primary_shard_size_avg_bytes", "primary_shard_size_avg",
155+
new ByteSizeValue(primaryShardCount == 0 ? 0 : (primaryByteCount / primaryShardCount)));
156+
builder.humanReadableField("primary_shard_size_median_bytes", "primary_shard_size_median_bytes",
157+
new ByteSizeValue(primaryByteCountMedian));
158+
builder.humanReadableField("primary_shard_size_mad_bytes", "primary_shard_size_mad_bytes",
159+
new ByteSizeValue(primaryShardBytesMAD));
160+
builder.endObject();
161+
return builder;
162+
}
163+
164+
@Override
165+
public int hashCode() {
166+
return Objects.hash(this.nodeCount, this.indexCount, this.totalShardCount, this.primaryShardCount, this.totalByteCount,
167+
this.primaryByteCount, this.docCount, this.primaryByteCountMedian, this.primaryShardBytesMAD);
168+
}
169+
170+
@Override
171+
public boolean equals(Object obj) {
172+
if (obj == null) {
173+
return false;
174+
}
175+
if (getClass() != obj.getClass()) {
176+
return false;
177+
}
178+
TierSpecificStats other = (TierSpecificStats) obj;
179+
return nodeCount == other.nodeCount &&
180+
indexCount == other.indexCount &&
181+
totalShardCount == other.totalShardCount &&
182+
primaryShardCount == other.primaryShardCount &&
183+
docCount == other.docCount &&
184+
totalByteCount == other.totalByteCount &&
185+
primaryByteCount == other.primaryByteCount &&
186+
primaryByteCountMedian == other.primaryByteCountMedian &&
187+
primaryShardBytesMAD == other.primaryShardBytesMAD;
188+
}
189+
190+
@Override
191+
public String toString() {
192+
return Strings.toString(this);
193+
}
194+
}
195+
}

0 commit comments

Comments
 (0)