Skip to content

Commit e57fc24

Browse files
dnhatndnhatn
authored
Add indices metrics for each index mode (#113737)
This change introduces index metrics per node, grouped by by index mode. For each index mode, we track the number of indices, document count, and store size. These metrics will help compare the usage of logsdb and time_series indices to standard indices. Other metrics, such as index longevity and newly created indices, could be added in a follow-up. Here is the list of 9 metrics introduced in this PR: es.indices.standard.total es.indices.standard.docs.total es.indices.standard.bytes.total es.indices.time_series.total es.indices.time_series.docs.total es.indices.time_series.bytes.total es.indices.logsdb.total es.indices.logsdb.docs.total es.indices.logsdb.bytes.total
1 parent a6b104d commit e57fc24

File tree

4 files changed

+429
-0
lines changed

4 files changed

+429
-0
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.monitor.metrics;
11+
12+
import org.elasticsearch.common.settings.Setting;
13+
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.core.TimeValue;
15+
import org.elasticsearch.plugins.Plugin;
16+
import org.elasticsearch.plugins.PluginsService;
17+
import org.elasticsearch.telemetry.Measurement;
18+
import org.elasticsearch.telemetry.TestTelemetryPlugin;
19+
import org.elasticsearch.test.ESIntegTestCase;
20+
import org.hamcrest.Matcher;
21+
22+
import java.util.Collection;
23+
import java.util.List;
24+
import java.util.Map;
25+
26+
import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
27+
import static org.hamcrest.Matchers.equalTo;
28+
import static org.hamcrest.Matchers.greaterThan;
29+
import static org.hamcrest.Matchers.hasSize;
30+
31+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
32+
public class IndicesMetricsIT extends ESIntegTestCase {
33+
34+
public static class TestAPMInternalSettings extends Plugin {
35+
@Override
36+
public List<Setting<?>> getSettings() {
37+
return List.of(
38+
Setting.timeSetting("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(0), Setting.Property.NodeScope)
39+
);
40+
}
41+
}
42+
43+
@Override
44+
protected Collection<Class<? extends Plugin>> nodePlugins() {
45+
return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class);
46+
}
47+
48+
@Override
49+
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
50+
return Settings.builder()
51+
.put(super.nodeSettings(nodeOrdinal, otherSettings))
52+
.put("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(0)) // disable metrics cache refresh delay
53+
.build();
54+
}
55+
56+
static final String STANDARD_INDEX_COUNT = "es.indices.standard.total";
57+
static final String STANDARD_DOCS_COUNT = "es.indices.standard.docs.total";
58+
static final String STANDARD_BYTES_SIZE = "es.indices.standard.bytes.total";
59+
60+
static final String TIME_SERIES_INDEX_COUNT = "es.indices.time_series.total";
61+
static final String TIME_SERIES_DOCS_COUNT = "es.indices.time_series.docs.total";
62+
static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.bytes.total";
63+
64+
static final String LOGSDB_INDEX_COUNT = "es.indices.logsdb.total";
65+
static final String LOGSDB_DOCS_COUNT = "es.indices.logsdb.docs.total";
66+
static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.bytes.total";
67+
68+
public void testIndicesMetrics() {
69+
String node = internalCluster().startNode();
70+
ensureStableCluster(1);
71+
final TestTelemetryPlugin telemetry = internalCluster().getInstance(PluginsService.class, node)
72+
.filterPlugins(TestTelemetryPlugin.class)
73+
.findFirst()
74+
.orElseThrow();
75+
telemetry.resetMeter();
76+
long numStandardIndices = randomIntBetween(1, 5);
77+
long numStandardDocs = populateStandardIndices(numStandardIndices);
78+
collectThenAssertMetrics(
79+
telemetry,
80+
1,
81+
Map.of(
82+
STANDARD_INDEX_COUNT,
83+
equalTo(numStandardIndices),
84+
STANDARD_DOCS_COUNT,
85+
equalTo(numStandardDocs),
86+
STANDARD_BYTES_SIZE,
87+
greaterThan(0L),
88+
89+
TIME_SERIES_INDEX_COUNT,
90+
equalTo(0L),
91+
TIME_SERIES_DOCS_COUNT,
92+
equalTo(0L),
93+
TIME_SERIES_BYTES_SIZE,
94+
equalTo(0L),
95+
96+
LOGSDB_INDEX_COUNT,
97+
equalTo(0L),
98+
LOGSDB_DOCS_COUNT,
99+
equalTo(0L),
100+
LOGSDB_BYTES_SIZE,
101+
equalTo(0L)
102+
)
103+
);
104+
105+
long numTimeSeriesIndices = randomIntBetween(1, 5);
106+
long numTimeSeriesDocs = populateTimeSeriesIndices(numTimeSeriesIndices);
107+
collectThenAssertMetrics(
108+
telemetry,
109+
2,
110+
Map.of(
111+
STANDARD_INDEX_COUNT,
112+
equalTo(numStandardIndices),
113+
STANDARD_DOCS_COUNT,
114+
equalTo(numStandardDocs),
115+
STANDARD_BYTES_SIZE,
116+
greaterThan(0L),
117+
118+
TIME_SERIES_INDEX_COUNT,
119+
equalTo(numTimeSeriesIndices),
120+
TIME_SERIES_DOCS_COUNT,
121+
equalTo(numTimeSeriesDocs),
122+
TIME_SERIES_BYTES_SIZE,
123+
greaterThan(20L),
124+
125+
LOGSDB_INDEX_COUNT,
126+
equalTo(0L),
127+
LOGSDB_DOCS_COUNT,
128+
equalTo(0L),
129+
LOGSDB_BYTES_SIZE,
130+
equalTo(0L)
131+
)
132+
);
133+
134+
long numLogsdbIndices = randomIntBetween(1, 5);
135+
long numLogsdbDocs = populateLogsdbIndices(numLogsdbIndices);
136+
collectThenAssertMetrics(
137+
telemetry,
138+
3,
139+
Map.of(
140+
STANDARD_INDEX_COUNT,
141+
equalTo(numStandardIndices),
142+
STANDARD_DOCS_COUNT,
143+
equalTo(numStandardDocs),
144+
STANDARD_BYTES_SIZE,
145+
greaterThan(0L),
146+
147+
TIME_SERIES_INDEX_COUNT,
148+
equalTo(numTimeSeriesIndices),
149+
TIME_SERIES_DOCS_COUNT,
150+
equalTo(numTimeSeriesDocs),
151+
TIME_SERIES_BYTES_SIZE,
152+
greaterThan(20L),
153+
154+
LOGSDB_INDEX_COUNT,
155+
equalTo(numLogsdbIndices),
156+
LOGSDB_DOCS_COUNT,
157+
equalTo(numLogsdbDocs),
158+
LOGSDB_BYTES_SIZE,
159+
greaterThan(0L)
160+
)
161+
);
162+
}
163+
164+
void collectThenAssertMetrics(TestTelemetryPlugin telemetry, int times, Map<String, Matcher<Long>> matchers) {
165+
telemetry.collect();
166+
for (Map.Entry<String, Matcher<Long>> e : matchers.entrySet()) {
167+
String name = e.getKey();
168+
List<Measurement> measurements = telemetry.getLongGaugeMeasurement(name);
169+
assertThat(name, measurements, hasSize(times));
170+
assertThat(name, measurements.getLast().getLong(), e.getValue());
171+
}
172+
}
173+
174+
int populateStandardIndices(long numIndices) {
175+
int totalDocs = 0;
176+
for (int i = 0; i < numIndices; i++) {
177+
String indexName = "standard-" + i;
178+
createIndex(indexName);
179+
int numDocs = between(1, 5);
180+
for (int d = 0; d < numDocs; d++) {
181+
indexDoc(indexName, Integer.toString(d), "f", Integer.toString(d));
182+
}
183+
totalDocs += numDocs;
184+
flush(indexName);
185+
}
186+
return totalDocs;
187+
}
188+
189+
int populateTimeSeriesIndices(long numIndices) {
190+
int totalDocs = 0;
191+
for (int i = 0; i < numIndices; i++) {
192+
String indexName = "time_series-" + i;
193+
Settings settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("host")).build();
194+
client().admin()
195+
.indices()
196+
.prepareCreate(indexName)
197+
.setSettings(settings)
198+
.setMapping(
199+
"@timestamp",
200+
"type=date",
201+
"host",
202+
"type=keyword,time_series_dimension=true",
203+
"cpu",
204+
"type=long,time_series_metric=gauge"
205+
)
206+
.get();
207+
long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z");
208+
int numDocs = between(1, 5);
209+
for (int d = 0; d < numDocs; d++) {
210+
timestamp += between(1, 5) * 1000L;
211+
client().prepareIndex(indexName)
212+
.setSource("@timestamp", timestamp, "host", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100))
213+
.get();
214+
}
215+
totalDocs += numDocs;
216+
flush(indexName);
217+
}
218+
return totalDocs;
219+
}
220+
221+
int populateLogsdbIndices(long numIndices) {
222+
int totalDocs = 0;
223+
for (int i = 0; i < numIndices; i++) {
224+
String indexName = "logsdb-" + i;
225+
Settings settings = Settings.builder().put("mode", "logsdb").build();
226+
client().admin()
227+
.indices()
228+
.prepareCreate(indexName)
229+
.setSettings(settings)
230+
.setMapping("@timestamp", "type=date", "host.name", "type=keyword", "cpu", "type=long")
231+
.get();
232+
long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z");
233+
int numDocs = between(1, 5);
234+
for (int d = 0; d < numDocs; d++) {
235+
timestamp += between(1, 5) * 1000L;
236+
client().prepareIndex(indexName)
237+
.setSource("@timestamp", timestamp, "host.name", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100))
238+
.get();
239+
}
240+
totalDocs += numDocs;
241+
flush(indexName);
242+
}
243+
return totalDocs;
244+
}
245+
}

0 commit comments

Comments
 (0)