Skip to content

Commit 623df95

Browse files
Tim-BrooksTim-Brooks
authored
Adding indexing pressure stats to node stats API (#59467)
We have recently added internal metrics to monitor the amount of indexing occurring on a node. These metrics introduce back pressure to indexing when memory utilization is too high. This commit exposes these stats through the node stats API.
1 parent dc7d4c6 commit 623df95

File tree

36 files changed

+535
-230
lines changed

36 files changed

+535
-230
lines changed

‎qa/smoke-test-http/src/test/java/org/elasticsearch/http/IndexingPressureRestIT.java

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.http;
20+
21+
import org.elasticsearch.client.Request;
22+
import org.elasticsearch.client.Response;
23+
import org.elasticsearch.client.ResponseException;
24+
import org.elasticsearch.common.settings.Settings;
25+
import org.elasticsearch.common.xcontent.XContentHelper;
26+
import org.elasticsearch.common.xcontent.json.JsonXContent;
27+
import org.elasticsearch.index.IndexingPressure;
28+
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
29+
import org.elasticsearch.test.ESIntegTestCase.Scope;
30+
import org.elasticsearch.test.XContentTestUtils;
31+
32+
import java.io.IOException;
33+
import java.util.ArrayList;
34+
import java.util.Map;
35+
36+
import static org.elasticsearch.rest.RestStatus.CREATED;
37+
import static org.elasticsearch.rest.RestStatus.OK;
38+
import static org.elasticsearch.rest.RestStatus.TOO_MANY_REQUESTS;
39+
import static org.hamcrest.Matchers.equalTo;
40+
import static org.hamcrest.Matchers.greaterThan;
41+
import static org.hamcrest.Matchers.lessThan;
42+
43+
/**
44+
* Test Indexing Pressure Metrics and Statistics
45+
*/
46+
@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0)
47+
public class IndexingPressureRestIT extends HttpSmokeTestCase {
48+
49+
private static final Settings unboundedWriteQueue = Settings.builder().put("thread_pool.write.queue_size", -1).build();
50+
51+
@Override
52+
protected Settings nodeSettings(int nodeOrdinal) {
53+
return Settings.builder()
54+
.put(super.nodeSettings(nodeOrdinal))
55+
.put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "1KB")
56+
.put(unboundedWriteQueue)
57+
.build();
58+
}
59+
60+
@SuppressWarnings("unchecked")
61+
public void testIndexingPressureStats() throws IOException {
62+
Request createRequest = new Request("PUT", "/index_name");
63+
createRequest.setJsonEntity("{\"settings\": {\"index\": {\"number_of_shards\": 1, \"number_of_replicas\": 1, " +
64+
"\"write.wait_for_active_shards\": 2}}}");
65+
final Response indexCreatedResponse = getRestClient().performRequest(createRequest);
66+
assertThat(indexCreatedResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus()));
67+
68+
Request successfulIndexingRequest = new Request("POST", "/index_name/_doc/");
69+
successfulIndexingRequest.setJsonEntity("{\"x\": \"small text\"}");
70+
final Response indexSuccessFul = getRestClient().performRequest(successfulIndexingRequest);
71+
assertThat(indexSuccessFul.getStatusLine().getStatusCode(), equalTo(CREATED.getStatus()));
72+
73+
Request getNodeStats = new Request("GET", "/_nodes/stats/indexing_pressure");
74+
final Response nodeStats = getRestClient().performRequest(getNodeStats);
75+
Map<String, Object> nodeStatsMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, nodeStats.getEntity().getContent(), true);
76+
ArrayList<Object> values = new ArrayList<>(((Map<Object, Object>) nodeStatsMap.get("nodes")).values());
77+
assertThat(values.size(), equalTo(2));
78+
XContentTestUtils.JsonMapView node1 = new XContentTestUtils.JsonMapView((Map<String, Object>) values.get(0));
79+
Integer node1IndexingBytes = node1.get("indexing_pressure.total.coordinating_and_primary_bytes");
80+
Integer node1ReplicaBytes = node1.get("indexing_pressure.total.replica_bytes");
81+
Integer node1Rejections = node1.get("indexing_pressure.total.coordinating_and_primary_memory_limit_rejections");
82+
XContentTestUtils.JsonMapView node2 = new XContentTestUtils.JsonMapView((Map<String, Object>) values.get(1));
83+
Integer node2IndexingBytes = node2.get("indexing_pressure.total.coordinating_and_primary_bytes");
84+
Integer node2ReplicaBytes = node2.get("indexing_pressure.total.replica_bytes");
85+
Integer node2Rejections = node2.get("indexing_pressure.total.coordinating_and_primary_memory_limit_rejections");
86+
87+
if (node1IndexingBytes == 0) {
88+
assertThat(node2IndexingBytes, greaterThan(0));
89+
assertThat(node2IndexingBytes, lessThan(1024));
90+
} else {
91+
assertThat(node1IndexingBytes, greaterThan(0));
92+
assertThat(node1IndexingBytes, lessThan(1024));
93+
}
94+
95+
if (node1ReplicaBytes == 0) {
96+
assertThat(node2ReplicaBytes, greaterThan(0));
97+
assertThat(node2ReplicaBytes, lessThan(1024));
98+
} else {
99+
assertThat(node2ReplicaBytes, equalTo(0));
100+
assertThat(node1ReplicaBytes, lessThan(1024));
101+
}
102+
103+
assertThat(node1Rejections, equalTo(0));
104+
assertThat(node2Rejections, equalTo(0));
105+
106+
Request failedIndexingRequest = new Request("POST", "/index_name/_doc/");
107+
String largeString = randomAlphaOfLength(10000);
108+
failedIndexingRequest.setJsonEntity("{\"x\": " + largeString + "}");
109+
ResponseException exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(failedIndexingRequest));
110+
assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(TOO_MANY_REQUESTS.getStatus()));
111+
112+
Request getNodeStats2 = new Request("GET", "/_nodes/stats/indexing_pressure");
113+
final Response nodeStats2 = getRestClient().performRequest(getNodeStats2);
114+
Map<String, Object> nodeStatsMap2 = XContentHelper.convertToMap(JsonXContent.jsonXContent, nodeStats2.getEntity().getContent(),
115+
true);
116+
ArrayList<Object> values2 = new ArrayList<>(((Map<Object, Object>) nodeStatsMap2.get("nodes")).values());
117+
assertThat(values2.size(), equalTo(2));
118+
XContentTestUtils.JsonMapView node1AfterRejection = new XContentTestUtils.JsonMapView((Map<String, Object>) values2.get(0));
119+
node1Rejections = node1AfterRejection.get("indexing_pressure.total.coordinating_and_primary_memory_limit_rejections");
120+
XContentTestUtils.JsonMapView node2AfterRejection = new XContentTestUtils.JsonMapView((Map<String, Object>) values2.get(1));
121+
node2Rejections = node2AfterRejection.get("indexing_pressure.total.coordinating_and_primary_memory_limit_rejections");
122+
123+
if (node1Rejections == 0) {
124+
assertThat(node2Rejections, equalTo(1));
125+
} else {
126+
assertThat(node1Rejections, equalTo(1));
127+
}
128+
}
129+
}

‎rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json

+8-4
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@
4444
"process",
4545
"thread_pool",
4646
"transport",
47-
"discovery"
47+
"discovery",
48+
"indexing_pressure"
4849
],
4950
"description":"Limit the information returned to the specified metrics"
5051
}
@@ -69,7 +70,8 @@
6970
"process",
7071
"thread_pool",
7172
"transport",
72-
"discovery"
73+
"discovery",
74+
"indexing_pressure"
7375
],
7476
"description":"Limit the information returned to the specified metrics"
7577
},
@@ -98,7 +100,8 @@
98100
"process",
99101
"thread_pool",
100102
"transport",
101-
"discovery"
103+
"discovery",
104+
"indexing_pressure"
102105
],
103106
"description":"Limit the information returned to the specified metrics"
104107
},
@@ -145,7 +148,8 @@
145148
"process",
146149
"thread_pool",
147150
"transport",
148-
"discovery"
151+
"discovery",
152+
"indexing_pressure"
149153
],
150154
"description":"Limit the information returned to the specified metrics"
151155
},

‎rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/50_indexing_pressure.yml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
---
2+
"Indexing pressure stats":
3+
- skip:
4+
version: " - 7.8.99"
5+
reason: "indexing_pressure was added in 7.9"
6+
features: [arbitrary_key]
7+
8+
- do:
9+
nodes.info: {}
10+
- set:
11+
nodes._arbitrary_key_: node_id
12+
13+
- do:
14+
nodes.stats:
15+
metric: [ indexing_pressure ]
16+
17+
- gte: { nodes.$node_id.indexing_pressure.total.coordinating_and_primary_bytes: 0 }
18+
- gte: { nodes.$node_id.indexing_pressure.total.replica_bytes: 0 }
19+
- gte: { nodes.$node_id.indexing_pressure.total.all_bytes: 0 }
20+
- gte: { nodes.$node_id.indexing_pressure.total.coordinating_and_primary_memory_limit_rejections: 0 }
21+
- gte: { nodes.$node_id.indexing_pressure.total.replica_memory_limit_rejections: 0 }
22+
- gte: { nodes.$node_id.indexing_pressure.current.coordinating_and_primary_bytes: 0 }
23+
- gte: { nodes.$node_id.indexing_pressure.current.replica_bytes: 0 }
24+
- gte: { nodes.$node_id.indexing_pressure.current.all_bytes: 0 }
25+
26+
# TODO:
27+
#
28+
# Change skipped version after backport

0 commit comments

Comments
 (0)