Skip to content

Commit ccfc8d7

Browse files
ycombinatorycombinator
authored
Get cluster UUID from Info API call (#8592)
Starting from 6.5.0, Elasticsearch [will return](elastic/elasticsearch#32206) the `cluster_uuid` as part of the `GET _cluster/stats` API response. However, the Elasticsearch metricbeat module has been in existence since 6.3.0 so it must fetch the cluster UUID from the `GET /` Elasticsearch API. This PR makes it so the `GET /` API is always called and the cluster UUID from it is used. This is obviously not ideal in terms of API calls, but it's the simplest fix (for now, until we can figure out a nicer way to deal with version differences).
1 parent cb7ddd8 commit ccfc8d7

File tree

5 files changed

+153
-165
lines changed

5 files changed

+153
-165
lines changed

‎metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json

+134-137
Original file line numberDiff line numberDiff line change
@@ -1,147 +1,144 @@
11
{
2-
"_nodes":{
3-
"total":2,
4-
"successful":2,
5-
"failed":0
2+
"_nodes": {
3+
"total": 2,
4+
"successful": 2,
5+
"failed": 0
66
},
7-
"cluster_name":"elasticsearch",
8-
"timestamp":1532433421874,
9-
"status":"yellow",
10-
"indices":{
11-
"count":4,
12-
"shards":{
13-
"total":16,
14-
"primaries":8,
15-
"replication":1,
16-
"index":{
17-
"shards":{
18-
"min":2,
19-
"max":10,
20-
"avg":4
21-
},
22-
"primaries":{
23-
"min":1,
24-
"max":5,
25-
"avg":2
26-
},
27-
"replication":{
28-
"min":1,
29-
"max":1,
30-
"avg":1
31-
}
32-
}
33-
},
34-
"docs":{
35-
"count":145,
36-
"deleted":0
37-
},
38-
"store":{
39-
"size_in_bytes":1030085
40-
},
41-
"fielddata":{
42-
"memory_size_in_bytes":0,
43-
"evictions":0
44-
},
45-
"query_cache":{
46-
"memory_size_in_bytes":0,
47-
"total_count":0,
48-
"hit_count":0,
49-
"miss_count":0,
50-
"cache_size":0,
51-
"cache_count":0,
52-
"evictions":0
53-
},
54-
"completion":{
55-
"size_in_bytes":0
56-
},
57-
"segments":{
58-
"count":26,
59-
"memory_in_bytes":108394,
60-
"terms_memory_in_bytes":83123,
61-
"stored_fields_memory_in_bytes":8112,
62-
"term_vectors_memory_in_bytes":0,
63-
"norms_memory_in_bytes":0,
64-
"points_memory_in_bytes":3047,
65-
"doc_values_memory_in_bytes":14112,
66-
"index_writer_memory_in_bytes":0,
67-
"version_map_memory_in_bytes":0,
68-
"fixed_bit_set_memory_in_bytes":0,
69-
"max_unsafe_auto_id_timestamp":1532433381676,
70-
"file_sizes":{
71-
7+
"cluster_name": "elasticsearch",
8+
"cluster_uuid": "zLXBfdzPSrO12OBIz3ybyg",
9+
"timestamp": 1532433421874,
10+
"status": "yellow",
11+
"indices": {
12+
"count": 4,
13+
"shards": {
14+
"total": 16,
15+
"primaries": 8,
16+
"replication": 1,
17+
"index": {
18+
"shards": {
19+
"min": 2,
20+
"max": 10,
21+
"avg": 4
22+
},
23+
"primaries": {
24+
"min": 1,
25+
"max": 5,
26+
"avg": 2
27+
},
28+
"replication": {
29+
"min": 1,
30+
"max": 1,
31+
"avg": 1
32+
}
33+
}
34+
},
35+
"docs": {
36+
"count": 145,
37+
"deleted": 0
38+
},
39+
"store": {
40+
"size_in_bytes": 1030085
41+
},
42+
"fielddata": {
43+
"memory_size_in_bytes": 0,
44+
"evictions": 0
45+
},
46+
"query_cache": {
47+
"memory_size_in_bytes": 0,
48+
"total_count": 0,
49+
"hit_count": 0,
50+
"miss_count": 0,
51+
"cache_size": 0,
52+
"cache_count": 0,
53+
"evictions": 0
54+
},
55+
"completion": {
56+
"size_in_bytes": 0
57+
},
58+
"segments": {
59+
"count": 26,
60+
"memory_in_bytes": 108394,
61+
"terms_memory_in_bytes": 83123,
62+
"stored_fields_memory_in_bytes": 8112,
63+
"term_vectors_memory_in_bytes": 0,
64+
"norms_memory_in_bytes": 0,
65+
"points_memory_in_bytes": 3047,
66+
"doc_values_memory_in_bytes": 14112,
67+
"index_writer_memory_in_bytes": 0,
68+
"version_map_memory_in_bytes": 0,
69+
"fixed_bit_set_memory_in_bytes": 0,
70+
"max_unsafe_auto_id_timestamp": 1532433381676,
71+
"file_sizes": {}
7272
}
73-
}
7473
},
75-
"nodes":{
76-
"count":{
77-
"total":2,
78-
"data":2,
79-
"coordinating_only":0,
80-
"master":2,
81-
"ingest":2
82-
},
83-
"versions":[
84-
"7.0.0-alpha1"
85-
],
86-
"os":{
87-
"available_processors":16,
88-
"allocated_processors":16,
89-
"names":[
90-
{
91-
"name":"Mac OS X",
92-
"count":2
93-
}
94-
],
95-
"mem":{
96-
"total_in_bytes":34359738368,
97-
"free_in_bytes":735543296,
98-
"used_in_bytes":33624195072,
99-
"free_percent":2,
100-
"used_percent":98
101-
}
102-
},
103-
"process":{
104-
"cpu":{
105-
"percent":0
74+
"nodes": {
75+
"count": {
76+
"total": 2,
77+
"data": 2,
78+
"coordinating_only": 0,
79+
"master": 2,
80+
"ingest": 2
10681
},
107-
"open_file_descriptors":{
108-
"min":364,
109-
"max":371,
110-
"avg":367
111-
}
112-
},
113-
"jvm":{
114-
"max_uptime_in_millis":228721,
115-
"versions":[
116-
{
117-
"version":"10.0.1",
118-
"vm_name":"Java HotSpot(TM) 64-Bit Server VM",
119-
"vm_version":"10.0.1+10",
120-
"vm_vendor":"Oracle Corporation",
121-
"count":2
122-
}
82+
"versions": [
83+
"7.0.0-alpha1"
12384
],
124-
"mem":{
125-
"heap_used_in_bytes":420771816,
126-
"heap_max_in_bytes":2075918336
85+
"os": {
86+
"available_processors": 16,
87+
"allocated_processors": 16,
88+
"names": [
89+
{
90+
"name": "Mac OS X",
91+
"count": 2
92+
}
93+
],
94+
"mem": {
95+
"total_in_bytes": 34359738368,
96+
"free_in_bytes": 735543296,
97+
"used_in_bytes": 33624195072,
98+
"free_percent": 2,
99+
"used_percent": 98
100+
}
101+
},
102+
"process": {
103+
"cpu": {
104+
"percent": 0
105+
},
106+
"open_file_descriptors": {
107+
"min": 364,
108+
"max": 371,
109+
"avg": 367
110+
}
111+
},
112+
"jvm": {
113+
"max_uptime_in_millis": 228721,
114+
"versions": [
115+
{
116+
"version": "10.0.1",
117+
"vm_name": "Java HotSpot(TM) 64-Bit Server VM",
118+
"vm_version": "10.0.1+10",
119+
"vm_vendor": "Oracle Corporation",
120+
"count": 2
121+
}
122+
],
123+
"mem": {
124+
"heap_used_in_bytes": 420771816,
125+
"heap_max_in_bytes": 2075918336
126+
},
127+
"threads": 164
127128
},
128-
"threads":164
129-
},
130-
"fs":{
131-
"total_in_bytes":499963170816,
132-
"free_in_bytes":410995167232,
133-
"available_in_bytes":408143978496
134-
},
135-
"plugins":[
136-
137-
],
138-
"network_types":{
139-
"transport_types":{
140-
"security4":2
129+
"fs": {
130+
"total_in_bytes": 499963170816,
131+
"free_in_bytes": 410995167232,
132+
"available_in_bytes": 408143978496
141133
},
142-
"http_types":{
143-
"security4":2
134+
"plugins": [],
135+
"network_types": {
136+
"transport_types": {
137+
"security4": 2
138+
},
139+
"http_types": {
140+
"security4": 2
141+
}
144142
}
145-
}
146143
}
147-
}
144+
}

‎metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go

+8-2
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,15 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) {
7373
return
7474
}
7575

76+
info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+clusterStatsPath)
77+
if err != nil {
78+
r.Error(errors.Wrap(err, "failed to get info from Elasticsearch"))
79+
return
80+
}
81+
7682
if m.MetricSet.XPack {
77-
eventMappingXPack(r, m, content)
83+
eventMappingXPack(r, m, *info, content)
7884
} else {
79-
eventMapping(r, content)
85+
eventMapping(r, *info, content)
8086
}
8187
}

‎metricbeat/module/elasticsearch/cluster_stats/data.go

+9-19
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ import (
2525
"github.com/elastic/beats/libbeat/common"
2626
s "github.com/elastic/beats/libbeat/common/schema"
2727
c "github.com/elastic/beats/libbeat/common/schema/mapstriface"
28-
"github.com/elastic/beats/metricbeat/helper/elastic"
2928
"github.com/elastic/beats/metricbeat/mb"
3029
"github.com/elastic/beats/metricbeat/module/elasticsearch"
3130
)
@@ -53,13 +52,20 @@ var (
5352
}
5453
)
5554

56-
func eventMapping(r mb.ReporterV2, content []byte) error {
55+
func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error {
56+
var event mb.Event
57+
event.RootFields = common.MapStr{}
58+
event.RootFields.Put("service.name", elasticsearch.ModuleName)
59+
60+
event.ModuleFields = common.MapStr{}
61+
event.ModuleFields.Put("cluster.name", info.ClusterName)
62+
event.ModuleFields.Put("cluster.id", info.ClusterID)
63+
5764
var data map[string]interface{}
5865
err := json.Unmarshal(content, &data)
5966
if err != nil {
6067
err = errors.Wrap(err, "failure parsing Elasticsearch Cluster Stats API response")
6168
r.Error(err)
62-
return err
6369
}
6470

6571
metricSetFields, err := schema.Apply(data)
@@ -69,22 +75,6 @@ func eventMapping(r mb.ReporterV2, content []byte) error {
6975
return err
7076
}
7177

72-
clusterName, ok := data["cluster_name"]
73-
if !ok {
74-
return elastic.ReportErrorForMissingField("cluster_name", elastic.Elasticsearch, r)
75-
}
76-
77-
var event mb.Event
78-
event.RootFields = common.MapStr{}
79-
event.RootFields.Put("service.name", elasticsearch.ModuleName)
80-
81-
event.ModuleFields = common.MapStr{}
82-
event.ModuleFields.Put("cluster.name", clusterName)
83-
clusterUUID, ok := data["cluster_uuid"]
84-
if ok {
85-
event.ModuleFields.Put("cluster.id", clusterUUID)
86-
}
87-
8878
event.MetricSetFields = metricSetFields
8979

9080
r.Event(event)

‎metricbeat/module/elasticsearch/cluster_stats/data_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@ import (
2626
)
2727

2828
func TestMapper(t *testing.T) {
29-
elasticsearch.TestMapper(t, "./_meta/test/cluster_stats.*.json", eventMapping)
29+
elasticsearch.TestMapperWithInfo(t, "./_meta/test/cluster_stats.*.json", eventMapping)
3030
}

‎metricbeat/module/elasticsearch/cluster_stats/data_xpack.go

+1-6
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ func apmIndicesExist(clusterState common.MapStr) (bool, error) {
142142
return false, nil
143143
}
144144

145-
func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error {
145+
func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error {
146146
var data map[string]interface{}
147147
err := json.Unmarshal(content, &data)
148148
if err != nil {
@@ -160,11 +160,6 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error {
160160
return fmt.Errorf("cluster name is not a string")
161161
}
162162

163-
info, err := elasticsearch.GetInfo(m.HTTP, m.HTTP.GetURI())
164-
if err != nil {
165-
return errors.Wrap(err, "failed to get info from Elasticsearch")
166-
}
167-
168163
license, err := elasticsearch.GetLicense(m.HTTP, m.HTTP.GetURI())
169164
if err != nil {
170165
return errors.Wrap(err, "failed to get license from Elasticsearch")

0 commit comments

Comments
 (0)