Skip to content

Commit 8b5187b

Browse files
committed
ApiRequestCount conditional gathering (openshift#492)
* ApiRequestCounts conditional gathering * Test and minor updates according to the review * Update name * Fix also the schema * Change Alert type to AlertLabels map
1 parent 51e4523 commit 8b5187b

11 files changed

+311
-16
lines changed

docs/gathered-data.md

+12
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,17 @@
11
This document is auto-generated by `make docs`
22

3+
## APIRequestCounts
4+
5+
BuildGatherApiRequestCounts creates a gathering closure which collects API requests counts for the
6+
resources mentioned in the alert provided as a string parameter
7+
Params is of type AlertIsFiringConditionParams:
8+
- alert_name string - name of the firing alert
9+
10+
* Location in archive: conditional/alerts/<alert_name>/api_request_counts.json
11+
* Since versions:
12+
* 4.10+
13+
14+
315
## CRD
416

517
collects the specified Custom Resource Definitions.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[
2+
{
3+
"resource": "customresourcedefinitions.v1beta1.apiextensions.k8s.io",
4+
"removed_in_release": "1.22",
5+
"total_request_count": 46,
6+
"last_day_request_count": 0
7+
}
8+
]

manifests/03-clusterrole.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,14 @@ rules:
212212
- get
213213
- list
214214
- watch
215+
- apiGroups:
216+
- apiserver.openshift.io
217+
resources:
218+
- apirequestcounts
219+
verbs:
220+
- get
221+
- list
222+
- watch
215223
---
216224
apiVersion: rbac.authorization.k8s.io/v1
217225
kind: ClusterRoleBinding

pkg/gatherers/conditional/conditional_gatherer.go

+28-9
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
var gatheringFunctionBuilders = map[GatheringFunctionName]GathererFunctionBuilderPtr{
2828
GatherLogsOfNamespace: (*Gatherer).BuildGatherLogsOfNamespace,
2929
GatherImageStreamsOfNamespace: (*Gatherer).BuildGatherImageStreamsOfNamespace,
30+
GatherAPIRequestCounts: (*Gatherer).BuildGatherAPIRequestCounts,
3031
}
3132

3233
// gatheringRules contains all the rules used to run conditional gatherings.
@@ -77,6 +78,21 @@ var defaultGatheringRules = []GatheringRule{
7778
},
7879
},
7980
},
81+
{
82+
Conditions: []ConditionWithParams{
83+
{
84+
Type: AlertIsFiring,
85+
Params: AlertIsFiringConditionParams{
86+
Name: "APIRemovedInNextEUSReleaseInUse",
87+
},
88+
},
89+
},
90+
GatheringFunctions: GatheringFunctions{
91+
GatherAPIRequestCounts: GatherAPIRequestCountsParams{
92+
AlertName: "APIRemovedInNextEUSReleaseInUse",
93+
},
94+
},
95+
},
8096
}
8197

8298
const canConditionalGathererFail = false
@@ -86,8 +102,9 @@ type Gatherer struct {
86102
gatherProtoKubeConfig *rest.Config
87103
metricsGatherKubeConfig *rest.Config
88104
imageKubeConfig *rest.Config
89-
firingAlerts map[string]bool // golang doesn't have sets :(
90-
gatheringRules []GatheringRule
105+
// there can be multiple instances of the same alert
106+
firingAlerts map[string][]AlertLabels
107+
gatheringRules []GatheringRule
91108
}
92109

93110
// New creates a new instance of conditional gatherer with the appropriate configs
@@ -138,7 +155,6 @@ func (g *Gatherer) GetGatheringFunctions(ctx context.Context) (map[string]gather
138155
if err != nil {
139156
return nil, err
140157
}
141-
142158
if allConditionsAreSatisfied {
143159
functions, errs := g.createGatheringClosures(conditionalGathering.GatheringFunctions)
144160
if len(errs) > 0 {
@@ -206,7 +222,7 @@ func (g *Gatherer) updateAlertsCache(ctx context.Context) error {
206222
func (g *Gatherer) updateAlertsCacheFromClient(ctx context.Context, metricsClient rest.Interface) error {
207223
const logPrefix = "conditional gatherer: "
208224

209-
g.firingAlerts = make(map[string]bool)
225+
g.firingAlerts = make(map[string][]AlertLabels)
210226

211227
data, err := metricsClient.Get().AbsPath("federate").
212228
Param("match[]", `ALERTS{alertstate="firing"}`).
@@ -237,17 +253,20 @@ func (g *Gatherer) updateAlertsCacheFromClient(ctx context.Context, metricsClien
237253
klog.Info(logPrefix + "metric is nil")
238254
continue
239255
}
240-
256+
alertLabels := make(map[string]string)
241257
for _, label := range metric.GetLabel() {
242258
if label == nil {
243259
klog.Info(logPrefix + "label is nil")
244260
continue
245261
}
246-
247-
if label.GetName() == "alertname" {
248-
g.firingAlerts[label.GetValue()] = true
249-
}
262+
alertLabels[label.GetName()] = label.GetValue()
263+
}
264+
alertName, ok := alertLabels["alertname"]
265+
if !ok {
266+
klog.Warningf("%s can't find \"alertname\" label in the metric: %v", logPrefix, metric)
267+
continue
250268
}
269+
g.firingAlerts[alertName] = append(g.firingAlerts[alertName], alertLabels)
251270
}
252271

253272
return nil

pkg/gatherers/conditional/conditional_gatherer_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ func Test_Gatherer_GatherConditionalGathererRules(t *testing.T) {
166166
err = json.Unmarshal(item, &gotGatheringRules)
167167
assert.NoError(t, err)
168168

169-
assert.Len(t, gotGatheringRules, 1)
169+
assert.Len(t, gotGatheringRules, 2)
170170
}
171171

172172
func newFakeClientWithMetrics(metrics string) *fake.RESTClient {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package conditional
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
"github.com/openshift/insights-operator/pkg/gatherers"
8+
"github.com/openshift/insights-operator/pkg/record"
9+
"github.com/openshift/insights-operator/pkg/utils"
10+
"k8s.io/apimachinery/pkg/api/errors"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/apimachinery/pkg/runtime/schema"
13+
"k8s.io/client-go/dynamic"
14+
)
15+
16+
// APIRequestCount defines a type used when marshaling into JSON
17+
type APIRequestCount struct {
18+
ResourceName string `json:"resource"`
19+
RemovedInRelease string `json:"removed_in_release"`
20+
TotalRequestCount int64 `json:"total_request_count"`
21+
LastDayRequestCount int64 `json:"last_day_request_count"`
22+
}
23+
24+
// BuildGatherApiRequestCounts creates a gathering closure which collects API requests counts for the
25+
// resources mentioned in the alert provided as a string parameter
26+
// Params is of type AlertIsFiringConditionParams:
27+
// - alert_name string - name of the firing alert
28+
//
29+
// * Location in archive: conditional/alerts/<alert_name>/api_request_counts.json
30+
// * Since versions:
31+
// * 4.10+
32+
func (g *Gatherer) BuildGatherAPIRequestCounts(paramsInterface interface{}) (gatherers.GatheringClosure, error) {
33+
params, ok := paramsInterface.(GatherAPIRequestCountsParams)
34+
if !ok {
35+
return gatherers.GatheringClosure{}, fmt.Errorf(
36+
"unexpected type in paramsInterface, expected %T, got %T",
37+
GatherAPIRequestCountsParams{}, paramsInterface,
38+
)
39+
}
40+
41+
return gatherers.GatheringClosure{
42+
Run: func(ctx context.Context) ([]record.Record, []error) {
43+
dynamicClient, err := dynamic.NewForConfig(g.gatherProtoKubeConfig)
44+
if err != nil {
45+
return nil, []error{err}
46+
}
47+
records, errs := g.gatherAPIRequestCounts(ctx, dynamicClient, params.AlertName)
48+
if errs != nil {
49+
return records, errs
50+
}
51+
return records, nil
52+
},
53+
CanFail: canConditionalGathererFail,
54+
}, nil
55+
}
56+
57+
func (g *Gatherer) gatherAPIRequestCounts(ctx context.Context,
58+
dynamicClient dynamic.Interface, alertName string) ([]record.Record, []error) {
59+
resources := make(map[string]struct{})
60+
for _, labels := range g.firingAlerts[alertName] {
61+
resourceName := fmt.Sprintf("%s.%s.%s", labels["resource"], labels["version"], labels["group"])
62+
resources[resourceName] = struct{}{}
63+
}
64+
65+
gvr := schema.GroupVersionResource{Group: "apiserver.openshift.io", Version: "v1", Resource: "apirequestcounts"}
66+
apiReqCountsList, err := dynamicClient.Resource(gvr).List(ctx, metav1.ListOptions{})
67+
if errors.IsNotFound(err) {
68+
return nil, nil
69+
}
70+
if err != nil {
71+
return nil, []error{err}
72+
}
73+
var records []record.Record
74+
var errrs []error
75+
var apiReqCounts []APIRequestCount
76+
for i := range apiReqCountsList.Items {
77+
it := apiReqCountsList.Items[i]
78+
79+
// filter only resources we're interested in
80+
if _, ok := resources[it.GetName()]; ok {
81+
totalReqCount, err := utils.NestedInt64Wrapper(it.Object, "status", "requestCount")
82+
if err != nil {
83+
errrs = append(errrs, err)
84+
}
85+
lastDayReqCount, err := utils.NestedInt64Wrapper(it.Object, "status", "currentHour", "requestCount")
86+
if err != nil {
87+
errrs = append(errrs, err)
88+
}
89+
removedInRel, err := utils.NestedStringWrapper(it.Object, "status", "removedInRelease")
90+
if err != nil {
91+
errrs = append(errrs, err)
92+
}
93+
apiReqCount := APIRequestCount{
94+
TotalRequestCount: totalReqCount,
95+
LastDayRequestCount: lastDayReqCount,
96+
ResourceName: it.GetName(),
97+
RemovedInRelease: removedInRel,
98+
}
99+
apiReqCounts = append(apiReqCounts, apiReqCount)
100+
}
101+
}
102+
records = append(records, record.Record{
103+
Name: fmt.Sprintf("%v/alerts/%s/api_request_counts", g.GetName(), alertName),
104+
Item: record.JSONMarshaller{Object: apiReqCounts},
105+
})
106+
return records, errrs
107+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
package conditional
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
"github.com/openshift/insights-operator/pkg/record"
8+
"github.com/stretchr/testify/assert"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
11+
"k8s.io/apimachinery/pkg/runtime"
12+
"k8s.io/apimachinery/pkg/runtime/schema"
13+
"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
14+
"k8s.io/client-go/dynamic"
15+
dynamicfake "k8s.io/client-go/dynamic/fake"
16+
)
17+
18+
var apiRequestCountYAML1 = `
19+
apiVersion: apiserver.openshift.io/v1
20+
kind: APIRequestCount
21+
metadata:
22+
name: test1.v1beta2.testapi.org
23+
status:
24+
currentHour:
25+
requestCount: 12
26+
requestCount: 13
27+
removedInRelease: "14.15"
28+
`
29+
30+
var apiRequestCountYAML2 = `
31+
apiVersion: apiserver.openshift.io/v1
32+
kind: APIRequestCount
33+
metadata:
34+
name: test2.v1beta1.testapi.org
35+
status:
36+
currentHour:
37+
requestCount: 2
38+
requestCount: 3
39+
removedInRelease: "1.123"
40+
`
41+
42+
func Test_GatherAPIRequestCount(t *testing.T) {
43+
gatherer := Gatherer{
44+
firingAlerts: map[string][]AlertLabels{
45+
"alertA": {
46+
{
47+
"alertname": "alertA",
48+
"resource": "test1",
49+
"group": "testapi.org",
50+
"version": "v1beta2",
51+
},
52+
{
53+
"alertname": "alertA",
54+
"resource": "test2",
55+
"group": "testapi.org",
56+
"version": "v1beta1",
57+
},
58+
},
59+
},
60+
}
61+
62+
gvr := schema.GroupVersionResource{Group: "apiserver.openshift.io", Version: "v1", Resource: "apirequestcounts"}
63+
client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{
64+
gvr: "APIRequestCountsList",
65+
})
66+
createResource(t, apiRequestCountYAML1, client, gvr)
67+
createResource(t, apiRequestCountYAML2, client, gvr)
68+
records, errs := gatherer.gatherAPIRequestCounts(context.Background(), client, "alertA")
69+
assert.Empty(t, errs, "Unexpected errors during gathering of API request counts")
70+
assert.Len(t, records, 1, "Unexpected number of records")
71+
72+
// check gathered data
73+
a, ok := records[0].Item.(record.JSONMarshaller).Object.([]APIRequestCount)
74+
assert.True(t, ok, "Failed to convert")
75+
assert.Len(t, a, 2, "Unexpected number of alerts")
76+
assert.Equal(t, a[0].ResourceName, "test1.v1beta2.testapi.org")
77+
assert.Equal(t, a[0].LastDayRequestCount, int64(12))
78+
assert.Equal(t, a[0].TotalRequestCount, int64(13))
79+
assert.Equal(t, a[0].RemovedInRelease, "14.15")
80+
81+
assert.Equal(t, a[1].ResourceName, "test2.v1beta1.testapi.org")
82+
assert.Equal(t, a[1].LastDayRequestCount, int64(2))
83+
assert.Equal(t, a[1].TotalRequestCount, int64(3))
84+
assert.Equal(t, a[1].RemovedInRelease, "1.123")
85+
}
86+
87+
func createResource(t *testing.T, yamlDef string, client dynamic.Interface, gvr schema.GroupVersionResource) {
88+
decUnstructured := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
89+
90+
testAPIRequestCountObj := &unstructured.Unstructured{}
91+
92+
_, _, err := decUnstructured.Decode([]byte(yamlDef), nil, testAPIRequestCountObj)
93+
assert.NoError(t, err, "Failed to decode API request count YAML definition")
94+
95+
_, err = client.Resource(gvr).Create(context.Background(), testAPIRequestCountObj, metav1.CreateOptions{})
96+
assert.NoError(t, err, "Failed to create fake API request count resource")
97+
}

pkg/gatherers/conditional/gathering_functions.go

+21-6
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,19 @@ type GatheringFunctions = map[GatheringFunctionName]interface{}
1313
// GatheringFunctionName defines functions of conditional gatherer
1414
type GatheringFunctionName string
1515

16-
// GatherLogsOfNamespace is a function collecting logs of the provided namespace.
17-
// See file gather_logs_of_namespace.go
18-
const GatherLogsOfNamespace GatheringFunctionName = "logs_of_namespace"
16+
const (
17+
// GatherLogsOfNamespace is a function collecting logs of the provided namespace.
18+
// See file gather_logs_of_namespace.go
19+
GatherLogsOfNamespace GatheringFunctionName = "logs_of_namespace"
1920

20-
// GatherImageStreamsOfNamespace is a function collecting image streams of the provided namespace.
21-
// See file gather_image_streams_of_namespace.go
22-
const GatherImageStreamsOfNamespace GatheringFunctionName = "image_streams_of_namespace"
21+
// GatherImageStreamsOfNamespace is a function collecting image streams of the provided namespace.
22+
// See file gather_image_streams_of_namespace.go
23+
GatherImageStreamsOfNamespace GatheringFunctionName = "image_streams_of_namespace"
24+
25+
// GatherAPIRequestCounts is a function collecting api request counts for the resources read
26+
// from the corresponding alert
27+
GatherAPIRequestCounts GatheringFunctionName = "api_request_counts_of_resource_from_alert"
28+
)
2329

2430
func (name GatheringFunctionName) NewParams(jsonParams []byte) (interface{}, error) {
2531
switch name {
@@ -31,6 +37,10 @@ func (name GatheringFunctionName) NewParams(jsonParams []byte) (interface{}, err
3137
var result GatherImageStreamsOfNamespaceParams
3238
err := json.Unmarshal(jsonParams, &result)
3339
return result, err
40+
case GatherAPIRequestCounts:
41+
var params GatherAPIRequestCountsParams
42+
err := json.Unmarshal(jsonParams, &params)
43+
return params, err
3444
}
3545
return nil, fmt.Errorf("unable to create params for %T: %v", name, name)
3646
}
@@ -50,3 +60,8 @@ type GatherImageStreamsOfNamespaceParams struct {
5060
// Namespace from which to collect image streams
5161
Namespace string `json:"namespace"`
5262
}
63+
64+
// GatherAPIRequestCountsParams defines parameters for api_request_counts gatherer
65+
type GatherAPIRequestCountsParams struct {
66+
AlertName string `json:"alert_name"`
67+
}

0 commit comments

Comments
 (0)