diff --git a/docs/gathered-data.md b/docs/gathered-data.md index fc94390fd..79cf58563 100644 --- a/docs/gathered-data.md +++ b/docs/gathered-data.md @@ -1,5 +1,17 @@ This document is auto-generated by `make docs` +## APIRequestCounts + +BuildGatherApiRequestCounts creates a gathering closure which collects API requests counts for the +resources mentioned in the alert provided as a string parameter +Params is of type AlertIsFiringConditionParams: + - alert_name string - name of the firing alert + +* Location in archive: conditional/alerts//api_request_counts.json +* Since versions: + * 4.10+ + + ## CRD collects the specified Custom Resource Definitions. diff --git a/docs/insights-archive-sample/conditional/api_request_counts.json b/docs/insights-archive-sample/conditional/api_request_counts.json new file mode 100644 index 000000000..3fce02bbe --- /dev/null +++ b/docs/insights-archive-sample/conditional/api_request_counts.json @@ -0,0 +1,8 @@ +[ + { + "resource": "customresourcedefinitions.v1beta1.apiextensions.k8s.io", + "removed_in_release": "1.22", + "total_request_count": 46, + "last_day_request_count": 0 + } +] \ No newline at end of file diff --git a/manifests/03-clusterrole.yaml b/manifests/03-clusterrole.yaml index 9c15b839b..03aa30c44 100644 --- a/manifests/03-clusterrole.yaml +++ b/manifests/03-clusterrole.yaml @@ -212,6 +212,14 @@ rules: - get - list - watch + - apiGroups: + - apiserver.openshift.io + resources: + - apirequestcounts + verbs: + - get + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/pkg/gatherers/conditional/conditional_gatherer.go b/pkg/gatherers/conditional/conditional_gatherer.go index 3ba6e7d9b..57ef1cb6d 100644 --- a/pkg/gatherers/conditional/conditional_gatherer.go +++ b/pkg/gatherers/conditional/conditional_gatherer.go @@ -27,6 +27,7 @@ import ( var gatheringFunctionBuilders = map[GatheringFunctionName]GathererFunctionBuilderPtr{ GatherLogsOfNamespace: (*Gatherer).BuildGatherLogsOfNamespace, GatherImageStreamsOfNamespace: (*Gatherer).BuildGatherImageStreamsOfNamespace, + GatherAPIRequestCounts: (*Gatherer).BuildGatherAPIRequestCounts, } // gatheringRules contains all the rules used to run conditional gatherings. @@ -77,6 +78,21 @@ var defaultGatheringRules = []GatheringRule{ }, }, }, + { + Conditions: []ConditionWithParams{ + { + Type: AlertIsFiring, + Params: AlertIsFiringConditionParams{ + Name: "APIRemovedInNextEUSReleaseInUse", + }, + }, + }, + GatheringFunctions: GatheringFunctions{ + GatherAPIRequestCounts: GatherAPIRequestCountsParams{ + AlertName: "APIRemovedInNextEUSReleaseInUse", + }, + }, + }, } const canConditionalGathererFail = false @@ -86,8 +102,9 @@ type Gatherer struct { gatherProtoKubeConfig *rest.Config metricsGatherKubeConfig *rest.Config imageKubeConfig *rest.Config - firingAlerts map[string]bool // golang doesn't have sets :( - gatheringRules []GatheringRule + // there can be multiple instances of the same alert + firingAlerts map[string][]AlertLabels + gatheringRules []GatheringRule } // New creates a new instance of conditional gatherer with the appropriate configs @@ -138,7 +155,6 @@ func (g *Gatherer) GetGatheringFunctions(ctx context.Context) (map[string]gather if err != nil { return nil, err } - if allConditionsAreSatisfied { functions, errs := g.createGatheringClosures(conditionalGathering.GatheringFunctions) if len(errs) > 0 { @@ -206,7 +222,7 @@ func (g *Gatherer) updateAlertsCache(ctx context.Context) error { func (g *Gatherer) updateAlertsCacheFromClient(ctx context.Context, metricsClient rest.Interface) error { const logPrefix = "conditional gatherer: " - g.firingAlerts = make(map[string]bool) + g.firingAlerts = make(map[string][]AlertLabels) data, err := metricsClient.Get().AbsPath("federate"). Param("match[]", `ALERTS{alertstate="firing"}`). @@ -237,17 +253,20 @@ func (g *Gatherer) updateAlertsCacheFromClient(ctx context.Context, metricsClien klog.Info(logPrefix + "metric is nil") continue } - + alertLabels := make(map[string]string) for _, label := range metric.GetLabel() { if label == nil { klog.Info(logPrefix + "label is nil") continue } - - if label.GetName() == "alertname" { - g.firingAlerts[label.GetValue()] = true - } + alertLabels[label.GetName()] = label.GetValue() + } + alertName, ok := alertLabels["alertname"] + if !ok { + klog.Warningf("%s can't find \"alertname\" label in the metric: %v", logPrefix, metric) + continue } + g.firingAlerts[alertName] = append(g.firingAlerts[alertName], alertLabels) } return nil diff --git a/pkg/gatherers/conditional/conditional_gatherer_test.go b/pkg/gatherers/conditional/conditional_gatherer_test.go index 43d2d5c20..d0ca4eabf 100644 --- a/pkg/gatherers/conditional/conditional_gatherer_test.go +++ b/pkg/gatherers/conditional/conditional_gatherer_test.go @@ -166,7 +166,7 @@ func Test_Gatherer_GatherConditionalGathererRules(t *testing.T) { err = json.Unmarshal(item, &gotGatheringRules) assert.NoError(t, err) - assert.Len(t, gotGatheringRules, 1) + assert.Len(t, gotGatheringRules, 2) } func newFakeClientWithMetrics(metrics string) *fake.RESTClient { diff --git a/pkg/gatherers/conditional/gather_api_request_count.go b/pkg/gatherers/conditional/gather_api_request_count.go new file mode 100644 index 000000000..11b15326f --- /dev/null +++ b/pkg/gatherers/conditional/gather_api_request_count.go @@ -0,0 +1,107 @@ +package conditional + +import ( + "context" + "fmt" + + "github.com/openshift/insights-operator/pkg/gatherers" + "github.com/openshift/insights-operator/pkg/record" + "github.com/openshift/insights-operator/pkg/utils" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" +) + +// APIRequestCount defines a type used when marshaling into JSON +type APIRequestCount struct { + ResourceName string `json:"resource"` + RemovedInRelease string `json:"removed_in_release"` + TotalRequestCount int64 `json:"total_request_count"` + LastDayRequestCount int64 `json:"last_day_request_count"` +} + +// BuildGatherApiRequestCounts creates a gathering closure which collects API requests counts for the +// resources mentioned in the alert provided as a string parameter +// Params is of type AlertIsFiringConditionParams: +// - alert_name string - name of the firing alert +// +// * Location in archive: conditional/alerts//api_request_counts.json +// * Since versions: +// * 4.10+ +func (g *Gatherer) BuildGatherAPIRequestCounts(paramsInterface interface{}) (gatherers.GatheringClosure, error) { + params, ok := paramsInterface.(GatherAPIRequestCountsParams) + if !ok { + return gatherers.GatheringClosure{}, fmt.Errorf( + "unexpected type in paramsInterface, expected %T, got %T", + GatherAPIRequestCountsParams{}, paramsInterface, + ) + } + + return gatherers.GatheringClosure{ + Run: func(ctx context.Context) ([]record.Record, []error) { + dynamicClient, err := dynamic.NewForConfig(g.gatherProtoKubeConfig) + if err != nil { + return nil, []error{err} + } + records, errs := g.gatherAPIRequestCounts(ctx, dynamicClient, params.AlertName) + if errs != nil { + return records, errs + } + return records, nil + }, + CanFail: canConditionalGathererFail, + }, nil +} + +func (g *Gatherer) gatherAPIRequestCounts(ctx context.Context, + dynamicClient dynamic.Interface, alertName string) ([]record.Record, []error) { + resources := make(map[string]struct{}) + for _, labels := range g.firingAlerts[alertName] { + resourceName := fmt.Sprintf("%s.%s.%s", labels["resource"], labels["version"], labels["group"]) + resources[resourceName] = struct{}{} + } + + gvr := schema.GroupVersionResource{Group: "apiserver.openshift.io", Version: "v1", Resource: "apirequestcounts"} + apiReqCountsList, err := dynamicClient.Resource(gvr).List(ctx, metav1.ListOptions{}) + if errors.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, []error{err} + } + var records []record.Record + var errrs []error + var apiReqCounts []APIRequestCount + for i := range apiReqCountsList.Items { + it := apiReqCountsList.Items[i] + + // filter only resources we're interested in + if _, ok := resources[it.GetName()]; ok { + totalReqCount, err := utils.NestedInt64Wrapper(it.Object, "status", "requestCount") + if err != nil { + errrs = append(errrs, err) + } + lastDayReqCount, err := utils.NestedInt64Wrapper(it.Object, "status", "currentHour", "requestCount") + if err != nil { + errrs = append(errrs, err) + } + removedInRel, err := utils.NestedStringWrapper(it.Object, "status", "removedInRelease") + if err != nil { + errrs = append(errrs, err) + } + apiReqCount := APIRequestCount{ + TotalRequestCount: totalReqCount, + LastDayRequestCount: lastDayReqCount, + ResourceName: it.GetName(), + RemovedInRelease: removedInRel, + } + apiReqCounts = append(apiReqCounts, apiReqCount) + } + } + records = append(records, record.Record{ + Name: fmt.Sprintf("%v/alerts/%s/api_request_counts", g.GetName(), alertName), + Item: record.JSONMarshaller{Object: apiReqCounts}, + }) + return records, errrs +} diff --git a/pkg/gatherers/conditional/gather_api_request_count_test.go b/pkg/gatherers/conditional/gather_api_request_count_test.go new file mode 100644 index 000000000..99acf6e42 --- /dev/null +++ b/pkg/gatherers/conditional/gather_api_request_count_test.go @@ -0,0 +1,97 @@ +package conditional + +import ( + "context" + "testing" + + "github.com/openshift/insights-operator/pkg/record" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + "k8s.io/client-go/dynamic" + dynamicfake "k8s.io/client-go/dynamic/fake" +) + +var apiRequestCountYAML1 = ` +apiVersion: apiserver.openshift.io/v1 +kind: APIRequestCount +metadata: + name: test1.v1beta2.testapi.org +status: + currentHour: + requestCount: 12 + requestCount: 13 + removedInRelease: "14.15" +` + +var apiRequestCountYAML2 = ` +apiVersion: apiserver.openshift.io/v1 +kind: APIRequestCount +metadata: + name: test2.v1beta1.testapi.org +status: + currentHour: + requestCount: 2 + requestCount: 3 + removedInRelease: "1.123" +` + +func Test_GatherAPIRequestCount(t *testing.T) { + gatherer := Gatherer{ + firingAlerts: map[string][]AlertLabels{ + "alertA": { + { + "alertname": "alertA", + "resource": "test1", + "group": "testapi.org", + "version": "v1beta2", + }, + { + "alertname": "alertA", + "resource": "test2", + "group": "testapi.org", + "version": "v1beta1", + }, + }, + }, + } + + gvr := schema.GroupVersionResource{Group: "apiserver.openshift.io", Version: "v1", Resource: "apirequestcounts"} + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ + gvr: "APIRequestCountsList", + }) + createResource(t, apiRequestCountYAML1, client, gvr) + createResource(t, apiRequestCountYAML2, client, gvr) + records, errs := gatherer.gatherAPIRequestCounts(context.Background(), client, "alertA") + assert.Empty(t, errs, "Unexpected errors during gathering of API request counts") + assert.Len(t, records, 1, "Unexpected number of records") + + // check gathered data + a, ok := records[0].Item.(record.JSONMarshaller).Object.([]APIRequestCount) + assert.True(t, ok, "Failed to convert") + assert.Len(t, a, 2, "Unexpected number of alerts") + assert.Equal(t, a[0].ResourceName, "test1.v1beta2.testapi.org") + assert.Equal(t, a[0].LastDayRequestCount, int64(12)) + assert.Equal(t, a[0].TotalRequestCount, int64(13)) + assert.Equal(t, a[0].RemovedInRelease, "14.15") + + assert.Equal(t, a[1].ResourceName, "test2.v1beta1.testapi.org") + assert.Equal(t, a[1].LastDayRequestCount, int64(2)) + assert.Equal(t, a[1].TotalRequestCount, int64(3)) + assert.Equal(t, a[1].RemovedInRelease, "1.123") +} + +func createResource(t *testing.T, yamlDef string, client dynamic.Interface, gvr schema.GroupVersionResource) { + decUnstructured := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + + testAPIRequestCountObj := &unstructured.Unstructured{} + + _, _, err := decUnstructured.Decode([]byte(yamlDef), nil, testAPIRequestCountObj) + assert.NoError(t, err, "Failed to decode API request count YAML definition") + + _, err = client.Resource(gvr).Create(context.Background(), testAPIRequestCountObj, metav1.CreateOptions{}) + assert.NoError(t, err, "Failed to create fake API request count resource") +} diff --git a/pkg/gatherers/conditional/gathering_functions.go b/pkg/gatherers/conditional/gathering_functions.go index 2259fe924..dac823e69 100644 --- a/pkg/gatherers/conditional/gathering_functions.go +++ b/pkg/gatherers/conditional/gathering_functions.go @@ -13,13 +13,19 @@ type GatheringFunctions = map[GatheringFunctionName]interface{} // GatheringFunctionName defines functions of conditional gatherer type GatheringFunctionName string -// GatherLogsOfNamespace is a function collecting logs of the provided namespace. -// See file gather_logs_of_namespace.go -const GatherLogsOfNamespace GatheringFunctionName = "logs_of_namespace" +const ( + // GatherLogsOfNamespace is a function collecting logs of the provided namespace. + // See file gather_logs_of_namespace.go + GatherLogsOfNamespace GatheringFunctionName = "logs_of_namespace" -// GatherImageStreamsOfNamespace is a function collecting image streams of the provided namespace. -// See file gather_image_streams_of_namespace.go -const GatherImageStreamsOfNamespace GatheringFunctionName = "image_streams_of_namespace" + // GatherImageStreamsOfNamespace is a function collecting image streams of the provided namespace. + // See file gather_image_streams_of_namespace.go + GatherImageStreamsOfNamespace GatheringFunctionName = "image_streams_of_namespace" + + // GatherAPIRequestCounts is a function collecting api request counts for the resources read + // from the corresponding alert + GatherAPIRequestCounts GatheringFunctionName = "api_request_counts_of_resource_from_alert" +) func (name GatheringFunctionName) NewParams(jsonParams []byte) (interface{}, error) { switch name { @@ -31,6 +37,10 @@ func (name GatheringFunctionName) NewParams(jsonParams []byte) (interface{}, err var result GatherImageStreamsOfNamespaceParams err := json.Unmarshal(jsonParams, &result) return result, err + case GatherAPIRequestCounts: + var params GatherAPIRequestCountsParams + err := json.Unmarshal(jsonParams, ¶ms) + return params, err } return nil, fmt.Errorf("unable to create params for %T: %v", name, name) } @@ -50,3 +60,8 @@ type GatherImageStreamsOfNamespaceParams struct { // Namespace from which to collect image streams Namespace string `json:"namespace"` } + +// GatherAPIRequestCountsParams defines parameters for api_request_counts gatherer +type GatherAPIRequestCountsParams struct { + AlertName string `json:"alert_name"` +} diff --git a/pkg/gatherers/conditional/gathering_rule.schema.json b/pkg/gatherers/conditional/gathering_rule.schema.json index eb27780a7..a6b4c0bb9 100644 --- a/pkg/gatherers/conditional/gathering_rule.schema.json +++ b/pkg/gatherers/conditional/gathering_rule.schema.json @@ -120,6 +120,20 @@ "pattern": "^openshift-[a-zA-Z0-9_.-]{1,128}$" } } + }, + "^api_request_counts_of_resource_from_alert$": { + "type": "object", + "title": "GatherApiRequestCountsParams", + "required": [ + "alert_name" + ], + "properties": { + "alert_name": { + "type": "string", + "title": "AlertName", + "pattern": "^[a-zA-Z0-9_]{1,128}$" + } + } } } } diff --git a/pkg/gatherers/conditional/types.go b/pkg/gatherers/conditional/types.go index 8f9804278..c9a6b978d 100644 --- a/pkg/gatherers/conditional/types.go +++ b/pkg/gatherers/conditional/types.go @@ -15,3 +15,6 @@ type GatheringRule struct { // GathererFunctionBuilderPtr defines a pointer to a gatherer function builder type GathererFunctionBuilderPtr = func(*Gatherer, interface{}) (gatherers.GatheringClosure, error) + +// AlertLabels defines alert labels as a string key/value pairs +type AlertLabels map[string]string diff --git a/pkg/utils/unstructerd_wrappers.go b/pkg/utils/unstructerd_wrappers.go index 184da559a..0095cbd17 100644 --- a/pkg/utils/unstructerd_wrappers.go +++ b/pkg/utils/unstructerd_wrappers.go @@ -31,6 +31,18 @@ func NestedSliceWrapper(obj map[string]interface{}, fields ...string) ([]interfa return s, nil } +func NestedInt64Wrapper(obj map[string]interface{}, fields ...string) (int64, error) { + i, ok, err := unstructured.NestedInt64(obj, fields...) + if !ok { + return 0, fmt.Errorf("can't find %s", formatSlice(fields...)) + } + if err != nil { + return 0, err + } + + return i, nil +} + func formatSlice(s ...string) string { var str string for _, f := range s {