Skip to content

Commit f9ac922

Browse files
author
Serhii Zakharov
authored
Conditional gathering (#433)
* refactoring and minor changes needed by conditional gathering * implemented conditional gathering * a few fixes after review * stop failing when metrics aren't available * a few fixes * calm down linters * calm down linters * fixes after review
1 parent c8e1838 commit f9ac922

29 files changed

+1166
-118
lines changed

cmd/gendoc/main.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ var (
2323
outPath string
2424
mdf *os.File
2525
randSource = rand.NewSource(time.Now().UnixNano())
26-
reGather = regexp.MustCompile(`^(Gather)(.*)`)
26+
reGather = regexp.MustCompile(`^((Build)?Gather)(.*)`)
2727
reExample = regexp.MustCompile(`^(Example)(.*)`)
2828
)
2929

@@ -137,9 +137,10 @@ func walkDir(cleanRoot string, md map[string]*DocBlock) error {
137137
// handle function declarations
138138
fn, ok := n.(*ast.FuncDecl)
139139
if ok {
140-
gatherMethodWithSuff := reGather.ReplaceAllString(fn.Name.Name, "$2")
140+
gatherMethodWithSuff := reGather.ReplaceAllString(fn.Name.Name, "$3")
141141
_, ok2 := md[gatherMethodWithSuff]
142-
if !ok2 && fn.Name.IsExported() && strings.HasPrefix(fn.Name.Name, "Gather") && len(fn.Name.Name) > len("Gather") {
142+
startsWithGatherOrBuildGather := strings.HasPrefix(fn.Name.Name, "Gather") || strings.HasPrefix(fn.Name.Name, "BuildGather")
143+
if !ok2 && fn.Name.IsExported() && startsWithGatherOrBuildGather && len(fn.Name.Name) > len("Gather") {
143144
doc := fn.Doc.Text()
144145
md[gatherMethodWithSuff] = parseDoc(fn.Name.Name, doc)
145146
fmt.Printf(fn.Name.Name + "\n")

docs/gathered-data.md

+37
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,11 @@ Response see https://docs.openshift.com/container-platform/4.3/rest_api/index.ht
195195
* Id in config: version
196196

197197

198+
## ConditionalGathererRules
199+
200+
stores the gathering rules in insights-operator/conditional-gatherer-rules.json
201+
202+
198203
## ConfigMaps
199204

200205
fetches the ConfigMaps from namespace openshift-config
@@ -264,6 +269,20 @@ Response see https://docs.openshift.com/container-platform/4.3/rest_api/index.ht
264269
* 4.6+
265270

266271

272+
## ImageStreamsOfNamespace
273+
274+
creates a gathering closure which collects image streams from the provided namespace
275+
Params:
276+
- namespace string - namespace from which to collect image streams
277+
278+
API reference:
279+
https://docs.openshift.com/container-platform/4.7/rest_api/image_apis/imagestream-image-openshift-io-v1.html#apisimage-openshift-iov1namespacesnamespaceimagestreams
280+
281+
* Location in archive: conditional/namespaces/{namespace}/imagestreams/{name}
282+
* Since versions:
283+
* 4.8+
284+
285+
267286
## InstallPlans
268287

269288
collects Top x InstallPlans from all openshift namespaces.
@@ -296,6 +315,24 @@ Response see:
296315
* 4.8+
297316

298317

318+
## LogsOfNamespace
319+
320+
creates a gathering closure which collects logs from pods in the provided namespace
321+
Params:
322+
- namespace string - namespace from which to collect image streams
323+
- label_selector string - a label selector to filter some pods (default to all pods)
324+
- tail_lines int64 - a number of log lines to keep for each container
325+
326+
The Kubernetes API:
327+
https://github.com/kubernetes/client-go/blob/master/kubernetes/typed/core/v1/pod_expansion.go#L48
328+
Response see:
329+
https://docs.openshift.com/container-platform/4.6/rest_api/workloads_apis/pod-core-v1.html#apiv1namespacesnamespacepodsnamelog
330+
331+
* Location in archive: conditional/namespaces/{namespace}/pods/{pod_name}/containers/{container_name}/logs/last-{n}-lines.log
332+
* Since versions:
333+
* 4.8+
334+
335+
299336
## MachineConfigPool
300337

301338
collects MachineConfigPool information
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"metadata": {
3+
"name": "example",
4+
"namespace": "openshift-cluster-samples-operator",
5+
"selfLink": "/apis/image.openshift.io/v1/namespaces/openshift-cluster-samples-operator/imagestreams/example",
6+
"uid": "bd11bcb8-b0be-4605-9f11-9697e84897f9",
7+
"resourceVersion": "79100",
8+
"generation": 1,
9+
"creationTimestamp": "2021-05-24T16:40:22Z"
10+
},
11+
"spec": {
12+
"lookupPolicy": {
13+
"local": false
14+
}
15+
},
16+
"status": {
17+
"dockerImageRepository": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
18+
}
19+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
I0524 13:15:17.126969 28 cmd.go:331] Waiting for process with process name "cluster-samples-operator" ...
2+
I0524 13:15:17.127720 28 cmd.go:341] Watching for changes in: ([]string) (len=2 cap=2) {
3+
(string) (len=33) "/proc/14/root/etc/secrets/tls.crt",
4+
(string) (len=33) "/proc/14/root/etc/secrets/tls.key"
5+
}
6+
I0524 13:15:17.136324 28 observer_polling.go:155] Starting file observer
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
time="2021-05-24T16:37:58Z" level=info msg="processing file processserver64-postgresql-persistent-s2i.json from dir /opt/openshift/operator/x86_64/processserver/templates"
2+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir python from dir /opt/openshift/operator/x86_64"
3+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir imagestreams from dir /opt/openshift/operator/x86_64/python"
4+
time="2021-05-24T16:37:58Z" level=info msg="processing file python-rhel.json from dir /opt/openshift/operator/x86_64/python/imagestreams"
5+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir rails from dir /opt/openshift/operator/x86_64"
6+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir templates from dir /opt/openshift/operator/x86_64/rails"
7+
time="2021-05-24T16:37:58Z" level=info msg="processing file rails-pgsql-persistent.json from dir /opt/openshift/operator/x86_64/rails/templates"
8+
time="2021-05-24T16:37:58Z" level=info msg="processing file rails-postgresql-example.json from dir /opt/openshift/operator/x86_64/rails/templates"
9+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir redis from dir /opt/openshift/operator/x86_64"
10+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir imagestreams from dir /opt/openshift/operator/x86_64/redis"
11+
time="2021-05-24T16:37:58Z" level=info msg="processing file redis-rhel.json from dir /opt/openshift/operator/x86_64/redis/imagestreams"
12+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir templates from dir /opt/openshift/operator/x86_64/redis"
13+
time="2021-05-24T16:37:58Z" level=info msg="processing file redis-ephemeral.json from dir /opt/openshift/operator/x86_64/redis/templates"
14+
time="2021-05-24T16:37:58Z" level=info msg="processing file redis-persistent.json from dir /opt/openshift/operator/x86_64/redis/templates"
15+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir rhdm from dir /opt/openshift/operator/x86_64"
16+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir imagestreams from dir /opt/openshift/operator/x86_64/rhdm"
17+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhdm-decisioncentral-rhel8.json from dir /opt/openshift/operator/x86_64/rhdm/imagestreams"
18+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhdm-kieserver-rhel8.json from dir /opt/openshift/operator/x86_64/rhdm/imagestreams"
19+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir templates from dir /opt/openshift/operator/x86_64/rhdm"
20+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhdm78-authoring-ha.json from dir /opt/openshift/operator/x86_64/rhdm/templates"
21+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhdm78-authoring.json from dir /opt/openshift/operator/x86_64/rhdm/templates"
22+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhdm78-kieserver.json from dir /opt/openshift/operator/x86_64/rhdm/templates"
23+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhdm78-prod-immutable-kieserver-amq.json from dir /opt/openshift/operator/x86_64/rhdm/templates"
24+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhdm78-prod-immutable-kieserver.json from dir /opt/openshift/operator/x86_64/rhdm/templates"
25+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhdm78-trial-ephemeral.json from dir /opt/openshift/operator/x86_64/rhdm/templates"
26+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir rhpam from dir /opt/openshift/operator/x86_64"
27+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir imagestreams from dir /opt/openshift/operator/x86_64/rhpam"
28+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam-businesscentral-monitoring-rhel8.json from dir /opt/openshift/operator/x86_64/rhpam/imagestreams"
29+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam-businesscentral-rhel8.json from dir /opt/openshift/operator/x86_64/rhpam/imagestreams"
30+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam-kieserver-rhel8.json from dir /opt/openshift/operator/x86_64/rhpam/imagestreams"
31+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam-smartrouter-rhel8.json from dir /opt/openshift/operator/x86_64/rhpam/imagestreams"
32+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir templates from dir /opt/openshift/operator/x86_64/rhpam"
33+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-authoring-ha.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
34+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-authoring.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
35+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-kieserver-externaldb.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
36+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-kieserver-mysql.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
37+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-kieserver-postgresql.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
38+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-managed.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
39+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-prod-immutable-kieserver-amq.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
40+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-prod-immutable-kieserver.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
41+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-prod-immutable-monitor.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
42+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-prod.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
43+
time="2021-05-24T16:37:58Z" level=info msg="processing file rhpam78-trial-ephemeral.json from dir /opt/openshift/operator/x86_64/rhpam/templates"
44+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir ruby from dir /opt/openshift/operator/x86_64"
45+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir imagestreams from dir /opt/openshift/operator/x86_64/ruby"
46+
time="2021-05-24T16:37:58Z" level=info msg="processing file ruby-rhel.json from dir /opt/openshift/operator/x86_64/ruby/imagestreams"
47+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir sso from dir /opt/openshift/operator/x86_64"
48+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir imagestreams from dir /opt/openshift/operator/x86_64/sso"
49+
time="2021-05-24T16:37:58Z" level=info msg="processing file redhat-sso70-openshift-rhel7.json from dir /opt/openshift/operator/x86_64/sso/imagestreams"
50+
time="2021-05-24T16:37:58Z" level=info msg="processing file redhat-sso71-openshift-rhel7.json from dir /opt/openshift/operator/x86_64/sso/imagestreams"
51+
time="2021-05-24T16:37:58Z" level=info msg="processing file redhat-sso72-openshift-rhel7.json from dir /opt/openshift/operator/x86_64/sso/imagestreams"
52+
time="2021-05-24T16:37:58Z" level=info msg="processing file redhat-sso73-openshift-rhel7.json from dir /opt/openshift/operator/x86_64/sso/imagestreams"
53+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso74-openshift-rhel8.json from dir /opt/openshift/operator/x86_64/sso/imagestreams"
54+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir templates from dir /opt/openshift/operator/x86_64/sso"
55+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso72-https.json from dir /opt/openshift/operator/x86_64/sso/templates"
56+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso72-mysql-persistent.json from dir /opt/openshift/operator/x86_64/sso/templates"
57+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso72-mysql.json from dir /opt/openshift/operator/x86_64/sso/templates"
58+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso72-postgresql-persistent.json from dir /opt/openshift/operator/x86_64/sso/templates"
59+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso72-postgresql.json from dir /opt/openshift/operator/x86_64/sso/templates"
60+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso73-https.json from dir /opt/openshift/operator/x86_64/sso/templates"
61+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso73-mysql-persistent.json from dir /opt/openshift/operator/x86_64/sso/templates"
62+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso73-mysql.json from dir /opt/openshift/operator/x86_64/sso/templates"
63+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso73-ocp4-x509-https.json from dir /opt/openshift/operator/x86_64/sso/templates"
64+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso73-ocp4-x509-mysql-persistent.json from dir /opt/openshift/operator/x86_64/sso/templates"
65+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso73-ocp4-x509-postgresql-persistent.json from dir /opt/openshift/operator/x86_64/sso/templates"
66+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso73-postgresql-persistent.json from dir /opt/openshift/operator/x86_64/sso/templates"
67+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso73-postgresql.json from dir /opt/openshift/operator/x86_64/sso/templates"
68+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso74-https.json from dir /opt/openshift/operator/x86_64/sso/templates"
69+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso74-ocp4-x509-https.json from dir /opt/openshift/operator/x86_64/sso/templates"
70+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso74-ocp4-x509-postgresql-persistent.json from dir /opt/openshift/operator/x86_64/sso/templates"
71+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso74-postgresql-persistent.json from dir /opt/openshift/operator/x86_64/sso/templates"
72+
time="2021-05-24T16:37:58Z" level=info msg="processing file sso74-postgresql.json from dir /opt/openshift/operator/x86_64/sso/templates"
73+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir webserver from dir /opt/openshift/operator/x86_64"
74+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir imagestreams from dir /opt/openshift/operator/x86_64/webserver"
75+
time="2021-05-24T16:37:58Z" level=info msg="processing file jboss-webserver31-tomcat7-openshift-rhel7.json from dir /opt/openshift/operator/x86_64/webserver/imagestreams"
76+
time="2021-05-24T16:37:58Z" level=info msg="processing file jboss-webserver31-tomcat8-openshift-rhel7.json from dir /opt/openshift/operator/x86_64/webserver/imagestreams"
77+
time="2021-05-24T16:37:58Z" level=info msg="processing file jboss-webserver53-openjdk11-tomcat9-openshift-rhel7.json from dir /opt/openshift/operator/x86_64/webserver/imagestreams"
78+
time="2021-05-24T16:37:58Z" level=info msg="processing file jboss-webserver53-openjdk8-tomcat9-openshift-rhel7.json from dir /opt/openshift/operator/x86_64/webserver/imagestreams"
79+
time="2021-05-24T16:37:58Z" level=info msg="processing subdir templates from dir /opt/openshift/operator/x86_64/webserver"
80+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat7-basic-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
81+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat7-https-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
82+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat7-mysql-persistent-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
83+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat7-mysql-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
84+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat7-postgresql-persistent-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
85+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat7-postgresql-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
86+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat8-basic-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
87+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat8-https-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
88+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat8-mysql-persistent-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
89+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat8-mysql-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
90+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws31-tomcat8-postgresql-persistent-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
91+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk11-tomcat9-basic-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
92+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk11-tomcat9-https-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
93+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk11-tomcat9-mysql-persistent-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
94+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk11-tomcat9-mysql-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
95+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk11-tomcat9-postgresql-persistent-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
96+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk8-tomcat9-basic-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
97+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk8-tomcat9-https-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
98+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk8-tomcat9-mysql-persistent-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
99+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk8-tomcat9-mysql-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
100+
time="2021-05-24T16:37:58Z" level=info msg="processing file jws53-openjdk8-tomcat9-postgresql-persistent-s2i.json from dir /opt/openshift/operator/x86_64/webserver/templates"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[
2+
{
3+
"conditions": [
4+
{
5+
"type": "alert_is_firing",
6+
"params": {
7+
"name": "SamplesImagestreamImportFailing"
8+
}
9+
}
10+
],
11+
"gathering_functions": {
12+
"image_stream_definitions_of_namespace": {
13+
"namespace": "openshift-cluster-samples-operator"
14+
},
15+
"logs_of_namespace": {
16+
"label_selector": "",
17+
"namespace": "openshift-cluster-samples-operator",
18+
"tail_lines": 100
19+
}
20+
}
21+
}
22+
]

docs/insights-archive-sample/insights-operator/gathers.json

+21
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,27 @@
265265
"records_count": 1,
266266
"errors": null,
267267
"panic": null
268+
},
269+
{
270+
"name": "conditional/conditional_gatherer_rules",
271+
"duration_in_ms": 0,
272+
"records_count": 1,
273+
"errors": null,
274+
"panic": null
275+
},
276+
{
277+
"name": "conditional/image_stream_definitions_of_namespace/namespace=openshift-cluster-samples-operator",
278+
"duration_in_ms": 135,
279+
"records_count": 1,
280+
"errors": null,
281+
"panic": null
282+
},
283+
{
284+
"name": "conditional/logs_of_namespace/namespace=openshift-cluster-samples-operator,tail_lines=100",
285+
"duration_in_ms": 498,
286+
"records_count": 2,
287+
"errors": null,
288+
"panic": null
268289
}
269290
],
270291
"memory_alloc_bytes": 17411816,

0 commit comments

Comments
 (0)