Skip to content

Commit bbe7160

Browse files
openshift-merge-robotopenshift-merge-robot
authored
Merge pull request #309 from Sergey1011010/collect-logs-from-openshift-sdn-namespace
Bug 1914975: Collect logs from openshift-sdn namespace
2 parents f768b31 + 3996670 commit bbe7160

8 files changed

+253
-136
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
## 4.7
44

5+
- [#309](https://github.com/openshift/insights-operator/pull/309) Collect logs from openshift-sdn namespace
56
- [#279](https://github.com/openshift/insights-operator/pull/279) Refactoring record and gatherer
67
- [#297](https://github.com/openshift/insights-operator/pull/297) Gather netnamespaces network info
78
- [#292](https://github.com/openshift/insights-operator/pull/292) Update initial waiting times and give TestIsIOHealthy more time

‎docs/gathered-data.md

+15-1
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,21 @@ collects logs from openshift-apiserver-operator with following substrings:
314314
The Kubernetes API https://github.com/kubernetes/client-go/blob/master/kubernetes/typed/core/v1/pod_expansion.go#L48
315315
Response see https://docs.openshift.com/container-platform/4.6/rest_api/workloads_apis/pod-core-v1.html#apiv1namespacesnamespacepodsnamelog
316316

317-
Location in archive: config/pod/{namespace-name}/logs/{pod-name}/errors.log
317+
Location in archive: config/pod/openshift-apiserver-operator/logs/{pod-name}/errors.log
318+
319+
320+
## OpenshiftSDNLogs
321+
322+
collects logs from pods in openshift-sdn namespace with following substrings:
323+
- "Got OnEndpointsUpdate for unknown Endpoints",
324+
- "Got OnEndpointsDelete for unknown Endpoints",
325+
- "Unable to update proxy firewall for policy",
326+
- "Failed to update proxy firewall for policy",
327+
328+
The Kubernetes API https://github.com/kubernetes/client-go/blob/master/kubernetes/typed/core/v1/pod_expansion.go#L48
329+
Response see https://docs.openshift.com/container-platform/4.6/rest_api/workloads_apis/pod-core-v1.html#apiv1namespacesnamespacepodsnamelog
330+
331+
Location in archive: config/pod/openshift-sdn/logs/{pod-name}/errors.log
318332

319333

320334
## PodDisruptionBudgets

‎pkg/gather/clusterconfig/0_gatherer.go

+1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ var gatherFunctions = map[string]gatherFunction{
6565
"stateful_sets": GatherStatefulSets,
6666
"netnamespaces": GatherNetNamespace,
6767
"openshift_apiserver_operator_logs": GatherOpenShiftAPIServerOperatorLogs,
68+
"openshift_sdn_logs": GatherOpenshiftSDNLogs,
6869
}
6970

7071
// New creates new Gatherer

‎pkg/gather/clusterconfig/gather_logs.go

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package clusterconfig
2+
3+
import (
4+
"bufio"
5+
"context"
6+
"fmt"
7+
"strings"
8+
9+
corev1 "k8s.io/api/core/v1"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
12+
restclient "k8s.io/client-go/rest"
13+
"k8s.io/klog/v2"
14+
15+
"github.com/openshift/insights-operator/pkg/record"
16+
)
17+
18+
// gatherLogsFromPodsInNamespace collects logs from the pods in provided namespace
19+
// - messagesToSearch are the messages to filter the logs(case-insensitive)
20+
// - sinceSeconds sets the moment to fetch logs from(current time - sinceSeconds)
21+
// - limitBytes sets the maximum amount of logs that can be fetched
22+
// - logFileName sets the name of the file to save logs to.
23+
// - labelSelector allows you to filter pods by their labels
24+
// Actual location is `config/pod/{namespace}/logs/{podName}/{fileName}.log`
25+
func gatherLogsFromPodsInNamespace(
26+
ctx context.Context,
27+
coreClient v1.CoreV1Interface,
28+
namespace string,
29+
messagesToSearch []string,
30+
sinceSeconds int64,
31+
limitBytes int64,
32+
logFileName string,
33+
labelSelector string,
34+
) ([]record.Record, error) {
35+
pods, err := coreClient.Pods(namespace).List(ctx, metav1.ListOptions{
36+
LabelSelector: labelSelector,
37+
})
38+
if err != nil {
39+
return nil, err
40+
}
41+
42+
var records []record.Record
43+
44+
for _, pod := range pods.Items {
45+
for _, container := range pod.Spec.Containers {
46+
request := coreClient.Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
47+
Container: container.Name,
48+
SinceSeconds: &sinceSeconds,
49+
LimitBytes: &limitBytes,
50+
})
51+
52+
logs, err := filterLogs(ctx, request, messagesToSearch)
53+
if err != nil {
54+
return nil, err
55+
}
56+
57+
if len(strings.TrimSpace(logs)) != 0 {
58+
records = append(records, record.Record{
59+
Name: fmt.Sprintf("config/pod/%s/logs/%s/%s.log", pod.Namespace, pod.Name, logFileName),
60+
Item: Raw{logs},
61+
})
62+
}
63+
}
64+
}
65+
66+
if len(pods.Items) == 0 {
67+
klog.Infof("no pods in %v namespace were found", namespace)
68+
}
69+
70+
return records, nil
71+
}
72+
73+
func filterLogs(ctx context.Context, request *restclient.Request, messagesToSearch []string) (string, error) {
74+
stream, err := request.Stream(ctx)
75+
if err != nil {
76+
return "", err
77+
}
78+
79+
defer func() {
80+
err := stream.Close()
81+
if err != nil {
82+
klog.Errorf("error during closing a stream: %v", err)
83+
}
84+
}()
85+
86+
scanner := bufio.NewScanner(stream)
87+
88+
var result string
89+
90+
for scanner.Scan() {
91+
line := scanner.Text()
92+
for _, messageToSearch := range messagesToSearch {
93+
if strings.Contains(strings.ToLower(line), strings.ToLower(messageToSearch)) {
94+
result += line + "\n"
95+
}
96+
}
97+
}
98+
99+
if err := scanner.Err(); err != nil {
100+
return "", err
101+
}
102+
103+
return result, nil
104+
}

‎pkg/gather/clusterconfig/gather_logs_test.go

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package clusterconfig
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
corev1 "k8s.io/api/core/v1"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
kubefake "k8s.io/client-go/kubernetes/fake"
12+
)
13+
14+
func TestGatherLogs(t *testing.T) {
15+
const testPodNamespace = "pod-namespace"
16+
const testLogFileName = "errors"
17+
// there's no way to specify logs fake pod will have, so we can only search for a hardcoded string "fake logs"
18+
const stringToSearch = "fake logs"
19+
20+
coreClient := kubefake.NewSimpleClientset().CoreV1()
21+
ctx := context.Background()
22+
23+
_, err := coreClient.Pods(testPodNamespace).Create(
24+
ctx,
25+
&corev1.Pod{
26+
ObjectMeta: metav1.ObjectMeta{
27+
Name: testPodNamespace,
28+
Namespace: testPodNamespace,
29+
},
30+
Status: corev1.PodStatus{
31+
Phase: corev1.PodRunning,
32+
ContainerStatuses: []corev1.ContainerStatus{
33+
{Name: testPodNamespace},
34+
},
35+
},
36+
Spec: corev1.PodSpec{
37+
Containers: []corev1.Container{
38+
{Name: testPodNamespace},
39+
},
40+
},
41+
},
42+
metav1.CreateOptions{},
43+
)
44+
if err != nil {
45+
t.Fatal(err)
46+
}
47+
48+
records, err := gatherLogsFromPodsInNamespace(
49+
ctx,
50+
coreClient,
51+
testPodNamespace,
52+
[]string{
53+
stringToSearch,
54+
},
55+
86400, // last day
56+
1024*64, // maximum 64 kb of logs
57+
testLogFileName,
58+
"",
59+
)
60+
if err != nil {
61+
t.Fatal(err)
62+
}
63+
64+
assert.Len(t, records, 1)
65+
assert.Equal(
66+
t,
67+
fmt.Sprintf("config/pod/%s/logs/%s/%s.log", testPodNamespace, testPodNamespace, testLogFileName),
68+
records[0].Name,
69+
)
70+
assert.Equal(t, Raw{stringToSearch + "\n"}, records[0].Item)
71+
}

‎pkg/gather/clusterconfig/openshift_apiserver_operator_logs.go

+12-88
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,7 @@
11
package clusterconfig
22

33
import (
4-
"bufio"
5-
"context"
6-
"fmt"
7-
"strings"
8-
9-
corev1 "k8s.io/api/core/v1"
10-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
114
"k8s.io/client-go/kubernetes"
12-
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
13-
restclient "k8s.io/client-go/rest"
14-
"k8s.io/klog/v2"
155

166
"github.com/openshift/insights-operator/pkg/record"
177
)
@@ -23,7 +13,7 @@ import (
2313
// The Kubernetes API https://github.com/kubernetes/client-go/blob/master/kubernetes/typed/core/v1/pod_expansion.go#L48
2414
// Response see https://docs.openshift.com/container-platform/4.6/rest_api/workloads_apis/pod-core-v1.html#apiv1namespacesnamespacepodsnamelog
2515
//
26-
// Location in archive: config/pod/{namespace-name}/logs/{pod-name}/errors.log
16+
// Location in archive: config/pod/openshift-apiserver-operator/logs/{pod-name}/errors.log
2717
func GatherOpenShiftAPIServerOperatorLogs(g *Gatherer) ([]record.Record, []error) {
2818
messagesToSearch := []string{
2919
"the server has received too many requests and has asked us",
@@ -35,87 +25,21 @@ func GatherOpenShiftAPIServerOperatorLogs(g *Gatherer) ([]record.Record, []error
3525
return nil, []error{err}
3626
}
3727

38-
client := gatherKubeClient.CoreV1()
39-
40-
records, err := gatherOpenShiftAPIServerOperatorLastDayLogs(g.ctx, client, messagesToSearch)
41-
if err != nil {
42-
return nil, []error{err}
43-
}
44-
45-
return records, nil
46-
}
28+
coreClient := gatherKubeClient.CoreV1()
4729

48-
func gatherOpenShiftAPIServerOperatorLastDayLogs(
49-
ctx context.Context, coreClient corev1client.CoreV1Interface, messagesToSearch []string,
50-
) ([]record.Record, error) {
51-
const namespace = "openshift-apiserver-operator"
52-
var (
53-
sinceSeconds int64 = 86400 // last day
54-
limitBytes int64 = 1024 * 64 // maximum 64 kb of logs
30+
records, err := gatherLogsFromPodsInNamespace(
31+
g.ctx,
32+
coreClient,
33+
"openshift-apiserver-operator",
34+
messagesToSearch,
35+
86400, // last day
36+
1024*64, // maximum 64 kb of logs
37+
"errors",
38+
"app=openshift-apiserver-operator",
5539
)
56-
57-
pods, err := coreClient.Pods(namespace).List(ctx, metav1.ListOptions{})
5840
if err != nil {
59-
return nil, err
60-
}
61-
62-
var records []record.Record
63-
64-
for _, pod := range pods.Items {
65-
request := coreClient.Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
66-
SinceSeconds: &sinceSeconds,
67-
LimitBytes: &limitBytes,
68-
})
69-
70-
logs, err := filterLogs(ctx, request, messagesToSearch)
71-
if err != nil {
72-
return nil, err
73-
}
74-
75-
if len(strings.TrimSpace(logs)) != 0 {
76-
records = append(records, record.Record{
77-
Name: fmt.Sprintf("config/pod/%s/logs/%s/errors.log", pod.Namespace, pod.Name),
78-
Item: Raw{logs},
79-
})
80-
}
81-
}
82-
83-
if len(pods.Items) == 0 {
84-
klog.Infof("no pods in %v namespace were found", namespace)
41+
return nil, []error{err}
8542
}
8643

8744
return records, nil
8845
}
89-
90-
func filterLogs(ctx context.Context, request *restclient.Request, messagesToSearch []string) (string, error) {
91-
stream, err := request.Stream(ctx)
92-
if err != nil {
93-
return "", err
94-
}
95-
96-
defer func() {
97-
err := stream.Close()
98-
if err != nil {
99-
klog.Errorf("error during closing a stream: %v", err)
100-
}
101-
}()
102-
103-
scanner := bufio.NewScanner(stream)
104-
105-
var result string
106-
107-
for scanner.Scan() {
108-
line := scanner.Text()
109-
for _, messageToSearch := range messagesToSearch {
110-
if strings.Contains(strings.ToLower(line), strings.ToLower(messageToSearch)) {
111-
result += line + "\n"
112-
}
113-
}
114-
}
115-
116-
if err := scanner.Err(); err != nil {
117-
return "", err
118-
}
119-
120-
return result, nil
121-
}

‎pkg/gather/clusterconfig/openshift_apiserver_operator_logs_test.go

-47
This file was deleted.

0 commit comments

Comments
 (0)