sap license management logs gatherer (#342)

* sap license management logs gatherer

* added docs

* fixed tests

* removed debug dockerfile

* fixes due to code review

* fixes due to code review

* merged-constants-files

* added sample for the archive

Author: Serhii Zakharov

Diff for: docs/gathered-data.md

@@ -399,6 +399,18 @@ Relevant OpenShift API docs:
 Location in archive: config/securitycontentconstraint/, config/clusterrolebinding/
 
 
+## SAPVsystemIptablesLogs
+
+collects logs from SAP vsystem-iptables containers
+including one from license management pods with the following substring:
+  - "can't initialize iptables table",
+
+The Kubernetes API https://github.com/kubernetes/client-go/blob/master/kubernetes/typed/core/v1/pod_expansion.go#L48
+Response see https://docs.openshift.com/container-platform/4.6/rest_api/workloads_apis/pod-core-v1.html#apiv1namespacesnamespacepodsnamelog
+
+Location in archive: config/pod/{namespace}/logs/{pod-name}/errors.log
+
+
 ## ServiceAccounts
 
 collects ServiceAccount stats

Diff for: docs/insights-archive-sample/config/pod/sdi/logs/license-manager-da1d2e8fadfb8dd7022f08-4hjh7-6887768c5b-qzxb6/errors.log

@@ -0,0 +1 @@
+iptables v1.6.2: can't initialize iptables table `nat': Permission denied

Diff for: pkg/gather/clusterconfig/0_gatherer.go

@@ -84,6 +84,7 @@ var gatherFunctions = map[string]gathering{
 	"openshift_sdn_controller_logs":     failable(GatherOpenshiftSDNControllerLogs),
 	"openshift_authentication_logs":     failable(GatherOpenshiftAuthenticationLogs),
 	"sap_config":                        failable(GatherSAPConfig),
+	"sap_license_management_logs":       failable(GatherSAPVsystemIptablesLogs),
 	"olm_operators":                     failable(GatherOLMOperators),
 }
 

Diff for: pkg/gather/clusterconfig/const.go

@@ -4,9 +4,9 @@ import (
 	"time"
 
 	registryv1 "github.com/openshift/api/imageregistry/v1"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 )
 
 const (
@@ -20,7 +20,10 @@ var (
 	// logTailLines sets maximum number of lines to fetch from pod logs
 	logTailLines = int64(100)
 
-	defaultNamespaces = []string{"default", "kube-system", "kube-public"}
+	defaultNamespaces           = []string{"default", "kube-system", "kube-public"}
+	datahubGroupVersionResource = schema.GroupVersionResource{
+		Group: "installers.datahub.sap.com", Version: "v1alpha1", Resource: "datahubs",
+	}
 )
 
 func init() {

Diff for: pkg/gather/clusterconfig/gather_logs.go

@@ -17,28 +17,42 @@ import (
 	"github.com/openshift/insights-operator/pkg/utils/marshal"
 )
 
-// gatherLogsFromPodsInNamespace collects logs from the pods in provided namespace
-//   - messagesToSearch are the messages to filter the logs(case-insensitive)
-//   - sinceSeconds sets the moment to fetch logs from(current time - sinceSeconds)
-//   - limitBytes sets the maximum amount of logs that can be fetched
-//   - logFileName sets the name of the file to save logs to.
-//   - labelSelector allows you to filter pods by their labels
-//   - regexSearch makes messagesToSearch regex patterns, so you can accomplish more complicated search
+type logContainersFilter struct {
+	namespace                string
+	labelSelector            string
+	containerNameRegexFilter string
+}
+
+type logMessagesFilter struct {
+	messagesToSearch []string
+	isRegexSearch    bool
+	sinceSeconds     int64
+	limitBytes       int64
+}
+
+// gatherLogsFromContainers collects logs from containers
+//   - containerFilter allows you to specify
+//     - namespace in which to search for pods
+//     - labelSelector to filter pods by their labels (keep empty to not filter)
+//     - containerNameRegexFilter to filter containers in the pod (keep empty to not filter)
+//   - logMessagesFilter allows you to specify
+//     - messagesToSearch to filter the logs by substrings (case-insensitive)
+//       or regex (add `(?i)` in the beginning to make search case-insensitive)
+//     - regexSearch which makes messagesToSearch regex patterns, so you can accomplish more complicated search
+//     - sinceSeconds which sets the moment to fetch the logs from (current time - sinceSeconds)
+//     - limitBytes which sets the maximum amount of logs that can be fetched
+//   - logFileName sets the name of the file to save the logs to.
 //
 // Location of the logs is `config/pod/{namespace}/logs/{podName}/{fileName}.log`
-func gatherLogsFromPodsInNamespace(
+func gatherLogsFromContainers(
 	ctx context.Context,
 	coreClient v1.CoreV1Interface,
-	namespace string,
-	messagesToSearch []string,
-	sinceSeconds int64,
-	limitBytes int64,
+	containersFilter logContainersFilter,
+	messagesFilter logMessagesFilter,
 	logFileName string,
-	labelSelector string,
-	regexSearch bool,
 ) ([]record.Record, error) {
-	pods, err := coreClient.Pods(namespace).List(ctx, metav1.ListOptions{
-		LabelSelector: labelSelector,
+	pods, err := coreClient.Pods(containersFilter.namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: containersFilter.labelSelector,
 	})
 	if err != nil {
 		return nil, err
@@ -47,14 +61,32 @@ func gatherLogsFromPodsInNamespace(
 	var records []record.Record
 
 	for _, pod := range pods.Items {
+		var containers []string
 		for _, container := range pod.Spec.Containers {
-			request := coreClient.Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
-				Container:    container.Name,
-				SinceSeconds: &sinceSeconds,
-				LimitBytes:   &limitBytes,
+			containers = append(containers, container.Name)
+		}
+		for _, container := range pod.Spec.InitContainers {
+			containers = append(containers, container.Name)
+		}
+
+		for _, container := range containers {
+			if len(containersFilter.containerNameRegexFilter) > 0 {
+				match, err := regexp.MatchString(containersFilter.containerNameRegexFilter, container)
+				if err != nil {
+					return nil, err
+				}
+				if !match {
+					continue
+				}
+			}
+
+			request := coreClient.Pods(containersFilter.namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
+				Container:    container,
+				SinceSeconds: &messagesFilter.sinceSeconds,
+				LimitBytes:   &messagesFilter.limitBytes,
 			})
 
-			logs, err := filterLogs(ctx, request, messagesToSearch, regexSearch)
+			logs, err := filterLogs(ctx, request, messagesFilter.messagesToSearch, messagesFilter.isRegexSearch)
 			if err != nil {
 				return nil, err
 			}
@@ -69,7 +101,7 @@ func gatherLogsFromPodsInNamespace(
 	}
 
 	if len(pods.Items) == 0 {
-		klog.Infof("no pods in %v namespace were found", namespace)
+		klog.Infof("no pods in %v namespace were found", containersFilter.namespace)
 	}
 
 	return records, nil

Diff for: pkg/gather/clusterconfig/gather_logs_test.go

@@ -45,18 +45,21 @@ func testGatherLogs(t *testing.T, regexSearch bool, stringToSearch string, shoul
 		t.Fatal(err)
 	}
 
-	records, err := gatherLogsFromPodsInNamespace(
+	records, err := gatherLogsFromContainers(
 		ctx,
 		coreClient,
-		testPodName,
-		[]string{
-			stringToSearch,
+		logContainersFilter{
+			namespace: testPodName,
+		},
+		logMessagesFilter{
+			messagesToSearch: []string{
+				stringToSearch,
+			},
+			isRegexSearch: regexSearch,
+			sinceSeconds:  86400,     // last day
+			limitBytes:    1024 * 64, // maximum 64 kb of logs
 		},
-		86400,   // last day
-		1024*64, // maximum 64 kb of logs
 		testLogFileName,
-		"",
-		regexSearch,
 	)
 	if err != nil {
 		t.Fatal(err)

Diff for: pkg/gather/clusterconfig/openshift_apiserver_operator_logs.go

@@ -14,9 +14,19 @@ import (
 // Location in archive: config/pod/{namespace-name}/logs/{pod-name}/errors.log
 func GatherOpenShiftAPIServerOperatorLogs(g *Gatherer, c chan<- gatherResult) {
 	defer close(c)
-	messagesToSearch := []string{
-		"the server has received too many requests and has asked us",
-		"because serving request timed out and response had been started",
+
+	containersFilter := logContainersFilter{
+		namespace:     "openshift-apiserver-operator",
+		labelSelector: "app=openshift-apiserver-operator",
+	}
+	messagesFilter := logMessagesFilter{
+		messagesToSearch: []string{
+			"the server has received too many requests and has asked us",
+			"because serving request timed out and response had been started",
+		},
+		isRegexSearch: false,
+		sinceSeconds:  86400,     // last day
+		limitBytes:    1024 * 64, // maximum 64 kb of logs
 	}
 
 	gatherKubeClient, err := kubernetes.NewForConfig(g.gatherProtoKubeConfig)
@@ -27,16 +37,12 @@ func GatherOpenShiftAPIServerOperatorLogs(g *Gatherer, c chan<- gatherResult) {
 
 	coreClient := gatherKubeClient.CoreV1()
 
-	records, err := gatherLogsFromPodsInNamespace(
+	records, err := gatherLogsFromContainers(
 		g.ctx,
 		coreClient,
-		"openshift-apiserver-operator",
-		messagesToSearch,
-		86400,   // last day
-		1024*64, // maximum 64 kb of logs
+		containersFilter,
+		messagesFilter,
 		"errors",
-		"app=openshift-apiserver-operator",
-		false,
 	)
 	if err != nil {
 		c <- gatherResult{nil, []error{err}}

Diff for: pkg/gather/clusterconfig/openshift_authentication_logs.go

@@ -12,8 +12,18 @@ import (
 //
 // Location in archive: config/pod/openshift-authentication/logs/{pod-name}/errors.log
 func GatherOpenshiftAuthenticationLogs(g *Gatherer, c chan<- gatherResult) {
-	messagesToSearch := []string{
-		"AuthenticationError: invalid resource name",
+	defer close(c)
+
+	containersFilter := logContainersFilter{
+		namespace: "openshift-authentication",
+	}
+	messagesFilter := logMessagesFilter{
+		messagesToSearch: []string{
+			"AuthenticationError: invalid resource name",
+		},
+		isRegexSearch: false,
+		sinceSeconds:  86400,     // last day
+		limitBytes:    1024 * 64, // maximum 64 kb of logs
 	}
 
 	gatherKubeClient, err := kubernetes.NewForConfig(g.gatherProtoKubeConfig)
@@ -24,16 +34,12 @@ func GatherOpenshiftAuthenticationLogs(g *Gatherer, c chan<- gatherResult) {
 
 	coreClient := gatherKubeClient.CoreV1()
 
-	records, err := gatherLogsFromPodsInNamespace(
+	records, err := gatherLogsFromContainers(
 		g.ctx,
 		coreClient,
-		"openshift-authentication",
-		messagesToSearch,
-		86400,   // last day
-		1024*64, // maximum 64 kb of logs
+		containersFilter,
+		messagesFilter,
 		"errors",
-		"",
-		false,
 	)
 	if err != nil {
 		c <- gatherResult{nil, []error{err}}

Diff for: pkg/gather/clusterconfig/openshift_sdn_controller_logs.go

@@ -21,11 +21,22 @@ import (
 //
 // Location in archive: config/pod/openshift-sdn/logs/{pod-name}/errors.log
 func GatherOpenshiftSDNControllerLogs(g *Gatherer, c chan<- gatherResult) {
-	messagesToSearch := []string{
-		"Node.+is not Ready",
-		"Node.+may be offline\\.\\.\\. retrying",
-		"Node.+is offline",
-		"Node.+is back online",
+	defer close(c)
+
+	containersFilter := logContainersFilter{
+		namespace:     "openshift-sdn",
+		labelSelector: "app=sdn-controller",
+	}
+	messagesFilter := logMessagesFilter{
+		messagesToSearch: []string{
+			"Node.+is not Ready",
+			"Node.+may be offline\\.\\.\\. retrying",
+			"Node.+is offline",
+			"Node.+is back online",
+		},
+		isRegexSearch: true,
+		sinceSeconds:  86400,     // last day
+		limitBytes:    1024 * 64, // maximum 64 kb of logs
 	}
 
 	gatherKubeClient, err := kubernetes.NewForConfig(g.gatherProtoKubeConfig)
@@ -36,16 +47,12 @@ func GatherOpenshiftSDNControllerLogs(g *Gatherer, c chan<- gatherResult) {
 
 	coreClient := gatherKubeClient.CoreV1()
 
-	records, err := gatherLogsFromPodsInNamespace(
+	records, err := gatherLogsFromContainers(
 		g.ctx,
 		coreClient,
-		"openshift-sdn",
-		messagesToSearch,
-		86400,   // last day
-		1024*64, // maximum 64 kb of logs
+		containersFilter,
+		messagesFilter,
 		"errors",
-		"app=sdn-controller",
-		true,
 	)
 	if err != nil {
 		c <- gatherResult{nil, []error{err}}

Diff for: pkg/gather/clusterconfig/openshift_sdn_logs.go

@@ -16,11 +16,21 @@ import (
 // Location in archive: config/pod/openshift-sdn/logs/{pod-name}/errors.log
 func GatherOpenshiftSDNLogs(g *Gatherer, c chan<- gatherResult) {
 	defer close(c)
-	messagesToSearch := []string{
-		"Got OnEndpointsUpdate for unknown Endpoints",
-		"Got OnEndpointsDelete for unknown Endpoints",
-		"Unable to update proxy firewall for policy",
-		"Failed to update proxy firewall for policy",
+
+	containersFilter := logContainersFilter{
+		namespace:     "openshift-sdn",
+		labelSelector: "app=sdn",
+	}
+	messagesFilter := logMessagesFilter{
+		messagesToSearch: []string{
+			"Got OnEndpointsUpdate for unknown Endpoints",
+			"Got OnEndpointsDelete for unknown Endpoints",
+			"Unable to update proxy firewall for policy",
+			"Failed to update proxy firewall for policy",
+		},
+		isRegexSearch: false,
+		sinceSeconds:  86400,
+		limitBytes:    1024 * 64,
 	}
 
 	gatherKubeClient, err := kubernetes.NewForConfig(g.gatherProtoKubeConfig)
@@ -31,16 +41,12 @@ func GatherOpenshiftSDNLogs(g *Gatherer, c chan<- gatherResult) {
 
 	coreClient := gatherKubeClient.CoreV1()
 
-	records, err := gatherLogsFromPodsInNamespace(
+	records, err := gatherLogsFromContainers(
 		g.ctx,
 		coreClient,
-		"openshift-sdn",
-		messagesToSearch,
-		86400,   // last day
-		1024*64, // maximum 64 kb of logs
+		containersFilter,
+		messagesFilter,
 		"errors",
-		"app=sdn",
-		false,
 	)
 	if err != nil {
 		c <- gatherResult{nil, []error{err}}

Diff for: pkg/gather/clusterconfig/sap_config.go

@@ -7,7 +7,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -56,9 +55,7 @@ func gatherSAPConfig(ctx context.Context, dynamicClient dynamic.Interface, coreC
 	sccToGather := []string{"anyuid", "privileged"}
 	crbToGather := []string{"system:openshift:scc:anyuid", "system:openshift:scc:privileged"}
 
-	datahubsResource := schema.GroupVersionResource{Group: "installers.datahub.sap.com", Version: "v1alpha1", Resource: "datahubs"}
-
-	datahubsList, err := dynamicClient.Resource(datahubsResource).List(ctx, metav1.ListOptions{})
+	datahubsList, err := dynamicClient.Resource(datahubGroupVersionResource).List(ctx, metav1.ListOptions{})
 	if errors.IsNotFound(err) {
 		return nil, nil
 	}