update config for tha latest opensource beats version

Author: devopstales

filebeat.daemonset.yml

@@ -1,25 +1,34 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   namespace: kube-system
   name: filebeat
   labels:
     app: filebeat
 spec:
+  selector:
+    matchLabels:
+      app: filebeat
   template:
     metadata:
       labels:
-       app: filebeat
+        app: filebeat
     spec:
+      tolerations:
+      # these tolerations are to have the daemonset runnable on control plane nodes
+      # remove them if your control plane nodes should not run pods
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        operator: Exists
       serviceAccountName: filebeat
       terminationGracePeriodSeconds: 30
-      # for openshift
-      securityContext:
-        runAsUser: 0
-        privileged: true
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: filebeat
-        image: docker.elastic.co/beats/filebeat:7.4.2
+        image: docker.elastic.co/beats/filebeat:7.12.1
         args: [
           "-c", "/etc/filebeat.yml",
           "-e",
@@ -29,15 +38,25 @@ spec:
           value: 192.168.0.31
         - name: GRAYLOG_PORT
           value: "5042"
+        #- name: ELASTICSEARCH_HOST
+        #  value: elasticsearch
+        #- name: ELASTICSEARCH_PORT
+        #  value: "9200"
+        #- name: ELASTICSEARCH_USERNAME
+        #  value: elastic
+        #- name: ELASTICSEARCH_PASSWORD
+        #  value: changeme
         - name: NODE_NAME
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
         securityContext:
           runAsUser: 0
+          # If using Red Hat OpenShift uncomment this:
+          #privileged: true
         resources:
           limits:
-             memory: 200Mi
+            memory: 200Mi
           requests:
             cpu: 100m
             memory: 100Mi
@@ -54,24 +73,20 @@ spec:
         - name: varlibdockercontainers
           mountPath: /var/lib/docker/containers
           readOnly: true
-        - name: dockersock
-          mountPath: /var/run/docker.sock
       volumes:
       - name: config
         configMap:
-          defaultMode: 0600
+          defaultMode: 0640
           name: filebeat-config
       - name: varlog
         hostPath:
           path: /var/log
       - name: varlibdockercontainers
         hostPath:
           path: /var/lib/docker/containers
-      - name: dockersock
-        hostPath:
-           path: /var/run/docker.sock
       # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
       - name: data
         hostPath:
+          # When filebeat runs as non-root user, this directory needs to be writable by group (g+w).
           path: /var/lib/filebeat-data
           type: DirectoryOrCreate

filebeat.permission.yml

@@ -1,8 +1,7 @@
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  namespace: kube-system
   name: filebeat
 subjects:
 - kind: ServiceAccount
@@ -13,10 +12,9 @@ roleRef:
   name: filebeat
   apiGroup: rbac.authorization.k8s.io
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  namespace: kube-system
   name: filebeat
   labels:
     app: filebeat
@@ -25,10 +23,15 @@ rules:
   resources:
   - namespaces
   - pods
+  - nodes
   verbs:
   - get
   - watch
   - list
+- apiGroups: ["apps"]
+  resources:
+    - replicasets
+  verbs: ["get", "list", "watch"]
 ---
 apiVersion: v1
 kind: ServiceAccount

filebeat.settings.configmap.yml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -10,62 +11,37 @@ data:
     filebeat.inputs:
     - type: container
       paths:
-	      - /var/log/containers/*.log
+        - /var/log/containers/*.log
       processors:
         - add_kubernetes_metadata:
-            in_cluster: true
             host: ${NODE_NAME}
             matchers:
             - logs_path:
                 logs_path: "/var/log/containers/"
       fields:
         cluster: cl02
 
-    filebeat.modules:
-      - module: system
-        syslog:
-          enabled: true
-        auth:
-          enabled: true
-
     filebeat.autodiscover:
       providers:
         - type: kubernetes
-          templates:
-            - condition.equals:
-                kubernetes.labels.app: mongo
-              config:
-                - module: mongodb
-                  enabled: true
-                  log:
-                    input:
-                      type: docker
-                      containers.ids:
-                        - ${data.kubernetes.container.id}
+          node: ${NODE_NAME}
+          hints.enabled: true
+          hints.default_config:
+            type: container
+            paths:
+              - /var/log/containers/*${data.kubernetes.container.id}.log
       fields:
         cluster: cl02
 
     processors:
-      - drop_event:
-          when.or:
-              - and:
-                  - regexp:
-                      message: '^\d+\.\d+\.\d+\.\d+ '
-                  - equals:
-                      fileset.name: error
-              - and:
-                  - not:
-                      regexp:
-                          message: '^\d+\.\d+\.\d+\.\d+ '
-                  - equals:
-                      fileset.name: access
       - add_cloud_metadata:
       - add_host_metadata:
-      - add_kubernetes_metadata:
-      - add_docker_metadata:
+      - add_kubernetes_metadata
 
     output.logstash:
       hosts: ['${GRAYLOG_HOST}:${GRAYLOG_PORT}']
 
-    setup.dashboards.enabled: false
-    setup.template.enabled: false
+    #output.elasticsearch:
+    #  hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
+    #  username: ${ELASTICSEARCH_USERNAME}
+    #  password: ${ELASTICSEARCH_PASSWORD}