Enable full advanced audit in origin

Author: soltysh

contrib/completions/bash/openshift

@@ -32067,6 +32067,8 @@ _openshift_start_kubernetes_apiserver()
     local_nonpersistent_flags+=("--anonymous-auth")
     flags+=("--apiserver-count=")
     local_nonpersistent_flags+=("--apiserver-count=")
+    flags+=("--audit-log-format=")
+    local_nonpersistent_flags+=("--audit-log-format=")
     flags+=("--audit-log-maxage=")
     local_nonpersistent_flags+=("--audit-log-maxage=")
     flags+=("--audit-log-maxbackup=")
@@ -33079,6 +33081,8 @@ _openshift_start_template-service-broker()
     flags_with_completion=()
     flags_completion=()
 
+    flags+=("--audit-log-format=")
+    local_nonpersistent_flags+=("--audit-log-format=")
     flags+=("--audit-log-maxage=")
     local_nonpersistent_flags+=("--audit-log-maxage=")
     flags+=("--audit-log-maxbackup=")

contrib/completions/zsh/openshift

@@ -32216,6 +32216,8 @@ _openshift_start_kubernetes_apiserver()
     local_nonpersistent_flags+=("--anonymous-auth")
     flags+=("--apiserver-count=")
     local_nonpersistent_flags+=("--apiserver-count=")
+    flags+=("--audit-log-format=")
+    local_nonpersistent_flags+=("--audit-log-format=")
     flags+=("--audit-log-maxage=")
     local_nonpersistent_flags+=("--audit-log-maxage=")
     flags+=("--audit-log-maxbackup=")
@@ -33228,6 +33230,8 @@ _openshift_start_template-service-broker()
     flags_with_completion=()
     flags_completion=()
 
+    flags+=("--audit-log-format=")
+    local_nonpersistent_flags+=("--audit-log-format=")
     flags+=("--audit-log-maxage=")
     local_nonpersistent_flags+=("--audit-log-maxage=")
     flags+=("--audit-log-maxbackup=")

pkg/cmd/server/api/types.go

@@ -464,7 +464,6 @@ type AggregatorConfig struct {
 // AuditConfig holds configuration for the audit capabilities
 type AuditConfig struct {
 	// If this flag is set, audit log will be printed in the logs.
-	// The logs contains, method, user and a requested URL.
 	Enabled bool
 	// All requests coming to the apiserver will be logged to this file.
 	AuditFilePath string
@@ -474,6 +473,17 @@ type AuditConfig struct {
 	MaximumRetainedFiles int
 	// Maximum size in megabytes of the log file before it gets rotated. Defaults to 100MB.
 	MaximumFileSizeMegabytes int
+
+	// Path to the file that defines the audit policy configuration.
+	PolicyFile string
+
+	// Format of saved audits (legacy or json).
+	LogFormat string
+
+	// Path to a kubeconfig formatted file that defines the audit webhook configuration.
+	WebhookConfigFile string
+	// Strategy for sending audit events (block or batch).
+	WebhookMode string
 }
 
 // JenkinsPipelineConfig holds configuration for the Jenkins pipeline strategy

pkg/cmd/server/api/v1/swagger_doc.go

@@ -90,6 +90,10 @@ var map_AuditConfig = map[string]string{
 	"maximumFileRetentionDays": "Maximum number of days to retain old log files based on the timestamp encoded in their filename.",
 	"maximumRetainedFiles":     "Maximum number of old log files to retain.",
 	"maximumFileSizeMegabytes": "Maximum size in megabytes of the log file before it gets rotated. Defaults to 100MB.",
+	"policyFile":               "Path to the file that defines the audit policy configuration.",
+	"logFormat":                "Format of saved audits (legacy or json).",
+	"webhookConfigFile":        "Path to a kubeconfig formatted file that defines the audit webhook configuration.",
+	"webhookMode":              "Strategy for sending audit events (block or batch).",
 }
 
 func (AuditConfig) SwaggerDoc() map[string]string {

pkg/cmd/server/api/v1/types.go

@@ -331,6 +331,17 @@ type AuditConfig struct {
 	MaximumRetainedFiles int `json:"maximumRetainedFiles"`
 	// Maximum size in megabytes of the log file before it gets rotated. Defaults to 100MB.
 	MaximumFileSizeMegabytes int `json:"maximumFileSizeMegabytes"`
+
+	// Path to the file that defines the audit policy configuration.
+	PolicyFile string `json:"policyFile"`
+
+	// Format of saved audits (legacy or json).
+	LogFormat string `json:"logFormat"`
+
+	// Path to a kubeconfig formatted file that defines the audit webhook configuration.
+	WebhookConfigFile string `json:"webhookConfigFile"`
+	// Strategy for sending audit events (block or batch).
+	WebhookMode string `json:"webhookMode"`
 }
 
 // JenkinsPipelineConfig holds configuration for the Jenkins pipeline strategy

pkg/cmd/server/api/v1/types_test.go

@@ -117,9 +117,13 @@ assetConfig:
 auditConfig:
   auditFilePath: ""
   enabled: false
+  logFormat: ""
   maximumFileRetentionDays: 0
   maximumFileSizeMegabytes: 0
   maximumRetainedFiles: 0
+  policyFile: ""
+  webhookConfigFile: ""
+  webhookMode: ""
 authConfig:
   requestHeader: null
 controllerConfig:

pkg/cmd/server/api/validation/master.go

@@ -14,6 +14,10 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	kuval "k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	auditpolicy "k8s.io/apiserver/pkg/audit/policy"
+	auditlog "k8s.io/apiserver/plugin/pkg/audit/log"
+	auditwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
+	"k8s.io/client-go/tools/clientcmd"
 	apiserveroptions "k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	kcmoptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
 	kvalidation "k8s.io/kubernetes/pkg/api/validation"
@@ -238,6 +242,9 @@ func ValidateAggregatorConfig(config api.AggregatorConfig, fldPath *field.Path)
 
 func ValidateAuditConfig(config api.AuditConfig, fldPath *field.Path) ValidationResults {
 	validationResults := ValidationResults{}
+	if !config.Enabled {
+		return validationResults
+	}
 
 	if len(config.AuditFilePath) == 0 {
 		// for backwards compatibility reasons we can't error this out
@@ -253,6 +260,35 @@ func ValidateAuditConfig(config api.AuditConfig, fldPath *field.Path) Validation
 		validationResults.AddErrors(field.Invalid(fldPath.Child("maximumFileSizeMegabytes"), config.MaximumFileSizeMegabytes, "must be greater than or equal to 0"))
 	}
 
+	// setting policy file will turn the advanced auditing on
+	if len(config.PolicyFile) > 0 {
+		policy, err := auditpolicy.LoadPolicyFromFile(config.PolicyFile)
+		if err != nil {
+			validationResults.AddErrors(field.Invalid(fldPath.Child("policyFile"), config.PolicyFile, err.Error()))
+		}
+		if len(policy.Rules) == 0 {
+			validationResults.AddErrors(field.Invalid(fldPath.Child("policyFile"), config.PolicyFile, "a policy file with 0 policies is not valid"))
+		}
+		if config.LogFormat != auditlog.FormatLegacy && config.LogFormat != auditlog.FormatJson {
+			validationResults.AddErrors(field.NotSupported(fldPath.Child("logFormat"), config.LogFormat, auditlog.AllowedFormats))
+		}
+		if len(config.AuditFilePath) == 0 {
+			validationResults.AddErrors(field.Required(fldPath.Child("auditFilePath"), "advanced audit requires a separate log file"))
+		}
+
+		if len(config.WebhookConfigFile) > 0 {
+			if config.WebhookMode != auditwebhook.ModeBatch && config.WebhookMode != auditwebhook.ModeBlocking {
+				validationResults.AddErrors(field.NotSupported(fldPath.Child("webhookMode"), config.WebhookMode, auditwebhook.AllowedModes))
+			}
+			loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+			loadingRules.ExplicitPath = config.WebhookConfigFile
+			loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+			if _, err := loader.ClientConfig(); err != nil {
+				validationResults.AddErrors(field.Invalid(fldPath.Child("webhookConfigFile"), config.WebhookConfigFile, err.Error()))
+			}
+		}
+	}
+
 	return validationResults
 }
 

pkg/cmd/server/kubernetes/master/master_config.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/admission"
 	auditinternal "k8s.io/apiserver/pkg/apis/audit"
+	"k8s.io/apiserver/pkg/audit"
 	auditpolicy "k8s.io/apiserver/pkg/audit/policy"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
@@ -45,6 +46,7 @@ import (
 	storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	auditlog "k8s.io/apiserver/plugin/pkg/audit/log"
+	auditwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
 	kapiserveroptions "k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
 	kapi "k8s.io/kubernetes/pkg/api"
@@ -177,12 +179,11 @@ func BuildKubeAPIserverOptions(masterConfig configapi.MasterConfig) (*kapiserver
 			args["feature-gates"] = []string{existing[0] + ",AdvancedAuditing=true"}
 		} else {
 			args["feature-gates"] = []string{"AdvancedAuditing=true"}
-
 		}
 	}
 	// TODO: this should be done in config validation (along with the above) so we can provide
 	// proper errors
-	if err := cmdflags.Resolve(masterConfig.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
+	if err := cmdflags.Resolve(args, server.AddFlags); len(err) > 0 {
 		return nil, kerrors.NewAggregate(err)
 	}
 
@@ -526,12 +527,33 @@ func buildKubeApiserverConfig(
 			// backwards compatible writer to regular log
 			writer = cmdutil.NewGLogWriterV(0)
 		}
-		genericConfig.AuditBackend = auditlog.NewBackend(writer)
+		genericConfig.AuditBackend = auditlog.NewBackend(writer, auditlog.FormatLegacy)
 		genericConfig.AuditPolicyChecker = auditpolicy.NewChecker(&auditinternal.Policy{
 			// This is for backwards compatibility maintaining the old visibility, ie. just
 			// raw overview of the requests comming in.
 			Rules: []auditinternal.PolicyRule{{Level: auditinternal.LevelMetadata}},
 		})
+
+		// when a policy file is defined we enable the advanced auditing
+		if len(masterConfig.AuditConfig.PolicyFile) > 0 {
+			// policy configuration
+			p, _ := auditpolicy.LoadPolicyFromFile(masterConfig.AuditConfig.PolicyFile)
+			genericConfig.AuditPolicyChecker = auditpolicy.NewChecker(p)
+
+			// log configuration, only when file path was provided
+			if len(masterConfig.AuditConfig.AuditFilePath) > 0 {
+				genericConfig.AuditBackend = auditlog.NewBackend(writer, masterConfig.AuditConfig.LogFormat)
+			}
+
+			// webhook configuration, only when config file was provided
+			if len(masterConfig.AuditConfig.WebhookConfigFile) > 0 {
+				webhook, err := auditwebhook.NewBackend(masterConfig.AuditConfig.WebhookConfigFile, masterConfig.AuditConfig.WebhookMode)
+				if err != nil {
+					glog.Fatalf("Audit webhook initialization failed: %v", err)
+				}
+				genericConfig.AuditBackend = audit.Union(genericConfig.AuditBackend, webhook)
+			}
+		}
 	}
 
 	kubeApiserverConfig := &master.Config{

pkg/cmd/server/kubernetes/master/master_config_test.go

@@ -119,6 +119,9 @@ func TestAPIServerDefaults(t *testing.T) {
 			HTTPTimeout: time.Duration(5) * time.Second,
 		},
 		Audit: &apiserveroptions.AuditOptions{
+			LogOptions: apiserveroptions.AuditLogOptions{
+				Format: "legacy",
+			},
 			WebhookOptions: apiserveroptions.AuditWebhookOptions{
 				Mode: "batch",
 			},