Support for App Protect module

Author: Rafal Wegrzycki

build/appprotect/DockerfileWithAppProtectForPlus

@@ -0,0 +1,128 @@
+ARG GOLANG_CONTAINER=golang:latest
+
+FROM debian:stretch-slim as base
+
+LABEL maintainer="NGINX Docker Maintainers <[email protected]>"
+
+ENV APPPROTECT_VERSION 21+2.52.1-1~stretch
+ENV APPPROTECT_SIG_VERSION 2020.06.18-1~stretch
+ENV NGINX_PLUS_VERSION 21-1~stretch
+ENV NGINX_PLUS_RELEASE R21
+ARG IC_VERSION
+
+# Download certificate and key from the customer portal (https://cs.nginx.com)
+# and copy to the build context
+COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/
+
+# Make sure the certificate and key have correct permissions
+RUN chmod 644 /etc/ssl/nginx/*
+
+# Install NGINX Plus
+RUN set -x \
+	&& apt-get update \
+	&& apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin wget \
+	&& \
+	NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
+	found=''; \
+	for server in \
+		ha.pool.sks-keyservers.net \
+		hkp://keyserver.ubuntu.com:80 \
+		hkp://p80.pool.sks-keyservers.net:80 \
+		pgp.mit.edu \
+	; do \
+		echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
+		apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
+  	done; \
+	test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
+	echo "Acquire::https::plus-pkgs.nginx.com::Verify-Peer \"true\";" >> /etc/apt/apt.conf.d/90nginx \
+	&& echo "Acquire::https::plus-pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \
+	&& echo "Acquire::https::plus-pkgs.nginx.com::SslCert     \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \
+	&& echo "Acquire::https::plus-pkgs.nginx.com::SslKey      \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \
+	&& echo "Acquire::https::plus-pkgs.nginx.com::User-Agent  \"k8s-ic-$IC_VERSION-app-$APPPROTECT_VERSION-apt\";" >> /etc/apt/apt.conf.d/90nginx \
+	&& echo "deb https://plus-pkgs.nginx.com/${NGINX_PLUS_RELEASE}/debian stretch nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \
+	&& echo "deb https://app-protect-sigs.nginx.com/debian/ stretch nginx-plus\n" | tee /etc/apt/sources.list.d/app-protect-sigs.list \
+	&& wget https://nginx.org/keys/app-protect-sigs.key && apt-key add app-protect-sigs.key \
+	&& echo "Acquire::https::app-protect-sigs.nginx.com::Verify-Peer \"true\";" >> /etc/apt/apt.conf.d/90app-protect-sigs \
+	&& echo "Acquire::https::app-protect-sigs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90app-protect-sigs \
+	&& echo "Acquire::https::app-protect-sigs.nginx.com::SslCert     \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90app-protect-sigs \
+	&& echo "Acquire::https::app-protect-sigs.nginx.com::SslKey      \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90app-protect-sigs \
+	&& apt-get update && apt-get install -y nginx-plus=$NGINX_PLUS_VERSION app-protect=$APPPROTECT_VERSION \ 
+	app-protect-attack-signatures${APPPROTECT_SIG_VERSION:+=$APPPROTECT_SIG_VERSION} \
+	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
+	&& apt-get remove --purge --auto-remove -y gnupg1 wget\
+	&& rm -rf /var/lib/apt/lists/* \
+	&& rm -rf /etc/ssl/nginx \
+	&& rm /etc/apt/apt.conf.d/90nginx /etc/apt/sources.list.d/nginx-plus.list \
+	&& rm /etc/apt/apt.conf.d/90app-protect-sigs /etc/apt/sources.list.d/app-protect-sigs.list
+
+RUN usermod -u 101 nginx \
+	&& groupmod -g 101 nginx
+
+# forward nginx access and error logs to stdout and stderr of the ingress
+# controller process
+RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
+	&& ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
+	&& ln -sf /proc/1/fd/2 /var/log/nginx/error.log
+
+RUN  mkdir -p /var/lib/nginx \
+	&& mkdir -p /etc/nginx/secrets \
+	&& mkdir -p /etc/nginx/waf \
+	&& mkdir -p /etc/nginx/waf/nac-policies \
+	&& mkdir -p /etc/nginx/waf/nac-logconfs \
+	&& mkdir -p /var/log/app_protect \
+	&& mkdir -p /opt/app_protect \
+	&& chown -R nginx:0 /etc/app_protect \
+	&& chown -R nginx:0 /usr/share/ts \
+	&& chown -R nginx:0 /etc/nginx \
+	&& chown -R nginx:0 /var/cache/nginx \
+	&& chown -R nginx:0 /var/lib/nginx/ \
+	&& chown -R nginx:0 /var/log/app_protect/ \
+	&& chown -R nginx:0 /opt/app_protect/ \
+	&& chown -R nginx:0 /var/log/nginx/ \
+	&& apt-get remove --purge -y libcap2-bin \
+	&& rm /etc/nginx/conf.d/*
+
+RUN printf "MODULE = ALL;\nLOG_LEVEL = TS_CRIT;\nFILE = 2;\n" > /etc/app_protect/bd/logger.cfg \
+	&&  printf "[config_set_compiler]\nlog_level=fatal\n" >> /etc/app_protect/tools/asm_logging.conf \
+	&& for v in \
+		asm_config_server \
+		lock_factory \
+		bd_agent \
+		import_export_policy \
+	set_active \
+	; do sed -i "/\[$v/a log_level=fatal" "/etc/app_protect/tools/asm_logging.conf" \
+	; done
+
+COPY --chown=nginx:0 build/appprotect/log-default.json /etc/nginx  
+
+EXPOSE 80 443
+
+COPY internal/configs/version1/nginx-plus.ingress.tmpl \
+	internal/configs/version1/nginx-plus.tmpl \
+	internal/configs/version2/nginx-plus.virtualserver.tmpl \
+	internal/configs/version2/nginx-plus.transportserver.tmpl /
+
+# Uncomment the line below if you would like to add the default.pem to the image
+# and use it as a certificate and key for the default server
+# ADD default.pem /etc/nginx/secrets/default
+
+USER nginx
+
+ENTRYPOINT ["/nginx-ingress"]
+
+FROM base AS local
+COPY nginx-ingress /
+
+
+FROM $GOLANG_CONTAINER AS builder
+ARG VERSION
+ARG GIT_COMMIT
+WORKDIR /go/src/github.com/nginxinc/kubernetes-ingress/nginx-ingress/cmd/nginx-ingress
+COPY . /go/src/github.com/nginxinc/kubernetes-ingress/nginx-ingress/
+RUN CGO_ENABLED=0 GOFLAGS='-mod=vendor' \
+	go build -installsuffix cgo -ldflags "-w -X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT}" -o /nginx-ingress
+
+
+FROM base AS container
+COPY --from=builder /nginx-ingress /

build/appprotect/log-default.json

@@ -0,0 +1,10 @@
+{
+    "filter": {
+        "request_type": "all"
+    },
+    "content": {
+        "format": "default",
+        "max_request_size": "any",
+        "max_message_size": "5k"
+    }
+}

cmd/nginx-ingress/main.go

@@ -29,6 +29,7 @@ import (
 	api_v1 "k8s.io/api/core/v1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -37,6 +38,7 @@ import (
 )
 
 var (
+
 	// Set during build
 	version   string
 	gitCommit string
@@ -62,6 +64,8 @@ var (
 
 	nginxPlus = flag.Bool("nginx-plus", false, "Enable support for NGINX Plus")
 
+	appProtect = flag.Bool("enable-app-protect", false, "Enable support for NGINX App Protect. Requires -nginx-plus.")
+
 	ingressClass = flag.String("ingress-class", "nginx",
 		`A class of the Ingress controller. The Ingress controller only processes Ingress resources that belong to its class
 	- i.e. have the annotation "kubernetes.io/ingress.class" or the "ingressClassName" field in VirtualServer/VirtualServerRoute equal to the class. Additionally,
@@ -118,6 +122,10 @@ var (
 	nginxDebug = flag.Bool("nginx-debug", false,
 		"Enable debugging for NGINX. Uses the nginx-debug binary. Requires 'error-log-level: debug' in the ConfigMap.")
 
+	nginxReloadTimeout = flag.Int("nginx-reload-timeout", 0,
+		`The timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start.
+		The default is 4000 (or 20000 if -enable-app-protect is true). If set to 0, the default value will be used`)
+
 	wildcardTLSSecret = flag.String("wildcard-tls-secret", "",
 		`A Secret with a TLS certificate and key for TLS termination of every Ingress host for which TLS termination is enabled but the Secret is not specified.
 		Format: <namespace>/<name>. If the argument is not set, for such Ingress hosts NGINX will break any attempt to establish a TLS connection.
@@ -190,6 +198,10 @@ func main() {
 		glog.Fatalf("enable-tls-passthrough flag requires -enable-custom-resources")
 	}
 
+	if *appProtect && !*nginxPlus {
+		glog.Fatal("NGINX App Protect support is for NGINX Plus only")
+	}
+
 	glog.Infof("Starting NGINX Ingress controller Version=%v GitCommit=%v\n", version, gitCommit)
 
 	var config *rest.Config
@@ -215,6 +227,13 @@ func main() {
 		glog.Fatalf("Failed to create client: %v.", err)
 	}
 
+	var dynClient dynamic.Interface
+	if *appProtect {
+		dynClient, err = dynamic.NewForConfig(config)
+		if err != nil {
+			glog.Fatalf("Failed to create dynamic client: %v.", err)
+		}
+	}
 	var confClient k8s_nginx.Interface
 	if *enableCustomResources {
 		confClient, err = k8s_nginx.NewForConfig(config)
@@ -296,7 +315,18 @@ func main() {
 	if useFakeNginxManager {
 		nginxManager = nginx.NewFakeManager("/etc/nginx")
 	} else {
-		nginxManager = nginx.NewLocalManager("/etc/nginx/", nginxBinaryPath, managerCollector)
+		nginxManager = nginx.NewLocalManager("/etc/nginx/", nginxBinaryPath, managerCollector, parseReloadTimeout(*appProtect, *nginxReloadTimeout))
+	}
+
+	var aPPluginDone chan error
+	var aPAgentDone chan error
+
+	if *appProtect {
+		aPPluginDone = make(chan error, 1)
+		aPAgentDone = make(chan error, 1)
+
+		nginxManager.AppProtectAgentStart(aPAgentDone, *nginxDebug)
+		nginxManager.AppProtectPluginStart(aPPluginDone)
 	}
 
 	if *defaultServerSecret != "" {
@@ -355,6 +385,7 @@ func main() {
 	}
 
 	cfgParams := configs.NewDefaultConfigParams()
+
 	if *nginxConfigMaps != "" {
 		ns, name, err := k8s.ParseNamespaceName(*nginxConfigMaps)
 		if err != nil {
@@ -364,7 +395,7 @@ func main() {
 		if err != nil {
 			glog.Fatalf("Error when getting %v: %v", *nginxConfigMaps, err)
 		}
-		cfgParams = configs.ParseConfigMap(cfm, *nginxPlus)
+		cfgParams = configs.ParseConfigMap(cfm, *nginxPlus, *appProtect)
 		if cfgParams.MainServerSSLDHParamFileContent != nil {
 			fileName, err := nginxManager.CreateDHParam(*cfgParams.MainServerSSLDHParamFileContent)
 			if err != nil {
@@ -386,7 +417,6 @@ func main() {
 			}
 		}
 	}
-
 	staticCfgParams := &configs.StaticConfigParams{
 		HealthStatus:                   *healthStatus,
 		HealthStatusURI:                *healthStatusURI,
@@ -397,6 +427,7 @@ func main() {
 		TLSPassthrough:                 *enableTLSPassthrough,
 		EnableSnippets:                 *enableSnippets,
 		SpiffeCerts:                    *spireAgentAddress != "",
+		MainAppProtectLoadModule:       *appProtect,
 	}
 
 	ngxConfig := configs.GenerateNginxMainConfig(staticCfgParams, cfgParams)
@@ -457,10 +488,12 @@ func main() {
 	lbcInput := k8s.NewLoadBalancerControllerInput{
 		KubeClient:                   kubeClient,
 		ConfClient:                   confClient,
+		DynClient:                    dynClient,
 		ResyncPeriod:                 30 * time.Second,
 		Namespace:                    *watchNamespace,
 		NginxConfigurator:            cnf,
 		DefaultServerSecret:          *defaultServerSecret,
+		AppProtectEnabled:            *appProtect,
 		IsNginxPlus:                  *nginxPlus,
 		IngressClass:                 *ingressClass,
 		UseIngressClassOnly:          *useIngressClassOnly,
@@ -481,7 +514,11 @@ func main() {
 
 	lbc := k8s.NewLoadBalancerController(lbcInput)
 
-	go handleTermination(lbc, nginxManager, nginxDone)
+	if *appProtect {
+		go handleTerminationWithAppProtect(lbc, nginxManager, nginxDone, aPAgentDone, aPPluginDone)
+	} else {
+		go handleTermination(lbc, nginxManager, nginxDone)
+	}
 	lbc.Run()
 
 	for {
@@ -631,3 +668,43 @@ func validateLocation(location string) error {
 	}
 	return nil
 }
+
+func handleTerminationWithAppProtect(lbc *k8s.LoadBalancerController, nginxManager nginx.Manager, nginxDone, agentDone, pluginDone chan error) {
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGTERM)
+
+	select {
+	case err := <-nginxDone:
+		glog.Fatalf("nginx command exited unexpectedly with status: %v", err)
+	case err := <-pluginDone:
+		glog.Fatalf("AppProtectPlugin command exited unexpectedly with status: %v", err)
+	case err := <-agentDone:
+		glog.Fatalf("AppProtectAgent command exited unexpectedly with status: %v", err)
+	case <-signalChan:
+		glog.Infof("Received SIGTERM, shutting down")
+		lbc.Stop()
+		nginxManager.Quit()
+		<-nginxDone
+		nginxManager.AppProtectPluginQuit()
+		<-pluginDone
+		nginxManager.AppProtectAgentQuit()
+		<-agentDone
+	}
+	glog.Info("Exiting successfully")
+	os.Exit(0)
+}
+
+func parseReloadTimeout(appProtectEnabled bool, timeout int) int {
+	const defaultTimeout = 4000
+	const defaultTimeoutAppProtect = 20000
+
+	if timeout != 0 {
+		return timeout
+	}
+
+	if appProtectEnabled {
+		return defaultTimeoutAppProtect
+	}
+
+	return defaultTimeout
+}

cmd/nginx-ingress/main_test.go

@@ -124,3 +124,39 @@ func TestValidateLocation(t *testing.T) {
 		}
 	}
 }
+
+func TestParseReloadTimeout(t *testing.T) {
+	tests := []struct {
+		timeout           int
+		appProtectEnabled bool
+		expected          int
+	}{
+		{
+			timeout:           0,
+			appProtectEnabled: true,
+			expected:          20000,
+		},
+		{
+			timeout:           0,
+			appProtectEnabled: false,
+			expected:          4000,
+		},
+		{
+			timeout:           1000,
+			appProtectEnabled: true,
+			expected:          1000,
+		},
+		{
+			timeout:           1000,
+			appProtectEnabled: false,
+			expected:          1000,
+		},
+	}
+
+	for _, test := range tests {
+		result := parseReloadTimeout(test.appProtectEnabled, test.timeout)
+		if result != test.expected {
+			t.Errorf("parseReloadTimeout(%v, %v) returned %v but expected %v", test.appProtectEnabled, test.timeout, result, test.expected)
+		}
+	}
+}