Run IC as non-root

Author: LorcanMcVeigh

build/Dockerfile

@@ -6,14 +6,24 @@ RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
 	&& ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
 	&& ln -sf /proc/1/fd/2 /var/log/nginx/error.log
 
-COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
-
-RUN rm /etc/nginx/conf.d/*
+RUN mkdir -p /var/lib/nginx \
+	&& mkdir -p /etc/nginx/secrets \
+	&& apt-get update \
+	&& apt-get install -y libcap2-bin \
+	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+	&& chown -R nginx:0 /etc/nginx \
+	&& chown -R nginx:0 /var/cache/nginx \
+	&& chown -R nginx:0 /var/lib/nginx \
+	&& apt-get remove --purge -y libcap2-bin \
+	&& rm /etc/nginx/conf.d/* \
+	&& rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /etc/nginx/secrets
+COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
 
 # Uncomment the line below if you would like to add the default.pem to the image
 # and use it as a certificate and key for the default server
 # ADD default.pem /etc/nginx/secrets/default
 
-ENTRYPOINT ["/nginx-ingress"]
+USER nginx
+
+ENTRYPOINT ["/nginx-ingress"]

build/DockerfileForAlpine

@@ -6,14 +6,23 @@ RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
 	&& ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
 	&& ln -sf /proc/1/fd/2 /var/log/nginx/error.log
 
-COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
-
-RUN rm /etc/nginx/conf.d/*
+RUN mkdir -p /etc/nginx/secrets \
+	&& mkdir -p /var/lib/nginx \
+	&& apk add --no-cache libcap \
+	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+	&& chown -R nginx:0 /etc/nginx \
+	&& chown -R nginx:0 /var/cache/nginx \
+	&& chown -R nginx:0 /var/lib/nginx \
+	&& apk del libcap \
+	&& rm /etc/nginx/conf.d/* \
+	&& rm -rf /var/cache/apk/*
 
-RUN mkdir -p /etc/nginx/secrets
+COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
 
 # Uncomment the line below if you would like to add the default.pem to the image
 # and use it as a certificate and key for the default server
 # ADD default.pem /etc/nginx/secrets/default
 
-ENTRYPOINT ["/nginx-ingress"]
+USER nginx
+
+ENTRYPOINT ["/nginx-ingress"]

build/DockerfileForPlus

@@ -16,7 +16,7 @@ RUN chmod 644 /etc/ssl/nginx/*
 # Install NGINX Plus
 RUN set -x \
   && apt-get update \
-  && apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 \
+  && apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin \
   && \
   NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
   found=''; \
@@ -37,6 +37,7 @@ RUN set -x \
   && echo "Acquire::https::plus-pkgs.nginx.com::User-Agent  \"k8s-ic-$IC_VERSION-apt\";" >> /etc/apt/apt.conf.d/90nginx \
   && printf "deb https://plus-pkgs.nginx.com/debian stretch nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \
   && apt-get update && apt-get install -y nginx-plus=${NGINX_PLUS_VERSION} \
+  && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
   && apt-get remove --purge --auto-remove -y gnupg1 \
   && rm -rf /var/lib/apt/lists/* \
   && rm -rf /etc/ssl/nginx \
@@ -46,19 +47,25 @@ RUN set -x \
 # forward nginx access and error logs to stdout and stderr of the ingress
 # controller process
 RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
-	&& ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
-	&& ln -sf /proc/1/fd/2 /var/log/nginx/error.log
+  && ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
+  && ln -sf /proc/1/fd/2 /var/log/nginx/error.log
 
+RUN  mkdir -p /var/lib/nginx \
+  && mkdir -p /etc/nginx/secrets \
+  && chown -R nginx:0 /etc/nginx \
+  && chown -R nginx:0 /var/cache/nginx \
+  && chown -R nginx:0 /var/lib/nginx/ \
+  && apt-get remove --purge -y libcap2-bin \
+  && rm /etc/nginx/conf.d/* 
 
 EXPOSE 80 443
 
 COPY nginx-ingress internal/configs/version1/nginx-plus.ingress.tmpl internal/configs/version1/nginx-plus.tmpl internal/configs/version2/nginx-plus.virtualserver.tmpl  /
 
-RUN rm /etc/nginx/conf.d/* \
-  && mkdir -p /etc/nginx/secrets
-
 # Uncomment the line below if you would like to add the default.pem to the image
 # and use it as a certificate and key for the default server
 # ADD default.pem /etc/nginx/secrets/default
 
+USER nginx
+
 ENTRYPOINT ["/nginx-ingress"]

build/DockerfileWithOpentracing

@@ -33,7 +33,8 @@ RUN set -x \
     && make && make install \
     && cd "$tempDir" \
     && NGINX_VERSION_BUILD=`nginx -v 2>&1` && NGINX_VERSION_BUILD=${NGINX_VERSION_BUILD#*nginx/} \
-    && echo "deb-src http://nginx.org/packages/mainline/debian/ stretch nginx" >> /etc/apt/sources.list \
+    && DEBIAN_VERSION="$(cat /etc/os-release | grep CODENAME | cut -f2 -d '=')" \
+    && echo "deb-src http://nginx.org/packages/mainline/debian/ $DEBIAN_VERSION nginx" >> /etc/apt/sources.list \
     && apt-get update \
     && apt-get build-dep -y nginx=${NGINX_VERSION_BUILD} \
     && wget -O nginx-release-${NGINX_VERSION_BUILD}.tar.gz https://github.com/nginx/nginx/archive/release-${NGINX_VERSION_BUILD}.tar.gz \
@@ -58,26 +59,38 @@ RUN set -x \
 # Final Image
 FROM nginx:${NGINX_VERSION}
 ARG OPENTRACING_CPP_VERSION
+
 # forward nginx access and error logs to stdout and stderr of the ingress
 # controller process
 RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
     && ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
     && ln -sf /proc/1/fd/2 /var/log/nginx/error.log
 
-COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
-
 COPY --from=opentracing-builder /ngx_http_opentracing_module.so /usr/lib/nginx/modules/ngx_http_opentracing_module.so
 COPY --from=opentracing-builder /usr/local/lib/libopentracing.so.${OPENTRACING_CPP_VERSION} /usr/local/lib/libopentracing.so.1
 # Edit the line below to use a different tracer
 COPY --from=tracer-downloader /usr/local/lib/libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so
+
 RUN ldconfig
 
-RUN rm /etc/nginx/conf.d/*
+RUN mkdir -p /var/lib/nginx \
+    && mkdir -p /etc/nginx/secrets \
+    && apt-get update \
+    && apt-get install -y libcap2-bin \
+    && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+    && chown -R nginx:0 /etc/nginx \
+    && chown -R nginx:0 /var/cache/nginx \
+    && chown -R nginx:0 /var/lib/nginx \
+    && apt-get remove --purge -y libcap2-bin \
+    && rm /etc/nginx/conf.d/* \
+    && rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /etc/nginx/secrets
+COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
 
 # Uncomment the line below if you would like to add the default.pem to the image
 # and use it as a certificate and key for the default server
 # ADD default.pem /etc/nginx/secrets/default
 
+USER nginx
+
 ENTRYPOINT ["/nginx-ingress"]

build/DockerfileWithOpentracingForPlus

@@ -4,8 +4,7 @@ ARG JAEGER_VERSION=v0.4.2
 RUN set -x \
     && apt-get update \
     && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates apt-transport-https wget \
-    && wget https://github.com/jaegertracing/jaeger-client-cpp/releases/download/${JAEGER_VERSION}/libjaegertracing_plugin.linux_amd64.so -O /usr/local/lib/libjaegertracing_plugin.so
-
+    && wget https://github.com/jaegertracing/jaeger-client-cpp/releases/download/${JAEGER_VERSION}/libjaegertracing_plugin.linux_amd64.so -O /usr/local/lib/libjaegertracing_plugin.so 
 
 # Final Image
 FROM debian:stretch-slim
@@ -27,7 +26,7 @@ RUN chmod 644 /etc/ssl/nginx/*
 # Install NGINX Plus
 RUN set -x \
     && apt-get update \
-    && apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 \
+    && apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin \
     && \
     NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
     found=''; \
@@ -51,6 +50,7 @@ RUN set -x \
     nginx-plus=${NGINX_PLUS_VERSION} \
     # Install OpenTracing module
     nginx-plus-module-opentracing=${NGINX_OPENTRACING_MODULE_VERSION} \
+    && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
     && apt-get remove --purge --auto-remove -y gnupg1 \
     && rm -rf /var/lib/apt/lists/* \
     && rm -rf /etc/ssl/nginx \
@@ -63,19 +63,25 @@ RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
     && ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
     && ln -sf /proc/1/fd/2 /var/log/nginx/error.log
 
-
 # Edit the line below to use a different tracer
 COPY --from=tracer-downloader /usr/local/lib/libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so
 
+RUN mkdir -p /var/lib/nginx \
+    && mkdir -p /etc/nginx/secrets \
+    && chown -R nginx:0 /etc/nginx \
+    && chown -R nginx:0 /var/cache/nginx \
+    && chown -R nginx:0 /var/lib/nginx/ \
+    && apt-get remove --purge -y libcap2-bin \
+    && rm /etc/nginx/conf.d/*
+
 EXPOSE 80 443
 
 COPY nginx-ingress internal/configs/version1/nginx-plus.ingress.tmpl internal/configs/version1/nginx-plus.tmpl internal/configs/version2/nginx-plus.virtualserver.tmpl  /
 
-RUN rm /etc/nginx/conf.d/* \
-    && mkdir -p /etc/nginx/secrets
-
 # Uncomment the line below if you would like to add the default.pem to the image
 # and use it as a certificate and key for the default server
 # ADD default.pem /etc/nginx/secrets/default
 
-ENTRYPOINT ["/nginx-ingress"]
+USER nginx
+
+ENTRYPOINT ["/nginx-ingress"]

cmd/nginx-ingress/main.go

@@ -12,14 +12,13 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/nginxinc/kubernetes-ingress/internal/configs/version2"
-	"github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors"
-
 	"github.com/golang/glog"
 	"github.com/nginxinc/kubernetes-ingress/internal/configs"
 	"github.com/nginxinc/kubernetes-ingress/internal/configs/version1"
+	"github.com/nginxinc/kubernetes-ingress/internal/configs/version2"
 	"github.com/nginxinc/kubernetes-ingress/internal/k8s"
 	"github.com/nginxinc/kubernetes-ingress/internal/metrics"
+	"github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors"
 	"github.com/nginxinc/kubernetes-ingress/internal/nginx"
 	k8s_nginx "github.com/nginxinc/kubernetes-ingress/pkg/client/clientset/versioned"
 	conf_scheme "github.com/nginxinc/kubernetes-ingress/pkg/client/clientset/versioned/scheme"
@@ -350,7 +349,7 @@ func main() {
 
 	var plusClient *client.NginxClient
 	if *nginxPlus && !useFakeNginxManager {
-		httpClient := getSocketClient("/var/run/nginx-plus-api.sock")
+		httpClient := getSocketClient("/var/lib/nginx/nginx-plus-api.sock")
 		plusClient, err = client.NewNginxClient(httpClient, "http://nginx-plus-api/api")
 		if err != nil {
 			glog.Fatalf("Failed to create NginxClient for Plus: %v", err)
@@ -362,7 +361,7 @@ func main() {
 		if *nginxPlus {
 			go metrics.RunPrometheusListenerForNginxPlus(*prometheusMetricsListenPort, plusClient, registry)
 		} else {
-			httpClient := getSocketClient("/var/run/nginx-status.sock")
+			httpClient := getSocketClient("/var/lib/nginx/nginx-status.sock")
 			client, err := metrics.NewNginxMetricsClient(httpClient)
 			if err != nil {
 				glog.Fatalf("Error creating the Nginx client for Prometheus metrics: %v", err)

deployments/daemon-set/nginx-ingress.yaml

@@ -29,6 +29,14 @@ spec:
           hostPort: 443
        #- name: prometheus
          #containerPort: 9113
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsUser: 101 #nginx
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
         env:
         - name: POD_NAMESPACE
           valueFrom:

deployments/daemon-set/nginx-plus-ingress.yaml

@@ -29,6 +29,14 @@ spec:
           hostPort: 443
        #- name: prometheus
          #containerPort: 9113
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsUser: 101 #nginx
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
         env:
         - name: POD_NAMESPACE
           valueFrom:

deployments/deployment/nginx-ingress.yaml

@@ -28,6 +28,14 @@ spec:
           containerPort: 443
        #- name: prometheus
          #containerPort: 9113
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsUser: 101 #nginx
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
         env:
         - name: POD_NAMESPACE
           valueFrom:

deployments/deployment/nginx-plus-ingress.yaml

@@ -28,6 +28,14 @@ spec:
           containerPort: 443
        #- name: prometheus
          #containerPort: 9113
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsUser: 101 #nginx
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
         env:
         - name: POD_NAMESPACE
           valueFrom:

deployments/helm-chart/templates/controller-daemonset.yaml

@@ -58,6 +58,14 @@ spec:
         - name: prometheus
           containerPort: {{ .Values.prometheus.port }}
 {{- end }}
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsUser: 101 #nginx
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
         env:
         - name: POD_NAMESPACE
           valueFrom:

deployments/helm-chart/templates/controller-deployment.yaml

@@ -58,6 +58,14 @@ spec:
 {{- end }}
         resources:
 {{ toYaml .Values.controller.resources | indent 10 }}
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsUser: 101 #nginx
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
         env:
         - name: POD_NAMESPACE
           valueFrom:

internal/configs/version1/nginx-plus.tmpl

@@ -1,5 +1,4 @@
 
-user  nginx;
 worker_processes  {{.WorkerProcesses}};
 {{- if .WorkerRlimitNofile}}
 worker_rlimit_nofile {{.WorkerRlimitNofile}};{{end}}
@@ -11,7 +10,7 @@ worker_shutdown_timeout {{.WorkerShutdownTimeout}};{{end}}
 daemon off;
 
 error_log  /var/log/nginx/error.log {{.ErrorLogLevel}};
-pid        /var/run/nginx.pid;
+pid        /var/lib/nginx/nginx.pid;
 
 {{- if .OpenTracingLoadModule}}
 load_module modules/ngx_http_opentracing_module.so;
@@ -80,7 +79,7 @@ http {
     opentracing on;
     {{end}}
     {{if .OpenTracingLoadModule}}
-    opentracing_load_tracer {{ .OpenTracingTracer }} /etc/tracer-config.json;
+    opentracing_load_tracer {{ .OpenTracingTracer }} /var/lib/nginx/tracer-config.json;
     {{end}}
 
     {{if .ResolverAddresses}}
@@ -145,7 +144,7 @@ http {
 
     # NGINX Plus API over unix socket
     server {
-        listen unix:/var/run/nginx-plus-api.sock;
+        listen unix:/var/lib/nginx/nginx-plus-api.sock;
         access_log off;
 
         {{if .OpenTracingEnabled}}