Build and use custom nginx container (nginx#934)

Replace the publicly available nginx image with a custom nginx image.

Problem: Using the publicly available nginx image requires users to create 
ConfigMaps for the nginx.conf file and the njs module and mount them to 
the NKG Pod as volumes. This pattern is not extensible and adds extra 
steps for developers and users. Additionally, an init container is required 
in order to set up the nginx config environment.

Solution: Build and use a custom nginx container. The nginx.conf and njs module 
are now baked into the nginx image. This eliminates the need for ConfigMaps. 
The config directories /etc/nginx/conf.d and /etc/nginx/secrets are created as 
volumes and mounted to the Pod with a group ID 1001. This allows the control 
plane to write to the directories and nginx to read from them. Both the nginx 
and nginx-gateway processes run under group ID 1001 but have different 
user IDs (101 and 102). The nginx container runs as user 101 instead of root
and runs with the minimum set of capabilities.

Author: miledxz

.github/workflows/ci.yml

@@ -55,9 +55,6 @@ jobs:
       - name: Check if generated go files are up to date
         run: make generate && git diff --exit-code
 
-      - name: Check if njs-modules yaml is up to date
-        run: make generate-njs-yaml && git diff --exit-code
-
       - name: Check if generated manifests are up to date
         run: make generate-manifests && git diff --exit-code
 
@@ -173,8 +170,8 @@ jobs:
       - name: Docker Buildx
         uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
 
-      - name: Docker meta
-        id: meta
+      - name: NKG Docker meta
+        id: nkg-meta
         uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
         with:
           images: |
@@ -185,37 +182,67 @@ jobs:
             type=ref,event=pr
             type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
 
-      - name: Build Docker Image
+      - name: NGINX Docker meta
+        id: nginx-meta
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: Build NKG Docker Image
         uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
         with:
           file: build/Dockerfile
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: ${{ steps.nkg-meta.outputs.tags }}
           context: "."
           target: goreleaser
           load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=nkg
+          cache-to: type=gha,scope=nkg,mode=max
+          pull: true
+
+      - name: Build NGINX Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: build/Dockerfile.nginx
+          tags: ${{ steps.nginx-meta.outputs.tags }}
+          context: "."
+          load: true
+          cache-from: type=gha,scope=nginx
+          cache-to: type=gha,scope=nginx,mode=max
           pull: true
+          build-args: |
+            NJS_DIR=internal/mode/static/nginx/modules/src
+            NGINX_CONF_DIR=internal/mode/static/nginx/conf
 
       - name: Deploy Kubernetes
         id: k8s
         run: |
           kube_config=${{ github.workspace }}/deploy/helm-chart/kube-${{ github.run_id }}-helm
           make create-kind-cluster KIND_KUBE_CONFIG=${kube_config}
           echo "KUBECONFIG=${kube_config}" >> "$GITHUB_ENV"
-          kind load docker-image ${{ steps.meta.outputs.tags }}
+          kind load docker-image ${{ steps.nkg-meta.outputs.tags }} ${{ steps.nginx-meta.outputs.tags }}
           kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml
           kubectl wait --for=condition=complete job/gateway-api-admission-patch job/gateway-api-admission -n gateway-system
 
       - name: Install Chart
         run: >
           helm install
-          helm-$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          helm-$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2)
           .
           --wait
           --create-namespace
-          --set controller.image.repository=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
-          --set controller.image.tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          --set nginxGateway.image.repository=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 1)
+          --set nginxGateway.image.tag=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2)
+          --set nginxGateway.image.pullPolicy=Never
+          --set nginx.image.repository=$(echo ${{ steps.nginx-meta.outputs.tags }} | cut -d ":" -f 1)
+          --set nginx.image.tag=$(echo ${{ steps.nginx-meta.outputs.tags }} | cut -d ":" -f 2)
+          --set nginx.image.pullPolicy=Never
           --set service.type=NodePort
           -n nginx-gateway
         working-directory: ${{ github.workspace }}/deploy/helm-chart
@@ -224,6 +251,10 @@ jobs:
     name: Build Image
     runs-on: ubuntu-22.04
     needs: [vars, binary]
+    strategy:
+      fail-fast: false
+      matrix:
+        container: [nkg, nginx]
     permissions:
       contents: read # for docker/build-push-action to read repo content
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -259,7 +290,7 @@ jobs:
         uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
         with:
           images: |
-            name=ghcr.io/nginxinc/nginx-kubernetes-gateway
+            name=ghcr.io/nginxinc/nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '/nginx' || '' }}
           tags: |
             type=semver,pattern={{version}}
             type=edge
@@ -269,42 +300,46 @@ jobs:
       - name: Build Docker Image
         uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
         with:
-          file: build/Dockerfile
+          file: ${{ matrix.container == 'nginx' && 'build/Dockerfile.nginx' || 'build/Dockerfile' }}
           context: "."
-          target: goreleaser
+          target: ${{ matrix.container == 'nkg' && 'goreleaser' || '' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           load: ${{ github.event_name == 'pull_request' }}
           push: ${{ github.event_name != 'pull_request' }}
           platforms: ${{ github.event_name != 'pull_request' && env.platforms || '' }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=${{ matrix.container }}
+          cache-to: type=gha,scope=${{ matrix.container }},mode=max
           pull: true
           no-cache: ${{ github.event_name != 'pull_request' }}
           sbom: ${{ github.event_name != 'pull_request' }}
           provenance: false
+          build-args: |
+            NJS_DIR=internal/mode/static/nginx/modules/src
+            NGINX_CONF_DIR=internal/mode/static/nginx/conf
+
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@41f05d9ecffa2ed3f1580af306000f734b733e54 # 0.11.2
         continue-on-error: true
         with:
-          image-ref: ghcr.io/nginxinc/nginx-kubernetes-gateway:${{ steps.meta.outputs.version }}
+          image-ref: ghcr.io/nginxinc/nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '/nginx' || '' }}:${{ steps.meta.outputs.version }}
           format: "sarif"
-          output: "trivy-results-nginx-kubernetes-gateway.sarif"
+          output: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif
           ignore-unfixed: "true"
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@5b6282e01c62d02e720b81eb8a51204f527c3624 # v2.21.3
         continue-on-error: true
         with:
-          sarif_file: "trivy-results-nginx-kubernetes-gateway.sarif"
+          sarif_file: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif
 
       - name: Upload Scan Results
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         continue-on-error: true
         with:
-          name: "trivy-results-nginx-kubernetes-gateway.sarif"
-          path: "trivy-results-nginx-kubernetes-gateway.sarif"
+          name: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif
+          path: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif
         if: always()
 
   publish-helm:

.github/workflows/conformance.yml

@@ -42,8 +42,8 @@ jobs:
       - name: Docker Buildx
         uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
 
-      - name: Docker meta
-        id: meta
+      - name: NKG Docker meta
+        id: nkg-meta
         uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
         with:
           images: |
@@ -54,10 +54,22 @@ jobs:
             type=ref,event=pr
             type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
 
+      - name: NGINX Docker meta
+        id: nginx-meta
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
       - name: Prepare NKG files
         run: |
-          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
-          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          nkg_prefix=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2)
           make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
         working-directory: ./conformance
 
@@ -67,17 +79,31 @@ jobs:
           version: latest
           args: build --snapshot --clean
 
-      - name: Build Docker Image
+      - name: Build NKG Docker Image
         uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
         with:
           file: build/Dockerfile
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: ${{ steps.nkg-meta.outputs.tags }}
           context: "."
           target: goreleaser
           load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=nkg
+          cache-to: type=gha,scope=nkg,mode=max
+          pull: true
+
+      - name: Build NGINX Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: build/Dockerfile.nginx
+          tags: ${{ steps.nginx-meta.outputs.tags }}
+          context: "."
+          load: true
+          cache-from: type=gha,scope=nginx
+          cache-to: type=gha,scope=nginx,mode=max
           pull: true
+          build-args: |
+            NJS_DIR=internal/mode/static/nginx/modules/src
+            NGINX_CONF_DIR=internal/mode/static/nginx/conf
 
       - name: Update Go Modules
         if: ${{ github.event_name == 'schedule' }}
@@ -104,8 +130,8 @@ jobs:
 
       - name: Setup conformance tests
         run: |
-          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
-          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          nkg_prefix=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2)
           if [ ${{ github.event_name }} == "schedule" ]; then
           export GW_API_VERSION=main
           fi

Makefile

@@ -3,16 +3,18 @@ VERSION = edge
 GIT_COMMIT = $(shell git rev-parse HEAD || echo "unknown")
 DATE = $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 MANIFEST_DIR = $(shell pwd)/deploy/manifests
-NJS_DIR = $(shell pwd)/internal/mode/static/nginx/modules/src
 CHART_DIR = $(shell pwd)/deploy/helm-chart
+NGINX_CONF_DIR = internal/mode/static/nginx/conf
+NJS_DIR = internal/mode/static/nginx/modules/src
 
 # go build flags - should not be overridden by the user
 GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE}
 GO_LINKER_FLAGS_OPTIMIZATIONS = -s -w
 GO_LINKER_FLAGS = $(GO_LINKER_FLAGS_OPTIMIZATIONS) $(GO_LINKER_FlAGS_VARS)
 
 # variables that can be overridden by the user
-PREFIX ?= nginx-kubernetes-gateway## The name of the image. For example, nginx-kubernetes-gateway
+PREFIX ?= nginx-kubernetes-gateway## The name of the NKG image. For example, nginx-kubernetes-gateway
+NGINX_PREFIX ?= $(PREFIX)/nginx## The name of the nginx image. For example: nginx-kubernetes-gateway/nginx
 TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 0.3.0
 TARGET ?= local## The target of the build. Possible values: local and container
 KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig
@@ -21,19 +23,29 @@ ARCH ?= amd64## The architecture of the image and/or binary. For example: amd64
 override HELM_TEMPLATE_COMMON_ARGS += --set creator=template --set nameOverride=nginx-gateway## The common options for the Helm template command.
 override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --set service.create=false## The options to be passed to the full Helm templating command only.
 override DOCKER_BUILD_OPTIONS += --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg DATE=$(DATE)## The options for the docker build command. For example, --pull
-
+override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR)
 .DEFAULT_GOAL := help
 
 .PHONY: help
 help: Makefile ## Display this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n    make \033[36m<target>\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'
 	@grep -E '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $< | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-.PHONY: container
-container: build ## Build the container
-	@docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code)
+.PHONY: build-images
+build-images: build-nkg-image build-nginx-image ## Build the NKG and nginx docker images
+
+.PHONY: build-nkg-image
+build-nkg-image: check-for-docker build ## Build the NKG docker image
 	docker build --platform linux/$(ARCH) $(strip $(DOCKER_BUILD_OPTIONS)) --target $(strip $(TARGET)) -f build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) .
 
+.PHONY: build-nginx-image
+build-nginx-image: check-for-docker ## Build the custom nginx image
+	docker build --platform linux/$(ARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) .
+
+.PHONY: check-for-docker
+check-for-docker: ## Check if Docker is installed
+	@docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code)
+
 .PHONY: build
 build: ## Build the binary
 ifeq (${TARGET},local)
@@ -103,10 +115,6 @@ njs-unit-test: ## Run unit tests for the njs httpmatches module
 		node:18 \
 		/bin/bash -c "npm install && npm test && npm run clean"
 
-.PHONY: generate-njs-yaml
-generate-njs-yaml: ## Generate the njs-modules ConfigMap
-	kubectl create configmap njs-modules --from-file=$(NJS_DIR)/httpmatches.js --dry-run=client --output=yaml > $(strip $(MANIFEST_DIR))/njs-modules.yaml
-
 .PHONY: lint-helm
 lint-helm: ## Run the helm chart linter
 	helm lint $(CHART_DIR)
@@ -116,8 +124,8 @@ debug-build: GO_LINKER_FLAGS=$(GO_LINKER_FlAGS_VARS)
 debug-build: ADDITIONAL_GO_BUILD_FLAGS=-gcflags "all=-N -l"
 debug-build: build ## Build binary with debug info, symbols, and no optimizations
 
-.PHONY: debug-container
-debug-container: debug-build container ## Build container with debug binary
+.PHONY: build-nkg-debug-image
+build-nkg-debug-image: debug-build build-nkg-image ## Build NKG image with debug binary
 
 .PHONY: generate-manifests
 generate-manifests: ## Generate manifests using Helm.

README.md

@@ -21,7 +21,7 @@ Learn about our [design principles](/docs/developer/design-principles.md) and [a
 
 1. [Quick Start on a kind cluster](docs/running-on-kind.md).
 2. [Install](docs/installation.md) NGINX Kubernetes Gateway.
-3. [Build](docs/building-the-image.md) an NGINX Kubernetes Gateway container image from source or use a pre-built image
+3. [Build](docs/building-the-images.md) an NGINX Kubernetes Gateway container image from source or use a pre-built image
    available
    on [GitHub Container Registry](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway).
 4. Deploy various [examples](examples).
@@ -59,7 +59,7 @@ The following table lists the software versions NGINX Kubernetes Gateway support
 
 | NGINX Kubernetes Gateway | Gateway API | Kubernetes | NGINX OSS |
 |--------------------------|-------------|------------|-----------|
-| Edge                     | 0.7.1       | 1.21+      | 1.25.x *  |
+| Edge                     | 0.7.1       | 1.21+      | 1.25.1    |
 | 0.5.0                    | 0.7.1       | 1.21+      | 1.25.x *  |
 | 0.4.0                    | 0.7.1       | 1.21+      | 1.25.x *  |
 | 0.3.0                    | 0.6.2       | 1.21+      | 1.23.x *  |

build/Dockerfile

@@ -31,7 +31,7 @@ COPY dist/gateway_linux_$TARGETARCH*/gateway /usr/bin/
 RUN setcap 'cap_kill=+ep' /usr/bin/gateway
 
 FROM scratch as common
-USER 1001:1001
+USER 102:1001
 ENTRYPOINT [ "/usr/bin/gateway" ]
 
 FROM common as container

build/Dockerfile.nginx

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1.4
+FROM nginx:1.25.1-alpine
+
+ARG NJS_DIR
+ARG NGINX_CONF_DIR
+
+RUN apk update && apk add --no-cache libcap \
+    && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
+    && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+    && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+    && apk del libcap
+
+COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
+COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
+
+RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx
+
+USER 101:1001