Skip to content

Commit 8986c96

Browse files
Raul MarreroRulox
Raul Marrero
authored andcommitted
Add support for Opentracing
1 parent 722c673 commit 8986c96

19 files changed

+478
-11
lines changed

build/DockerfileWithOpentracing

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
ARG NGINX_VERSION=1.17.1
2+
ARG OPENTRACING_CPP_VERSION=1.5.1
3+
4+
FROM nginx:${NGINX_VERSION} AS opentracing-builder
5+
ARG OPENTRACING_CPP_VERSION
6+
7+
RUN set -x \
8+
&& apt-get update \
9+
&& apt-get install --no-install-recommends --no-install-suggests -y \
10+
libcurl4-openssl-dev \
11+
build-essential \
12+
cmake \
13+
git \
14+
ca-certificates \
15+
pkg-config \
16+
wget \
17+
libz-dev \
18+
automake \
19+
autogen \
20+
autoconf \
21+
libtool \
22+
libpcre3 \
23+
libpcre3-dev \
24+
&& tempDir="$(mktemp -d)" \
25+
&& chmod 777 "$tempDir" \
26+
&& git clone https://github.com/opentracing-contrib/nginx-opentracing src \
27+
&& cd "$tempDir" \
28+
&& git clone -b v$OPENTRACING_CPP_VERSION https://github.com/opentracing/opentracing-cpp.git \
29+
&& cd opentracing-cpp \
30+
&& mkdir .build && cd .build \
31+
&& cmake -DCMAKE_BUILD_TYPE=Release \
32+
-DBUILD_TESTING=OFF .. \
33+
&& make && make install \
34+
&& cd "$tempDir" \
35+
&& NGINX_VERSION_BUILD=`nginx -v 2>&1` && NGINX_VERSION_BUILD=${NGINX_VERSION_BUILD#*nginx/} \
36+
&& echo "deb-src http://nginx.org/packages/mainline/debian/ stretch nginx" >> /etc/apt/sources.list \
37+
&& apt-get update \
38+
&& apt-get build-dep -y nginx=${NGINX_VERSION_BUILD} \
39+
&& wget -O nginx-release-${NGINX_VERSION_BUILD}.tar.gz https://github.com/nginx/nginx/archive/release-${NGINX_VERSION_BUILD}.tar.gz \
40+
&& tar zxf nginx-release-${NGINX_VERSION_BUILD}.tar.gz \
41+
&& cd nginx-release-${NGINX_VERSION_BUILD} \
42+
&& auto/configure \
43+
--with-compat \
44+
--add-dynamic-module=/src/opentracing \
45+
--with-debug \
46+
&& make modules \
47+
&& cp objs/ngx_http_opentracing_module.so /ngx_http_opentracing_module.so
48+
49+
50+
FROM debian:stretch-slim AS tracer-downloader
51+
ARG JAEGER_VERSION=v0.4.2
52+
RUN set -x \
53+
&& apt-get update \
54+
&& apt-get install --no-install-recommends --no-install-suggests -y ca-certificates apt-transport-https wget \
55+
&& wget https://github.com/jaegertracing/jaeger-client-cpp/releases/download/${JAEGER_VERSION}/libjaegertracing_plugin.linux_amd64.so -O /usr/local/lib/libjaegertracing_plugin.so
56+
57+
58+
# Final Image
59+
FROM nginx:${NGINX_VERSION}
60+
ARG OPENTRACING_CPP_VERSION
61+
# forward nginx access and error logs to stdout and stderr of the ingress
62+
# controller process
63+
RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
64+
&& ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
65+
&& ln -sf /proc/1/fd/2 /var/log/nginx/error.log
66+
67+
COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
68+
69+
COPY --from=opentracing-builder /ngx_http_opentracing_module.so /usr/lib/nginx/modules/ngx_http_opentracing_module.so
70+
COPY --from=opentracing-builder /usr/local/lib/libopentracing.so.${OPENTRACING_CPP_VERSION} /usr/local/lib/libopentracing.so.1
71+
# Edit the line below to use a different tracer
72+
COPY --from=tracer-downloader /usr/local/lib/libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so
73+
RUN ldconfig
74+
75+
RUN rm /etc/nginx/conf.d/*
76+
77+
RUN mkdir -p /etc/nginx/secrets
78+
79+
# Uncomment the line below if you would like to add the default.pem to the image
80+
# and use it as a certificate and key for the default server
81+
# ADD default.pem /etc/nginx/secrets/default
82+
83+
ENTRYPOINT ["/nginx-ingress"]
+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
FROM debian:stretch-slim AS tracer-downloader
2+
ARG JAEGER_VERSION=v0.4.2
3+
4+
RUN set -x \
5+
&& apt-get update \
6+
&& apt-get install --no-install-recommends --no-install-suggests -y ca-certificates apt-transport-https wget \
7+
&& wget https://github.com/jaegertracing/jaeger-client-cpp/releases/download/${JAEGER_VERSION}/libjaegertracing_plugin.linux_amd64.so -O /usr/local/lib/libjaegertracing_plugin.so
8+
9+
10+
# Final Image
11+
FROM debian:stretch-slim
12+
LABEL maintainer="NGINX Docker Maintainers <[email protected]>"
13+
14+
ENV NGINX_PLUS_VERSION 18-1~stretch
15+
ARG IC_VERSION
16+
17+
# Download certificate and key from the customer portal (https://cs.nginx.com)
18+
# and copy to the build context
19+
COPY nginx-repo.crt /etc/ssl/nginx/
20+
COPY nginx-repo.key /etc/ssl/nginx/
21+
22+
# Make sure the certificate and key have correct permissions
23+
RUN chmod 644 /etc/ssl/nginx/*
24+
25+
# Install NGINX Plus
26+
RUN set -x \
27+
&& apt-get update \
28+
&& apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 \
29+
&& \
30+
NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
31+
found=''; \
32+
for server in \
33+
ha.pool.sks-keyservers.net \
34+
hkp://keyserver.ubuntu.com:80 \
35+
hkp://p80.pool.sks-keyservers.net:80 \
36+
pgp.mit.edu \
37+
; do \
38+
echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
39+
apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
40+
done; \
41+
test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
42+
echo "Acquire::https::plus-pkgs.nginx.com::Verify-Peer \"true\";" >> /etc/apt/apt.conf.d/90nginx \
43+
&& echo "Acquire::https::plus-pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \
44+
&& echo "Acquire::https::plus-pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \
45+
&& echo "Acquire::https::plus-pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \
46+
&& echo "Acquire::https::plus-pkgs.nginx.com::User-Agent \"k8s-ic-$IC_VERSION-apt\";" >> /etc/apt/apt.conf.d/90nginx \
47+
&& printf "deb https://plus-pkgs.nginx.com/debian stretch nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \
48+
&& apt-get update && apt-get install -y \
49+
nginx-plus=${NGINX_PLUS_VERSION} \
50+
# Install OpenTracing module
51+
nginx-plus-module-opentracing \
52+
&& apt-get remove --purge --auto-remove -y gnupg1 \
53+
&& rm -rf /var/lib/apt/lists/* \
54+
&& rm -rf /etc/ssl/nginx \
55+
&& rm /etc/apt/apt.conf.d/90nginx /etc/apt/sources.list.d/nginx-plus.list
56+
57+
58+
# forward nginx access and error logs to stdout and stderr of the ingress
59+
# controller process
60+
RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
61+
&& ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
62+
&& ln -sf /proc/1/fd/2 /var/log/nginx/error.log
63+
64+
65+
# Edit the line below to use a different tracer
66+
COPY --from=tracer-downloader /usr/local/lib/libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so
67+
68+
EXPOSE 80 443
69+
70+
COPY nginx-ingress internal/configs/version1/nginx-plus.ingress.tmpl internal/configs/version1/nginx-plus.tmpl internal/configs/version2/nginx-plus.virtualserver.tmpl /
71+
72+
RUN rm /etc/nginx/conf.d/* \
73+
&& mkdir -p /etc/nginx/secrets
74+
75+
# Uncomment the line below if you would like to add the default.pem to the image
76+
# and use it as a certificate and key for the default server
77+
# ADD default.pem /etc/nginx/secrets/default
78+
79+
ENTRYPOINT ["/nginx-ingress"]

build/README.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,12 @@ The **Makefile** contains the following main variables for you to customize (eit
7474
* **VERSION** -- the current version of the controller.
7575
* **TAG** -- the tag added to the image. It's set to the value of the `VERSION` variable by default.
7676
* **PUSH_TO_GCR**. If you’re running your Kubernetes in GCE and using Google Container Registry, make sure that `PUSH_TO_GCR = 1`. This means using the `gcloud docker push` command to push the image, which is convenient when pushing images to GCR. By default, the variable is unset and the regular `docker push` command is used to push the image to the registry.
77-
* **DOCKERFILE** -- the path to a Dockerfile. We provide three Dockerfiles:
77+
* **DOCKERFILE** -- the path to a Dockerfile. We provide five Dockerfiles:
7878
1. `Dockerfile`, for building a debian-based image with NGINX. It's used by default.
7979
1. `DockerfileForAlpine`, for building an alpine-based image with NGINX.
8080
1. `DockerfileForPlus`, for building an debian-based image with NGINX Plus.
81+
1. `DockerfileWithOpentracing`, for building a debian-based image with NGINX, [opentracing](https://github.com/opentracing-contrib/nginx-opentracing) module and the [Jaeger](https://www.jaegertracing.io/) tracer.
82+
1. `DockerfileWithOpentracingForPlus`, for building a debian-based image with NGINX Plus, [opentracing](https://github.com/opentracing-contrib/nginx-opentracing) module and the [Jaeger](https://www.jaegertracing.io/) tracer.
8183
* **GENERATE_DEFAULT_CERT_AND_KEY** - The Ingress controller requires a certificate and a key for the default HTTP/HTTPS server. You can reference them in a TLS Secret in a command-line argument to the Ingress controller. As an alternative, you can add a file in the PEM format with your certificate and key to the image as `/etc/nginx/secrets/default`. Optionally, you can generate a self-signed certificate and a key during the build process. Set `GENERATE_DEFAULT_CERT_AND_KEY` to `1` to generate a certificate and a key in the `default.pem` file. Note that you must add the `ADD` instruction in the Dockerfile to copy the cert and the key to the image. The default value of `GENERATE_DEFAULT_CERT_AND_KEY` is `0`.
8284
* **DOCKER_BUILD_OPTIONS** -- the [options](https://docs.docker.com/engine/reference/commandline/build/#options) for the `docker build` command. For example, `--pull`.
8385
* **BUILD_IN_CONTAINER** -- By default, to compile the controller we use the [golang](https://hub.docker.com/_/golang/) container that we run as part of the building process. If you want to compile the controller using your local golang environment:

cmd/nginx-ingress/main.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,17 @@ func main() {
327327
glog.Fatalf("Error generating NGINX main config: %v", err)
328328
}
329329
nginxManager.CreateMainConfig(content)
330-
nginxManager.UpdateConfigVersionFile()
330+
331+
nginxManager.UpdateConfigVersionFile(ngxConfig.OpenTracingLoadModule)
332+
333+
nginxManager.SetOpenTracing(ngxConfig.OpenTracingLoadModule)
334+
335+
if ngxConfig.OpenTracingLoadModule {
336+
err := nginxManager.CreateOpenTracingTracerConfig(cfgParams.MainOpenTracingTracerConfig)
337+
if err != nil {
338+
glog.Fatalf("Error creating OpenTracing tracer config file: %v", err)
339+
}
340+
}
331341

332342
nginxDone := make(chan error, 1)
333343
nginxManager.Start(nginxDone)

docs/configmap-and-annotations.md

+8
Original file line numberDiff line numberDiff line change
@@ -184,3 +184,11 @@ spec:
184184
| N/A | `stream-snippets` | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](../examples/tcp-udp). |
185185
| N/A | `main-template` | Sets the main NGINX configuration template. | By default the template is read from the file in the container. | [Custom Templates](../examples/custom-templates). |
186186
| N/A | `ingress-template` | Sets the NGINX configuration template for an Ingress resource. | By default the template is read from the file on the container. | [Custom Templates](../examples/custom-templates). |
187+
188+
189+
### Modules
190+
| Annotation | ConfigMap Key | Description | Default | Example |
191+
| ---------- | -------------- | ----------- | ------- | ------- |
192+
| N/A | `opentracing` | Enables [OpenTracing](https://opentracing.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Note: requires the Ingress Controller image with OpenTracing module and a tracer. See the [docs](./opentracing.md) for more information. | `False` | [Support for OpenTracing](../examples/opentracing/README.md). |
193+
| N/A | `opentracing-tracer` | Sets the path to the vendor tracer binary plugin.| N/A | [Support for OpenTracing](../examples/opentracing/README.md). |
194+
| N/A | `opentracing-tracer-config` | Sets the tracer configuration in JSON format.| N/A | [Support for OpenTracing](../examples/opentracing/README.md). |

docs/opentracing.md

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# OpenTracing Support
2+
The Ingress Controller supports [OpenTracing](http://opentracing.io/) with the third-party module [opentracing-contrib/nginx-opentracing](https://github.com/opentracing-contrib/nginx-opentracing).
3+
4+
This document explains how to use OpenTracing with the Ingress Controller. Additionally, we have an [example](../examples/opentracing/README.md) on how to enable OpenTracing for a simple web application using Jaeger as a tracer.
5+
6+
## Prerequisites
7+
1. **Use the Ingress Controller image with OpenTracing.** The default Ingress Controller images don’t include the OpenTraing module. To use OpenTracing, you need to build the image with that module. Follow the build instructions to build the image using `DockerfileWithOpentracing` for NGINX or `DockerfileWithOpentracingForPlus` for NGINX Plus.
8+
By default, the Dockerfiles install Jaeger as a tracer. However, it is possible to replace Jaeger with other supported [tracers](https://github.com/opentracing-contrib/nginx-opentracing#building-from-source). For that, please modify the Dockerfile accordingly:
9+
1. Change the download line in the tracer-downloader stage of the Dockerfile to download the right tracer.
10+
1. Edit the COPY line of the final image to copy the previously downloaded tracer to the image
11+
12+
1. **Load the OpenTracing module.** You need to load the module with the configuration for the chosen tracer using the following ConfigMap keys:
13+
* `opentracing-tracer`: sets the path to the vendor tracer binary plugin. This is the path you used in the COPY line of step *ii* above.
14+
* `opentracing-tracer-config`: sets the tracer configuration in JSON format.
15+
16+
Below an example on how to use those keys to load the module with Jaeger tracer:
17+
```yaml
18+
opentracing-tracer: "/usr/local/lib/libjaegertracing_plugin.so"
19+
opentracing-tracer-config: |
20+
{
21+
"service_name": "nginx-ingress",
22+
"sampler": {
23+
"type": "const",
24+
"param": 1
25+
},
26+
"reporter": {
27+
"localAgentHostPort": "jaeger-agent.default.svc.cluster.local:6831"
28+
}
29+
}
30+
```
31+
32+
## Enable OpenTracing Globally
33+
To enable OpenTracing globally (for all Ingress, VirtualServer and VirtualServerRoute resources), set the `opentracing` ConfigMap key to `True`:
34+
35+
```yaml
36+
opentracing: True
37+
```
38+
39+
## Enable/Disable OpenTracing per Ingress Resource
40+
41+
It is possible to use annotations to enable or disable OpenTracing for a specific Ingress Resource. As mentioned in the prerequisites section, both `opentracing-tracer` and `opentracing-tracer-config` must be configured.
42+
43+
Consider the following two cases:
44+
1. OpenTracing is globally disabled.
45+
1. To enable OpenTracing for a specific Ingress Resource, use the server snippet annotation:
46+
```yaml
47+
nginx.org/server-snippets: |
48+
opentracing on;
49+
```
50+
1. To enable OpenTracing for specific paths, (1) you need to use [Mergeable Ingress resources](../examples/mergeable-ingress-types/README.md) and (2) use the location snippets annotation to enable OpenTracing for the paths of a specific Minion Ingress resource:
51+
```yaml
52+
nginx.org/location-snippets: |
53+
opentracing on;
54+
```
55+
56+
2. OpenTracing is globally enabled:
57+
1. To disable OpenTracing for a specific Ingress Resource, use the server snippet annotation:
58+
```yaml
59+
nginx.org/server-snippets: |
60+
opentracing off;
61+
```
62+
63+
1. To disable OpenTracing for specific paths, (1) you need to use [Mergeable Ingress resources](../examples/mergeable-ingress-types/README.md) and (2) use the location snippets annotation to disable OpenTracing for the paths of a specific Minion Ingress resource:
64+
```yaml
65+
nginx.org/location-snippets: |
66+
opentracing off;
67+
```
68+
69+
## Customize OpenTracing
70+
71+
You can customize OpenTracing though the supported [OpenTracing module directives](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md). Use the snippets ConfigMap keys or annotations to insert those directives into the http, server or location contexts of the generated NGINX configuration.
72+
73+
For example, to propagate the active span context for upstream requests, it is required to set the `opentracing_propagate_context` directive, which you can add to an Ingress resource using the location snippets annotation:
74+
75+
```yaml
76+
nginx.org/location-snippets: |
77+
opentracing_propagate_context;
78+
```
79+
80+
**Note**: `opentracing_propagate_context` and `opentracing_grpc_propagate_context` directives can be used in http, server or location contexts according to the [module documentation](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md#opentracing_propagate_context). However, because of the way the module works and how the Ingress Controller generates the NGINX configuration, it is only possible to use the directive in the location context.

examples/opentracing/README.md

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
2+
# OpenTracing Support
3+
4+
In this example we deploy the NGINX or NGINX Plus Ingress Controller and a simple web application. Then we enable OpenTracing and use a tracer (Jaeger) for tracing the requests that go through NGINX or NGINX Plus to the web application.
5+
6+
## Prerequisites
7+
8+
The default Ingress Controller images don’t include the OpenTracing module required for this example. See Step 1 of the [Prerequisites section](../../docs/opentracing.md#Prerequisites) in the OpenTracing doc for the instructions on how to get the right image with Jaeger tracer.
9+
10+
## Step 1 - Deploy Ingress Controller and the Cafe App
11+
12+
Follow steps 1-3 of the [complete example](../complete-example/README.md) to deploy the Ingress Controller and the cafe app. Make sure to use the Ingress Controller image with the OpenTracing module and the tracer installed.
13+
14+
## Step 2 - Deploy a Tracer
15+
16+
1. Use the [all-in-one dev template](https://github.com/jaegertracing/jaeger-kubernetes#development-setup) to deploy Jaeger in the default namespace. **Note:** This template should be only used for development or testing.
17+
```
18+
kubectl create -f https://raw.githubusercontent.com/jaegertracing/jaeger-kubernetes/master/all-in-one/jaeger-all-in-one-template.yml
19+
```
20+
21+
2. Wait for the jaeger pod to be ready:
22+
```
23+
$ kubectl get pod
24+
25+
NAME READY STATUS
26+
jaeger-6c996dbcd9-j5jzf 1/1 Running
27+
```
28+
29+
## Step 3 - Enable OpenTracing
30+
1. Update the ConfigMap with the keys required to load OpenTracing module with Jaeger and enable OpenTracing for all Ingress resources.
31+
```
32+
kubectl apply -f nginx-config.yaml
33+
```
34+
35+
## Step 4 - Test Tracing
36+
1. Make a request to the app.
37+
38+
**Note:** $IC_HTTPS_PORT and $IC_IP env variables should have been set from the Prerequisites step in the complete-example installation instructions.
39+
```
40+
curl --resolve cafe.example.com:$IC_HTTPS_PORT:$IC_IP https://cafe.example.com:$IC_HTTPS_PORT/coffee --insecure
41+
```
42+
1. Forward a local port to the Jaeger UI port on the Jaeger pod:
43+
```
44+
kubectl port-forward <YOUR_JAEGER_POD> 16686:16686
45+
```
46+
1. Open Jaeger dashboard in your browser available via http://localhost:16686. Search for the traces by specifying the name of the service to `nginx-ingress` and clicking `Find Traces`. You will see:
47+
48+
![Jaeger UI](./jaeger-ui.png)

examples/opentracing/jaeger-ui.png

295 KB
Loading
+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
kind: ConfigMap
2+
apiVersion: v1
3+
metadata:
4+
name: nginx-config
5+
namespace: nginx-ingress
6+
data:
7+
opentracing: "True"
8+
opentracing-tracer: "/usr/local/lib/libjaegertracing_plugin.so"
9+
opentracing-tracer-config: |
10+
{
11+
"service_name": "nginx-ingress",
12+
"sampler": {
13+
"type": "const",
14+
"param": 1
15+
},
16+
"reporter": {
17+
"localAgentHostPort": "jaeger-agent.default.svc.cluster.local:6831"
18+
}
19+
}
20+

internal/configs/config_params.go

+4
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ type ConfigParams struct {
5454
MainKeepaliveRequests int64
5555
VariablesHashBucketSize uint64
5656
VariablesHashMaxSize uint64
57+
MainOpenTracingLoadModule bool
58+
MainOpenTracingEnabled bool
59+
MainOpenTracingTracer string
60+
MainOpenTracingTracerConfig string
5761

5862
RealIPHeader string
5963
SetRealIPFrom []string

0 commit comments

Comments
 (0)