Merge pull request #1498 from mboersma/export-to-jaeger

Export traces to Jaeger and update OpenTelemetry

Author: k8s-ci-robot

Tiltfile

@@ -121,11 +121,23 @@ COPY manager .
 def observability():
     instrumentation_key = os.getenv("AZURE_INSTRUMENTATION_KEY", "")
     if instrumentation_key == "":
-        warn("AZURE_INSTRUMENTATION_KEY is not set, so tracing won't be exported to Application Insights")
+        warn("AZURE_INSTRUMENTATION_KEY is not set, so traces won't be exported to Application Insights")
+        trace_links = []
+    else:
+        trace_links = [link("https://ms.portal.azure.com/#blade/HubsExtension/BrowseResource/resourceType/microsoft.insights%2Fcomponents", "App Insights")]
     k8s_yaml(helm("./hack/observability/opentelemetry/chart",
          name="opentelemetry-collector", namespace="capz-system",
          values=["./hack/observability/opentelemetry/values.yaml"],
          set=["config.exporters.azuremonitor.instrumentation_key="+instrumentation_key]))
+    k8s_yaml(helm("./hack/observability/jaeger/chart",
+         name="jaeger-all-in-one", namespace="capz-system",
+         set=["crd.install=false", "rbac.create=false",
+              "resources.limits.cpu=200m", "resources.limits.memory=256Mi"]))
+    k8s_resource(workload="jaeger-all-in-one", new_name="traces: jaeger-all-in-one",
+        port_forwards=[port_forward(16686, name="View traces", link_path='/search?service=capz')],
+        links=trace_links)
+    k8s_resource(workload="prometheus-operator", new_name="metrics: prometheus-operator",
+        port_forwards=[port_forward(9090, name="View metrics")], extra_pod_selectors=[{"app": "prometheus"}])
 
 # Build CAPZ and add feature gates
 def capz():

docs/book/src/developers/development.md

@@ -213,19 +213,29 @@ make delete-workload-cluster
 > Check out the [troubleshooting guide](../topics/troubleshooting.md) for common errors you might run into.
 
 #### Viewing Telemetry
-The CAPZ controller emits tracing and metrics data. When run in Tilt, the KinD cluster is provisioned with a development
-deployment of OpenTelemetry for distributed tracing, and Prometheus for metrics scraping and visualization.
 
-The OpenTelemetry and Prometheus deployments are for development purposes only. These illustrate the hooks for tracing and
-metrics, but lack the robustness of production cluster deployments.
+The CAPZ controller emits tracing and metrics data. When run in Tilt, the KinD management cluster is
+provisioned with development deployments of OpenTelemetry for collecting distributed traces, Jaeger
+for viewing traces, and Prometheus for scraping and visualizing metrics.
 
-After the Tilt cluster has been initialized, if you followed the [tracing documentation](../../../../hack/observability/opentelemetry/readme.md),
-you can view distributed traces in the Azure Portal. Open the App Insights resource identified by the
-`AZURE_INSTRUMENTATION_KEY` you specified, choose "Transaction search" from the "Investigate" menu on the left,
-and click "Refresh" or make a specific query of the trace data.
+The OpenTelemetry, Jaeger, and Prometheus deployments are for development purposes only. These
+illustrate the hooks for tracing and metrics, but lack the robustness of production cluster
+deployments. For example, the Jaeger "all-in-one" component only keeps traces in memory, not in a
+persistent store.
 
-To view metrics, run `kubectl port-forward -n capz-system prometheus-prometheus-0 9090` and open
-`http://localhost:9090` to see the Prometheus UI.
+To view traces in the Jaeger interface, wait until the Tilt cluster is fully initialized. Then open
+the Tilt web interface, select the "traces: jaeger-all-in-one" resource, and click "View traces"
+near the top of the screen. Or visit http://localhost:16686/ in your browser. <!-- markdown-link-check-disable-line -->
+
+To view traces in App Insights, follow the
+[tracing documentation](../../../../hack/observability/opentelemetry/readme.md) before running
+`make tilt-up`. Then open the Azure Portal in your browser. Find the App Insights resource you
+specified in `AZURE_INSTRUMENTATION_KEY`, choose "Transaction search" on the left, and click
+"Refresh" to see recent trace data.
+
+To view metrics in the Prometheus interface, open the Tilt web interface, select the
+"metrics: prometheus-operator" resource, and click "View metrics" near the top of the screen. Or
+visit http://localhost:9090/ in your browser. <!-- markdown-link-check-disable-line -->
 
 ### Manual Testing
 

go.mod

@@ -25,12 +25,15 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/spf13/pflag v1.0.5
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0
-	go.opentelemetry.io/otel v0.20.0
-	go.opentelemetry.io/otel/exporters/metric/prometheus v0.20.0
-	go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.1-0.20210504183141-c99d5e999c69
-	go.opentelemetry.io/otel/sdk v0.20.1-0.20210504183141-c99d5e999c69
-	go.opentelemetry.io/otel/trace v0.20.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.22.0
+	go.opentelemetry.io/otel v1.0.0-RC2
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.0-RC2
+	go.opentelemetry.io/otel/exporters/prometheus v0.22.0
+	go.opentelemetry.io/otel/metric v0.22.0
+	go.opentelemetry.io/otel/sdk v1.0.0-RC2
+	go.opentelemetry.io/otel/sdk/export/metric v0.22.0
+	go.opentelemetry.io/otel/sdk/metric v0.22.0
+	go.opentelemetry.io/otel/trace v1.0.0-RC2
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/mod v0.4.2
 	k8s.io/api v0.21.2