TEP-0124 implement opentelemetry and jaeger tracing

Adds opentelemetry instrumentation code to pipelinerun and taskrun
reconcilers. Also made required changes in the main.go to include jaeger
as tracing backend

Author: kmjayadeep

cmd/controller/main.go

@@ -17,10 +17,12 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"log"
 	"net/http"
 	"os"
+	"time"
 
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
@@ -36,11 +38,23 @@ import (
 	"knative.dev/pkg/injection"
 	"knative.dev/pkg/injection/sharedmain"
 	"knative.dev/pkg/signals"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	tracesdk "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
+	"go.opentelemetry.io/otel/trace"
 )
 
 const (
 	// ControllerLogKey is the name of the logger for the controller cmd
 	ControllerLogKey = "tekton-pipelines-controller"
+	// TraceProvider name for pipeline reconciler
+	TracerProviderPipelineRun = "pipeline-reconciler"
+	// TraceProvider name for taskrun reconciler
+	TracerProviderTaskRun = "taskrun-reconciler"
 )
 
 func main() {
@@ -102,17 +116,80 @@ func main() {
 		log.Fatal(http.ListenAndServe(":"+port, mux))
 	}()
 
+	// initialize opentelemetry
+	tpPipelineRun, err := tracerProvider(TracerProviderPipelineRun)
+	if err != nil {
+		log.Printf("failed to initialize tracerProvider for pipelinerun, falling back to no-op provider, %s", err.Error())
+		tpPipelineRun = trace.NewNoopTracerProvider()
+	}
+	tpTaskrun, err := tracerProvider(TracerProviderTaskRun)
+	if err != nil {
+		log.Printf("failed to initialize tracerProvider for taskrun, falling back to no-op provider, %s", err.Error())
+		tpTaskrun = trace.NewNoopTracerProvider()
+	}
+	otel.SetTextMapPropagator(propagation.TraceContext{})
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	ctx = filteredinformerfactory.WithSelectors(ctx, v1beta1.ManagedByLabelKey)
 	sharedmain.MainWithConfig(ctx, ControllerLogKey, cfg,
-		taskrun.NewController(opts, clock.RealClock{}),
-		pipelinerun.NewController(opts, clock.RealClock{}),
+		taskrun.NewController(opts, clock.RealClock{}, tpTaskrun),
+		pipelinerun.NewController(opts, clock.RealClock{}, tpPipelineRun),
 		run.NewController(),
 		resolutionrequest.NewController(clock.RealClock{}),
 		// TODO(jerop, abayer) uncomment after we align on retries in customruns
 		// customrun.NewController(),
 	)
+
+	// Cleanly shutdown and flush telemetry when the application exits.
+	defer func(ctx context.Context) {
+		// Do not make the application hang when it is shutdown.
+		ctx, cancel = context.WithTimeout(ctx, time.Second*5)
+		defer cancel()
+
+		// shutdown is only needed when tracerProvider is inialized with jaeger
+		// not needed when tracerProvider is NewNoopTracerProvider
+		if tp, ok := tpPipelineRun.(*tracesdk.TracerProvider); ok {
+			tp.Shutdown(ctx)
+		}
+		if tp, ok := tpTaskrun.(*tracesdk.TracerProvider); ok {
+			tp.Shutdown(ctx)
+		}
+	}(ctx)
 }
 
 func handler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
+
+// tracerProvider returns an OpenTelemetry TracerProvider configured to use
+// the Jaeger exporter that will send spans to the provided url. The returned
+// TracerProvider will also use a Resource configured with all the information
+// about the application.
+func tracerProvider(service string) (trace.TracerProvider, error) {
+	// Create the Jaeger exporter
+	// The following env variables are used by the sdk for creating the exporter
+	// - OTEL_EXPORTER_JAEGER_ENDPOINT is the HTTP endpoint for sending spans directly to a collector.
+	// - OTEL_EXPORTER_JAEGER_USER is the username to be sent as authentication to the collector endpoint.
+	// - OTEL_EXPORTER_JAEGER_PASSWORD is the password to be sent as authentication to the collector endpoint.
+
+	if _, e := os.LookupEnv("OTEL_EXPORTER_JAEGER_ENDPOINT"); !e {
+		// jaeger endpoint is not defined, disable tracing and return no-op tracerProvider
+		return trace.NewNoopTracerProvider(), nil
+	}
+
+	exp, err := jaeger.New(jaeger.WithCollectorEndpoint())
+	if err != nil {
+		return nil, err
+	}
+	// Initialize tracerProvider with the jaeger exporter
+	tp := tracesdk.NewTracerProvider(
+		tracesdk.WithBatcher(exp),
+		// Record information about the service in a Resource.
+		tracesdk.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceNameKey.String(service),
+		)),
+	)
+	return tp, nil
+}

config/controller.yaml

@@ -124,6 +124,13 @@ spec:
           value: /etc/ssl/certs
         - name: METRICS_DOMAIN
           value: tekton.dev/pipeline
+       # The following variables can be uncommented with correct values to enable Jaeger tracing
+       #- name: OTEL_EXPORTER_JAEGER_ENDPOINT
+       #  value: http://jaeger-collector.jaeger:14268/api/traces
+       #- name: OTEL_EXPORTER_JAEGER_USER
+       #  value: username
+       #- name: OTEL_EXPORTER_JAEGER_PASSWORD
+       #  value: password
         securityContext:
           allowPrivilegeEscalation: false
           capabilities: