Merge pull request #1348 from vimalk78/log-2263-prom-exporter

LOG-2263: Vector: Added prometheus_exporter sink

Author: openshift-merge-robot

internal/generator/vector/conf_test.go

@@ -118,6 +118,9 @@ type = "file"
 ignore_older_secs = 600
 include = ["/var/log/oauth-apiserver.audit.log"]
 
+[sources.internal_metrics]
+type = "internal_metrics"
+
 [transforms.container_logs]
 type = "remap"
 inputs = ["raw_container_logs"]
@@ -233,6 +236,12 @@ key_file = "/var/run/ocp-collector/secrets/kafka-receiver-1/tls.key"
 crt_file = "/var/run/ocp-collector/secrets/kafka-receiver-1/tls.crt"
 ca_file = "/var/run/ocp-collector/secrets/kafka-receiver-1/ca-bundle.crt"
 enabled = true
+
+[sinks.prometheus_output]
+type = "prometheus_exporter"
+inputs = ["internal_metrics"]
+address = "0.0.0.0:24231"
+default_namespace = "collector"
 `,
 		}),
 		Entry("with complex spec for elastic-search", generator.ConfGenerateTest{
@@ -313,6 +322,9 @@ type = "file"
 ignore_older_secs = 600
 include = ["/var/log/oauth-apiserver.audit.log"]
 
+[sources.internal_metrics]
+type = "internal_metrics"
+
 [transforms.container_logs]
 type = "remap"
 inputs = ["raw_container_logs"]
@@ -563,6 +575,11 @@ id_key = "_id"
 key_file = "/var/run/ocp-collector/secrets/es-2/tls.key"
 crt_file = "/var/run/ocp-collector/secrets/es-2/tls.crt"
 ca_file = "/var/run/ocp-collector/secrets/es-2/ca-bundle.crt"
+[sinks.prometheus_output]
+type = "prometheus_exporter"
+inputs = ["internal_metrics"]
+address = "0.0.0.0:24231"
+default_namespace = "collector"
 `,
 		}),
 		Entry("with multiple pipelines for elastic-search", generator.ConfGenerateTest{
@@ -649,6 +666,9 @@ type = "file"
 ignore_older_secs = 600
 include = ["/var/log/oauth-apiserver.audit.log"]
 
+[sources.internal_metrics]
+type = "internal_metrics"
+
 [transforms.container_logs]
 type = "remap"
 inputs = ["raw_container_logs"]
@@ -907,6 +927,11 @@ id_key = "_id"
 key_file = "/var/run/ocp-collector/secrets/es-2/tls.key"
 crt_file = "/var/run/ocp-collector/secrets/es-2/tls.crt"
 ca_file = "/var/run/ocp-collector/secrets/es-2/ca-bundle.crt"
+[sinks.prometheus_output]
+type = "prometheus_exporter"
+inputs = ["internal_metrics"]
+address = "0.0.0.0:24231"
+default_namespace = "collector"
 `,
 		}),
 	)

internal/generator/vector/metrics.go

@@ -0,0 +1,47 @@
+package vector
+
+const (
+	InternalMetricsSourceName = "internal_metrics"
+	PrometheusOutputSinkName  = "prometheus_output"
+	PrometheusExporterAddress = "0.0.0.0:24231"
+)
+
+type InternalMetrics struct {
+	ID                string
+	ScrapeIntervalSec int
+}
+
+func (InternalMetrics) Name() string {
+	return "internalMetricsTemplate"
+}
+
+//#namespace = "collector"
+//#scrape_interval_secs = {{.ScrapeIntervalSec}}
+func (i InternalMetrics) Template() string {
+	return `
+{{define "` + i.Name() + `" -}}
+[sources.{{.ID}}]
+type = "internal_metrics"
+{{end}}
+`
+}
+
+type PrometheusExporter struct {
+	ID      string
+	Inputs  string
+	Address string
+}
+
+func (p PrometheusExporter) Name() string {
+	return "PrometheusExporterTemplate"
+}
+
+func (p PrometheusExporter) Template() string {
+	return `{{define "` + p.Name() + `" -}}
+[sinks.{{.ID}}]
+type = "prometheus_exporter"
+inputs = {{.Inputs}}
+address = "{{.Address}}"
+default_namespace = "collector"
+{{end}}`
+}

internal/generator/vector/outputs.go

@@ -3,6 +3,7 @@ package vector
 import (
 	logging "github.com/openshift/cluster-logging-operator/apis/logging/v1"
 	"github.com/openshift/cluster-logging-operator/internal/generator"
+	"github.com/openshift/cluster-logging-operator/internal/generator/vector/helpers"
 	"github.com/openshift/cluster-logging-operator/internal/generator/vector/output/elasticsearch"
 	"github.com/openshift/cluster-logging-operator/internal/generator/vector/output/kafka"
 	"github.com/openshift/cluster-logging-operator/internal/generator/vector/output/loki"
@@ -25,5 +26,14 @@ func Outputs(clspec *logging.ClusterLoggingSpec, secrets map[string]*corev1.Secr
 			outputs = generator.MergeElements(outputs, elasticsearch.Conf(o, inputs, secret, op))
 		}
 	}
+	outputs = append(outputs, PrometheusOutput(PrometheusOutputSinkName, []string{InternalMetricsSourceName}))
 	return outputs
 }
+
+func PrometheusOutput(id string, inputs []string) generator.Element {
+	return PrometheusExporter{
+		ID:      id,
+		Inputs:  helpers.MakeInputs(inputs...),
+		Address: PrometheusExporterAddress,
+	}
+}

internal/generator/vector/sources.go

@@ -14,6 +14,7 @@ import (
 func Sources(spec *logging.ClusterLogForwarderSpec, op generator.Options) []generator.Element {
 	return generator.MergeElements(
 		LogSources(spec, op),
+		MetricsSources(InternalMetricsSourceName),
 	)
 }
 
@@ -87,3 +88,12 @@ func ExcludeContainerPaths() string {
 		", ",
 	))
 }
+
+func MetricsSources(id string) []generator.Element {
+	return []generator.Element{
+		InternalMetrics{
+			ID:                id,
+			ScrapeIntervalSec: 2,
+		},
+	}
+}