From a8682f3795cf6737a2308b93c9bc4b2a8ae9493e Mon Sep 17 00:00:00 2001 From: Vimal Kumar Date: Tue, 22 Feb 2022 17:15:30 +0530 Subject: [PATCH] LOG-2263: Vector: Added prometheus_exporter sink --- internal/generator/vector/conf_test.go | 25 ++++++++++++++ internal/generator/vector/metrics.go | 47 ++++++++++++++++++++++++++ internal/generator/vector/outputs.go | 10 ++++++ internal/generator/vector/sources.go | 10 ++++++ 4 files changed, 92 insertions(+) create mode 100644 internal/generator/vector/metrics.go diff --git a/internal/generator/vector/conf_test.go b/internal/generator/vector/conf_test.go index dcf8dbd752..ae316cb81d 100644 --- a/internal/generator/vector/conf_test.go +++ b/internal/generator/vector/conf_test.go @@ -118,6 +118,9 @@ type = "file" ignore_older_secs = 600 include = ["/var/log/oauth-apiserver.audit.log"] +[sources.internal_metrics] +type = "internal_metrics" + [transforms.container_logs] type = "remap" inputs = ["raw_container_logs"] @@ -233,6 +236,12 @@ key_file = "/var/run/ocp-collector/secrets/kafka-receiver-1/tls.key" crt_file = "/var/run/ocp-collector/secrets/kafka-receiver-1/tls.crt" ca_file = "/var/run/ocp-collector/secrets/kafka-receiver-1/ca-bundle.crt" enabled = true + +[sinks.prometheus_output] +type = "prometheus_exporter" +inputs = ["internal_metrics"] +address = "0.0.0.0:24231" +default_namespace = "collector" `, }), Entry("with complex spec for elastic-search", generator.ConfGenerateTest{ @@ -313,6 +322,9 @@ type = "file" ignore_older_secs = 600 include = ["/var/log/oauth-apiserver.audit.log"] +[sources.internal_metrics] +type = "internal_metrics" + [transforms.container_logs] type = "remap" inputs = ["raw_container_logs"] @@ -563,6 +575,11 @@ id_key = "_id" key_file = "/var/run/ocp-collector/secrets/es-2/tls.key" crt_file = "/var/run/ocp-collector/secrets/es-2/tls.crt" ca_file = "/var/run/ocp-collector/secrets/es-2/ca-bundle.crt" +[sinks.prometheus_output] +type = "prometheus_exporter" +inputs = ["internal_metrics"] +address = "0.0.0.0:24231" +default_namespace = "collector" `, }), Entry("with multiple pipelines for elastic-search", generator.ConfGenerateTest{ @@ -649,6 +666,9 @@ type = "file" ignore_older_secs = 600 include = ["/var/log/oauth-apiserver.audit.log"] +[sources.internal_metrics] +type = "internal_metrics" + [transforms.container_logs] type = "remap" inputs = ["raw_container_logs"] @@ -907,6 +927,11 @@ id_key = "_id" key_file = "/var/run/ocp-collector/secrets/es-2/tls.key" crt_file = "/var/run/ocp-collector/secrets/es-2/tls.crt" ca_file = "/var/run/ocp-collector/secrets/es-2/ca-bundle.crt" +[sinks.prometheus_output] +type = "prometheus_exporter" +inputs = ["internal_metrics"] +address = "0.0.0.0:24231" +default_namespace = "collector" `, }), ) diff --git a/internal/generator/vector/metrics.go b/internal/generator/vector/metrics.go new file mode 100644 index 0000000000..69151e2b26 --- /dev/null +++ b/internal/generator/vector/metrics.go @@ -0,0 +1,47 @@ +package vector + +const ( + InternalMetricsSourceName = "internal_metrics" + PrometheusOutputSinkName = "prometheus_output" + PrometheusExporterAddress = "0.0.0.0:24231" +) + +type InternalMetrics struct { + ID string + ScrapeIntervalSec int +} + +func (InternalMetrics) Name() string { + return "internalMetricsTemplate" +} + +//#namespace = "collector" +//#scrape_interval_secs = {{.ScrapeIntervalSec}} +func (i InternalMetrics) Template() string { + return ` +{{define "` + i.Name() + `" -}} +[sources.{{.ID}}] +type = "internal_metrics" +{{end}} +` +} + +type PrometheusExporter struct { + ID string + Inputs string + Address string +} + +func (p PrometheusExporter) Name() string { + return "PrometheusExporterTemplate" +} + +func (p PrometheusExporter) Template() string { + return `{{define "` + p.Name() + `" -}} +[sinks.{{.ID}}] +type = "prometheus_exporter" +inputs = {{.Inputs}} +address = "{{.Address}}" +default_namespace = "collector" +{{end}}` +} diff --git a/internal/generator/vector/outputs.go b/internal/generator/vector/outputs.go index c7392ec3ca..7824389d71 100644 --- a/internal/generator/vector/outputs.go +++ b/internal/generator/vector/outputs.go @@ -3,6 +3,7 @@ package vector import ( logging "github.com/openshift/cluster-logging-operator/apis/logging/v1" "github.com/openshift/cluster-logging-operator/internal/generator" + "github.com/openshift/cluster-logging-operator/internal/generator/vector/helpers" "github.com/openshift/cluster-logging-operator/internal/generator/vector/output/elasticsearch" "github.com/openshift/cluster-logging-operator/internal/generator/vector/output/kafka" "github.com/openshift/cluster-logging-operator/internal/generator/vector/output/loki" @@ -25,5 +26,14 @@ func Outputs(clspec *logging.ClusterLoggingSpec, secrets map[string]*corev1.Secr outputs = generator.MergeElements(outputs, elasticsearch.Conf(o, inputs, secret, op)) } } + outputs = append(outputs, PrometheusOutput(PrometheusOutputSinkName, []string{InternalMetricsSourceName})) return outputs } + +func PrometheusOutput(id string, inputs []string) generator.Element { + return PrometheusExporter{ + ID: id, + Inputs: helpers.MakeInputs(inputs...), + Address: PrometheusExporterAddress, + } +} diff --git a/internal/generator/vector/sources.go b/internal/generator/vector/sources.go index 91f69f9dd4..fff10617d6 100644 --- a/internal/generator/vector/sources.go +++ b/internal/generator/vector/sources.go @@ -14,6 +14,7 @@ import ( func Sources(spec *logging.ClusterLogForwarderSpec, op generator.Options) []generator.Element { return generator.MergeElements( LogSources(spec, op), + MetricsSources(InternalMetricsSourceName), ) } @@ -87,3 +88,12 @@ func ExcludeContainerPaths() string { ", ", )) } + +func MetricsSources(id string) []generator.Element { + return []generator.Element{ + InternalMetrics{ + ID: id, + ScrapeIntervalSec: 2, + }, + } +}