Add pressure stall information metrics

issues: #3052, #3083, kubernetes/enhancements#4205

This change adds metrics for pressure stall information, that indicate
why some or all tasks of a cgroupv2 have waited due to resource
congestion (cpu, memory, io). The change exposes this information by
including the _PSIStats_ of each controller in it's stats, i.e.
_CPUStats.PSI_, _MemoryStats.PSI_ and _DiskStats.PSI_.

The information is additionally exposed as Prometheus metrics. The
metrics follow the naming outlined by the prometheus/node-exporter,
where stalled eq full and waiting eq some.

```
container_pressure_cpu_stalled_seconds_total
container_pressure_cpu_waiting_seconds_total
container_pressure_memory_stalled_seconds_total
container_pressure_memory_waiting_seconds_total
container_pressure_io_stalled_seconds_total
container_pressure_io_waiting_seconds_total
```

Signed-off-by: Felix Ehrenpfort <[email protected]>

Author: xinau

Diff for: cmd/cadvisor_test.go

@@ -112,8 +112,7 @@ func TestToIncludedMetrics(t *testing.T) {
 			container.ResctrlMetrics:                 struct{}{},
 			container.CPUSetMetrics:                  struct{}{},
 			container.OOMMetrics:                     struct{}{},
-			container.PSITotalMetrics:                struct{}{},
-			container.PSIAvgMetrics:                  struct{}{},
+			container.PressureMetrics:                struct{}{},
 		},
 		container.AllMetrics,
 		{},

Diff for: cmd/go.mod

@@ -130,4 +130,3 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/opencontainers/runc => github.com/dqminh/runc v0.0.0-20220513155811-6414629ada8a

Diff for: container/factory.go

@@ -66,8 +66,7 @@ const (
 	ResctrlMetrics                 MetricKind = "resctrl"
 	CPUSetMetrics                  MetricKind = "cpuset"
 	OOMMetrics                     MetricKind = "oom_event"
-	PSITotalMetrics                MetricKind = "psi_total"
-	PSIAvgMetrics                  MetricKind = "psi_avg"
+	PressureMetrics                MetricKind = "pressure"
 )
 
 // AllMetrics represents all kinds of metrics that cAdvisor supported.
@@ -93,8 +92,7 @@ var AllMetrics = MetricSet{
 	ResctrlMetrics:                 struct{}{},
 	CPUSetMetrics:                  struct{}{},
 	OOMMetrics:                     struct{}{},
-	PSITotalMetrics:                struct{}{},
-	PSIAvgMetrics:                  struct{}{},
+	PressureMetrics:                struct{}{},
 }
 
 // AllNetworkMetrics represents all network metrics that cAdvisor supports.

Diff for: container/libcontainer/handler.go

@@ -763,20 +763,6 @@ func (h *Handler) GetProcesses() ([]int, error) {
 	return pids, nil
 }
 
-// Convert libcontainer cgroups.PSIData to info.PSIData
-func convertPSIData(from *cgroups.PSIData, to *info.PSIData) {
-	to.Avg10 = from.Avg10
-	to.Avg60 = from.Avg60
-	to.Avg300 = from.Avg300
-	to.Total = from.Total
-}
-
-// Convert libcontainer cgroups.PSIStats to info.PSIStats
-func convertPSI(from *cgroups.PSIStats, to *info.PSIStats) {
-	convertPSIData(&from.Some, &to.Some)
-	convertPSIData(&from.Full, &to.Full)
-}
-
 // Convert libcontainer stats to info.ContainerStats.
 func setCPUStats(s *cgroups.Stats, ret *info.ContainerStats, withPerCPU bool) {
 	ret.Cpu.Usage.User = s.CpuStats.CpuUsage.UsageInUsermode
@@ -785,8 +771,7 @@ func setCPUStats(s *cgroups.Stats, ret *info.ContainerStats, withPerCPU bool) {
 	ret.Cpu.CFS.Periods = s.CpuStats.ThrottlingData.Periods
 	ret.Cpu.CFS.ThrottledPeriods = s.CpuStats.ThrottlingData.ThrottledPeriods
 	ret.Cpu.CFS.ThrottledTime = s.CpuStats.ThrottlingData.ThrottledTime
-
-	convertPSI(&s.CpuStats.PSI, &ret.Cpu.PSI)
+	setPSIStats(s.CpuStats.PSI, &ret.Cpu.PSI)
 
 	if !withPerCPU {
 		return
@@ -808,17 +793,15 @@ func setDiskIoStats(s *cgroups.Stats, ret *info.ContainerStats) {
 	ret.DiskIo.IoWaitTime = diskStatsCopy(s.BlkioStats.IoWaitTimeRecursive)
 	ret.DiskIo.IoMerged = diskStatsCopy(s.BlkioStats.IoMergedRecursive)
 	ret.DiskIo.IoTime = diskStatsCopy(s.BlkioStats.IoTimeRecursive)
-
-	convertPSI(&s.BlkioStats.PSI, &ret.DiskIo.PSI)
+	setPSIStats(s.BlkioStats.PSI, &ret.DiskIo.PSI)
 }
 
 func setMemoryStats(s *cgroups.Stats, ret *info.ContainerStats) {
 	ret.Memory.Usage = s.MemoryStats.Usage.Usage
 	ret.Memory.MaxUsage = s.MemoryStats.Usage.MaxUsage
 	ret.Memory.Failcnt = s.MemoryStats.Usage.Failcnt
 	ret.Memory.KernelUsage = s.MemoryStats.KernelUsage.Usage
-
-	convertPSI(&s.MemoryStats.PSI, &ret.Memory.PSI)
+	setPSIStats(s.MemoryStats.PSI, &ret.Memory.PSI)
 
 	if cgroups.IsCgroup2UnifiedMode() {
 		ret.Memory.Cache = s.MemoryStats.Stats["file"]
@@ -904,6 +887,22 @@ func setHugepageStats(s *cgroups.Stats, ret *info.ContainerStats) {
 	}
 }
 
+func setPSIData(d *cgroups.PSIData, ret *info.PSIData) {
+	if d != nil {
+		ret.Total = d.Total
+		ret.Avg10 = d.Avg10
+		ret.Avg60 = d.Avg60
+		ret.Avg300 = d.Avg300
+	}
+}
+
+func setPSIStats(s *cgroups.PSIStats, ret *info.PSIStats) {
+	if s != nil {
+		setPSIData(&s.Full, &ret.Full)
+		setPSIData(&s.Some, &ret.Some)
+	}
+}
+
 // read from pids path not cpu
 func setThreadsStats(s *cgroups.Stats, ret *info.ContainerStats) {
 	if s != nil {

Diff for: container/libcontainer/handler_test.go

@@ -110,17 +110,17 @@ func TestSetCPUStats(t *testing.T) {
 				UsageInKernelmode: 734746 * nanosecondsInSeconds / clockTicks,
 				UsageInUsermode:   2767637 * nanosecondsInSeconds / clockTicks,
 			},
-			PSI: cgroups.PSIStats{
-				Some: cgroups.PSIData{
-					Avg10:  0.1,
+			PSI: &cgroups.PSIStats{
+				Full: cgroups.PSIData{
+					Avg10:  0.3,
 					Avg60:  0.2,
-					Avg300: 0.3,
+					Avg300: 0.1,
 					Total:  100,
 				},
-				Full: cgroups.PSIData{
-					Avg10:  0.4,
-					Avg60:  0.5,
-					Avg300: 0.6,
+				Some: cgroups.PSIData{
+					Avg10:  0.6,
+					Avg60:  0.4,
+					Avg300: 0.2,
 					Total:  200,
 				},
 			},
@@ -138,16 +138,16 @@ func TestSetCPUStats(t *testing.T) {
 				Total:  33802947350272,
 			},
 			PSI: info.PSIStats{
-				Some: info.PSIData{
-					Avg10:  0.1,
+				Full: info.PSIData{
+					Avg10:  0.3,
 					Avg60:  0.2,
-					Avg300: 0.3,
+					Avg300: 0.1,
 					Total:  100,
 				},
-				Full: info.PSIData{
-					Avg10:  0.4,
-					Avg60:  0.5,
-					Avg300: 0.6,
+				Some: info.PSIData{
+					Avg10:  0.6,
+					Avg60:  0.4,
+					Avg300: 0.2,
 					Total:  200,
 				},
 			},

Diff for: info/v1/container.go

@@ -261,16 +261,24 @@ func (ci *ContainerInfo) StatsEndTime() time.Time {
 	return ret
 }
 
-type PSIData struct {
-	Avg10  float64 `json:"avg10"`
-	Avg60  float64 `json:"avg60"`
-	Avg300 float64 `json:"avg300"`
-	Total  uint64  `json:"total"`
-}
-
+// PSI statistics for an individual resource.
 type PSIStats struct {
-	Some PSIData `json:"some,omitempty"`
+	// PSI data for all tasks of in the cgroup.
 	Full PSIData `json:"full,omitempty"`
+	// PSI data for some tasks in the cgroup.
+	Some PSIData `json:"some,omitempty"`
+}
+
+type PSIData struct {
+	// Total time duration for tasks in the cgroup have waited due to congestion.
+	// Unit: nanoseconds.
+	Total uint64 `json:"total"`
+	// The average (in %) tasks have waited due to congestion over a 10 second window.
+	Avg10 float64 `json:"avg10"`
+	// The average (in %) tasks have waited due to congestion over a 60 second window.
+	Avg60 float64 `json:"avg60"`
+	// The average (in %) tasks have waited due to congestion over a 300 second window.
+	Avg300 float64 `json:"avg300"`
 }
 
 // This mirrors kernel internal structure.
@@ -346,9 +354,8 @@ type CpuStats struct {
 	// from LoadStats.NrRunning.
 	LoadAverage int32 `json:"load_average"`
 	// from LoadStats.NrUninterruptible
-	LoadDAverage int32 `json:"load_d_average"`
-
-	PSI PSIStats `json:"psi,omitempty"`
+	LoadDAverage int32    `json:"load_d_average"`
+	PSI          PSIStats `json:"psi"`
 }
 
 type PerDiskStats struct {
@@ -367,8 +374,7 @@ type DiskIoStats struct {
 	IoWaitTime     []PerDiskStats `json:"io_wait_time,omitempty"`
 	IoMerged       []PerDiskStats `json:"io_merged,omitempty"`
 	IoTime         []PerDiskStats `json:"io_time,omitempty"`
-
-	PSI PSIStats `json:"psi,omitempty"`
+	PSI            PSIStats       `json:"psi"`
 }
 
 type HugetlbStats struct {
@@ -428,7 +434,7 @@ type MemoryStats struct {
 	ContainerData    MemoryStatsMemoryData `json:"container_data,omitempty"`
 	HierarchicalData MemoryStatsMemoryData `json:"hierarchical_data,omitempty"`
 
-	PSI PSIStats `json:"psi,omitempty"`
+	PSI PSIStats `json:"psi"`
 }
 
 type CPUSetStats struct {

Diff for: metrics/prometheus.go

@@ -1746,64 +1746,54 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
 		})
 	}
 
-	if includedMetrics.Has(container.PSITotalMetrics) {
+	if includedMetrics.Has(container.PressureMetrics) {
 		c.containerMetrics = append(c.containerMetrics, []containerMetric{
 			{
-				name:        "container_cpu_psi_total_seconds",
-				help:        "Total time spent under cpu pressure in seconds.",
-				valueType:   prometheus.CounterValue,
-				extraLabels: []string{"kind"},
+				name:      "container_pressure_cpu_stalled_seconds_total",
+				help:      "Total time duration no tasks in the container could make progress due to CPU congestion.",
+				valueType: prometheus.CounterValue,
 				getValues: func(s *info.ContainerStats) metricValues {
-					return getPSIValues(s, &s.Cpu.PSI, "total")
+					return metricValues{{value: float64(s.Cpu.PSI.Full.Total) / 1000.0 / 1000.0, timestamp: s.Timestamp}}
 				},
 			}, {
-				name:        "container_memory_psi_total_seconds",
-				help:        "Total container time spent under memory pressure in seconds.",
-				valueType:   prometheus.CounterValue,
-				extraLabels: []string{"kind"},
+				name:      "container_pressure_cpu_waiting_seconds_total",
+				help:      "Total time duration tasks in the container have waited due to CPU congestion.",
+				valueType: prometheus.CounterValue,
 				getValues: func(s *info.ContainerStats) metricValues {
-					return getPSIValues(s, &s.Memory.PSI, "total")
+					return metricValues{{value: float64(s.Cpu.PSI.Some.Total) / 1000.0 / 1000.0, timestamp: s.Timestamp}}
 				},
 			}, {
-				name:        "container_io_psi_total_seconds",
-				help:        "Total time spent under io pressure in seconds.",
-				valueType:   prometheus.CounterValue,
-				extraLabels: []string{"kind"},
+				name:      "container_pressure_memory_stalled_seconds_total",
+				help:      "Total time duration no tasks in the container could make progress due to memory congestion.",
+				valueType: prometheus.CounterValue,
+				getValues: func(s *info.ContainerStats) metricValues {
+					return metricValues{{value: float64(s.Memory.PSI.Full.Total) / 1000.0 / 1000.0, timestamp: s.Timestamp}}
+				},
+			}, {
+				name:      "container_pressure_memory_waiting_seconds_total",
+				help:      "Total time duration tasks in the container have waited due to memory congestion.",
+				valueType: prometheus.CounterValue,
+				getValues: func(s *info.ContainerStats) metricValues {
+					return metricValues{{value: float64(s.Memory.PSI.Some.Total) / 1000.0 / 1000.0, timestamp: s.Timestamp}}
+				},
+			}, {
+				name:      "container_pressure_io_stalled_seconds_total",
+				help:      "Total time duration no tasks in the container could make progress due to IO congestion.",
+				valueType: prometheus.CounterValue,
+				getValues: func(s *info.ContainerStats) metricValues {
+					return metricValues{{value: float64(s.DiskIo.PSI.Full.Total) / 1000.0 / 1000.0, timestamp: s.Timestamp}}
+				},
+			}, {
+				name:      "container_pressure_io_waiting_seconds_total",
+				help:      "Total time duration tasks in the container have waited due to IO congestion.",
+				valueType: prometheus.CounterValue,
 				getValues: func(s *info.ContainerStats) metricValues {
-					return getPSIValues(s, &s.DiskIo.PSI, "total")
+					return metricValues{{value: float64(s.DiskIo.PSI.Some.Total) / 1000.0 / 1000.0, timestamp: s.Timestamp}}
 				},
 			},
 		}...)
 	}
 
-	if includedMetrics.Has(container.PSIAvgMetrics) {
-		makePSIAvgMetric := func(controller, window string) containerMetric {
-			return containerMetric{
-				name:        fmt.Sprintf("container_%s_psi_avg%s_ratio", controller, window),
-				help:        fmt.Sprintf("Ratio of time spent under %s pressure over time window of %s seconds", controller, window),
-				valueType:   prometheus.GaugeValue,
-				extraLabels: []string{"kind"},
-				getValues: func(s *info.ContainerStats) metricValues {
-					switch controller {
-					case "cpu":
-						return getPSIValues(s, &s.Cpu.PSI, "avg"+window)
-					case "memory":
-						return getPSIValues(s, &s.Memory.PSI, "avg"+window)
-					case "io":
-						return getPSIValues(s, &s.DiskIo.PSI, "avg"+window)
-					default:
-						return nil
-					}
-				},
-			}
-		}
-		for _, controller := range []string{"cpu", "memory", "io"} {
-			for _, window := range []string{"10", "60", "300"} {
-				c.containerMetrics = append(c.containerMetrics, makePSIAvgMetric(controller, window))
-			}
-		}
-	}
-
 	return c
 }
 
@@ -2096,23 +2086,3 @@ func getMinCoreScalingRatio(s *info.ContainerStats) metricValues {
 	}
 	return values
 }
-
-func getPSIValues(s *info.ContainerStats, psi *info.PSIStats, psiMetric string) metricValues {
-	v := make(metricValues, 0, 2)
-	switch psiMetric {
-	case "avg10":
-		v = append(v, metricValue{value: psi.Some.Avg10, timestamp: s.Timestamp, labels: []string{"some"}})
-		v = append(v, metricValue{value: psi.Full.Avg10, timestamp: s.Timestamp, labels: []string{"full"}})
-	case "avg60":
-		v = append(v, metricValue{value: psi.Some.Avg60, timestamp: s.Timestamp, labels: []string{"some"}})
-		v = append(v, metricValue{value: psi.Full.Avg60, timestamp: s.Timestamp, labels: []string{"full"}})
-	case "avg300":
-		v = append(v, metricValue{value: psi.Some.Avg300, timestamp: s.Timestamp, labels: []string{"some"}})
-		v = append(v, metricValue{value: psi.Full.Avg300, timestamp: s.Timestamp, labels: []string{"full"}})
-	case "total":
-		// total is measured as microseconds
-		v = append(v, metricValue{value: float64(time.Duration(psi.Some.Total)*time.Microsecond) / float64(time.Second), timestamp: s.Timestamp, labels: []string{"some"}})
-		v = append(v, metricValue{value: float64(time.Duration(psi.Full.Total)*time.Microsecond) / float64(time.Second), timestamp: s.Timestamp, labels: []string{"full"}})
-	}
-	return v
-}