Add ulimit metrics

Author: sahilrihan

container/libcontainer/handler.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -44,6 +45,8 @@ type Handler struct {
 	pidMetricsCache map[int]*info.CpuSchedstat
 }
 
+var whitelistedUlimits = [...]string{"max_open_files"}
+
 func NewHandler(cgroupManager cgroups.Manager, rootFs string, pid int, includedMetrics container.MetricSet) *Handler {
 	return &Handler{
 		cgroupManager:   cgroupManager,
@@ -125,7 +128,7 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) {
 		if !ok {
 			klog.V(4).Infof("Could not find cgroups CPU for container %d", h.pid)
 		} else {
-			stats.Processes, err = processStatsFromProcs(h.rootFs, path)
+			stats.Processes, err = processStatsFromProcs(h.rootFs, path, h.pid)
 			if err != nil {
 				klog.V(4).Infof("Unable to get Process Stats: %v", err)
 			}
@@ -143,7 +146,76 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) {
 	return stats, nil
 }
 
-func processStatsFromProcs(rootFs string, cgroupPath string) (info.ProcessStats, error) {
+func parseUlimit(value string) (int64, error) {
+	num, err := strconv.ParseInt(value, 10, 64)
+	if err != nil {
+		if strings.EqualFold(value, "unlimited") {
+			// -1 implies unlimited except for priority and nice; man limits.conf
+			num = -1
+		} else {
+			// Value is not a number or "unlimited"; return an error
+			return 0, fmt.Errorf("unable to parse limit: %s", value)
+		}
+	}
+	return num, nil
+}
+
+func isUlimitWhitelisted(name string) bool {
+	for _, whitelist := range whitelistedUlimits {
+		if name == whitelist {
+			return true
+		}
+	}
+	return false
+}
+
+func processLimitsFile(fileData string) []info.UlimitSpec {
+	limits := strings.Split(fileData, "\n")
+	ulimits := make([]info.UlimitSpec, 0, len(limits))
+	for _, lim := range limits {
+		// Skip any headers/footers
+		if strings.HasPrefix(lim, "Max") {
+
+			// Line format: Max open files            16384                16384                files
+			fields := regexp.MustCompile("[\\s]{2,}").Split(lim, -1)
+			name := strings.Replace(strings.ToLower(strings.TrimSpace(fields[0])), " ", "_", -1)
+
+			found := isUlimitWhitelisted(name)
+			if !found {
+				continue
+			}
+
+			soft := strings.TrimSpace(fields[1])
+			soft_num, soft_err := parseUlimit(soft)
+
+			hard := strings.TrimSpace(fields[2])
+			hard_num, hard_err := parseUlimit(hard)
+
+			// Omit metric if there were any parsing errors
+			if soft_err == nil && hard_err == nil {
+				ulimitSpec := info.UlimitSpec{
+					Name:      name,
+					SoftLimit: int64(soft_num),
+					HardLimit: int64(hard_num),
+				}
+				ulimits = append(ulimits, ulimitSpec)
+			}
+		}
+	}
+	return ulimits
+}
+
+func processRootProcUlimits(rootFs string, rootPid int) []info.UlimitSpec {
+	filePath := path.Join(rootFs, "/proc", strconv.Itoa(rootPid), "limits")
+	out, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		klog.V(4).Infof("error while listing directory %q to read ulimits: %v", filePath, err)
+		return []info.UlimitSpec{}
+	}
+	return processLimitsFile(string(out))
+}
+
+func processStatsFromProcs(rootFs string, cgroupPath string, rootPid int) (info.ProcessStats, error) {
 	var fdCount, socketCount uint64
 	filePath := path.Join(cgroupPath, "cgroup.procs")
 	out, err := ioutil.ReadFile(filePath)
@@ -180,11 +252,13 @@ func processStatsFromProcs(rootFs string, cgroupPath string) (info.ProcessStats,
 			}
 		}
 	}
+	ulimits := processRootProcUlimits(rootFs, rootPid)
 
 	processStats := info.ProcessStats{
 		ProcessCount: uint64(len(pids)),
 		FdCount:      fdCount,
 		SocketCount:  socketCount,
+		Ulimits:      ulimits,
 	}
 
 	return processStats, nil

container/libcontainer/handler_test.go

@@ -16,6 +16,7 @@ package libcontainer
 
 import (
 	"os"
+	"reflect"
 	"testing"
 
 	info "github.com/google/cadvisor/info/v1"
@@ -175,3 +176,42 @@ func TestSetProcessesStats(t *testing.T) {
 	}
 
 }
+
+func TestParseLimitsFile(t *testing.T) {
+	var testData = []struct {
+		limitLine string
+		expected  []info.UlimitSpec
+	}{
+		{
+			"Limit                     Soft Limit           Hard Limit           Units   \n",
+			[]info.UlimitSpec{},
+		},
+		{
+			"Max open files            8192                 8192                 files   \n",
+			[]info.UlimitSpec{{Name: "max_open_files", SoftLimit: 8192, HardLimit: 8192}},
+		},
+		{
+			"Max open files            85899345920          85899345920          files   \n",
+			[]info.UlimitSpec{{Name: "max_open_files", SoftLimit: 85899345920, HardLimit: 85899345920}},
+		},
+		{
+			"Max open files            gibberish1           8192                 files   \n",
+			[]info.UlimitSpec{},
+		},
+		{
+			"Max open files            8192                 0xbaddata            files   \n",
+			[]info.UlimitSpec{},
+		},
+		{
+			"Max stack size            8192                 8192                 files   \n",
+			[]info.UlimitSpec{},
+		},
+	}
+
+	for _, testItem := range testData {
+		actual := processLimitsFile(testItem.limitLine)
+		if reflect.DeepEqual(actual, testItem.expected) == false {
+			t.Fatalf("Parsed ulimit doesn't match expected values for line: %s", testItem.limitLine)
+		}
+	}
+}

info/v1/container.go

@@ -564,6 +564,12 @@ type AcceleratorStats struct {
 	DutyCycle uint64 `json:"duty_cycle"`
 }
 
+type UlimitSpec struct {
+	Name      string `json:"name"`
+	SoftLimit int64  `json:"soft_limit"`
+	HardLimit int64  `json:"hard_limit"`
+}
+
 type ProcessStats struct {
 	// Number of processes
 	ProcessCount uint64 `json:"process_count"`
@@ -579,6 +585,9 @@ type ProcessStats struct {
 
 	// Maxium number of threads allowed in container
 	ThreadsMax uint64 `json:"threads_max,omitempty"`
+
+	// Ulimits for the top-level container process
+	Ulimits []UlimitSpec `json:"ulimits,omitempty"`
 }
 
 type ContainerStats struct {

metrics/prometheus.go

@@ -1079,6 +1079,23 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
 					}
 				},
 			},
+			{
+				name:        "container_ulimits_soft",
+				help:        "Soft ulimit values for the container root process. Unlimited if -1, except priority and nice",
+				valueType:   prometheus.GaugeValue,
+				extraLabels: []string{"ulimit"},
+				getValues: func(s *info.ContainerStats) metricValues {
+					values := make(metricValues, 0, len(s.Processes.Ulimits))
+					for _, ulimit := range s.Processes.Ulimits {
+						values = append(values, metricValue{
+							value:     float64(ulimit.SoftLimit),
+							labels:    []string{ulimit.Name},
+							timestamp: s.Timestamp,
+						})
+					}
+					return values
+				},
+			},
 		}...)
 
 	}

metrics/prometheus_test.go

@@ -260,6 +260,13 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container
 						SocketCount:    3,
 						ThreadsCurrent: 5,
 						ThreadsMax:     100,
+						Ulimits: []info.UlimitSpec{
+							{
+								Name:      "max_open_files",
+								SoftLimit: 16384,
+								HardLimit: 16384,
+							},
+						},
 					},
 					TaskStats: info.LoadStats{
 						NrSleeping:        50,

metrics/testdata/prometheus_metrics

@@ -238,6 +238,9 @@ container_threads{container_env_foo_env="prod",container_label_foo_label="bar",i
 # HELP container_threads_max Maximum number of threads allowed inside the container, infinity if value is zero
 # TYPE container_threads_max gauge
 container_threads_max{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 100 1395066363000
+# HELP container_ulimits_soft Soft ulimit values for the container root process. Unlimited if -1, except priority and nice
+# TYPE container_ulimits_soft gauge
+container_ulimits_soft{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",ulimit="max_open_files",zone_name="hello"} 16384 1395066363000
 # HELP machine_cpu_cores Number of CPU cores on the machine.
 # TYPE machine_cpu_cores gauge
 machine_cpu_cores 4