From 4b002b3bd303af7c15b75c147f83b7ec6d2edc59 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 31 Aug 2017 18:05:53 +0200 Subject: [PATCH] *: add CRI-O handler Signed-off-by: Antonio Murdaca --- container/container.go | 1 + container/crio/client.go | 130 +++++++++++++ container/crio/client_test.go | 49 +++++ container/crio/factory.go | 170 +++++++++++++++++ container/crio/factory_test.go | 47 +++++ container/crio/handler.go | 334 +++++++++++++++++++++++++++++++++ container/crio/handler_test.go | 119 ++++++++++++ fs/fs.go | 24 +++ manager/manager.go | 18 ++ 9 files changed, 892 insertions(+) create mode 100644 container/crio/client.go create mode 100644 container/crio/client_test.go create mode 100644 container/crio/factory.go create mode 100644 container/crio/factory_test.go create mode 100644 container/crio/handler.go create mode 100644 container/crio/handler_test.go diff --git a/container/container.go b/container/container.go index 0927d327c6..697626b8eb 100644 --- a/container/container.go +++ b/container/container.go @@ -34,6 +34,7 @@ const ( ContainerTypeDocker ContainerTypeRkt ContainerTypeSystemd + ContainerTypeCrio ) // Interface for container operation handlers. diff --git a/container/crio/client.go b/container/crio/client.go new file mode 100644 index 0000000000..b588552cf4 --- /dev/null +++ b/container/crio/client.go @@ -0,0 +1,130 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crio + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "syscall" + "time" +) + +const ( + CrioSocket = "/var/run/crio.sock" + maxUnixSocketPathSize = len(syscall.RawSockaddrUnix{}.Path) +) + +// Info represents CRI-O information as sent by the CRI-O server +type Info struct { + StorageDriver string `json:"storage_driver"` + StorageRoot string `json:"storage_root"` +} + +// ContainerInfo represents a given container information +type ContainerInfo struct { + Name string `json:"name"` + Pid int `json:"pid"` + Image string `json:"image"` + CreatedTime int64 `json:"created_time"` + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + LogPath string `json:"log_path"` + Root string `json:"root"` + IP string `json:"ip_address"` +} + +type crioClient interface { + Info() (Info, error) + ContainerInfo(string) (*ContainerInfo, error) +} + +type crioClientImpl struct { + client *http.Client +} + +func configureUnixTransport(tr *http.Transport, proto, addr string) error { + if len(addr) > maxUnixSocketPathSize { + return fmt.Errorf("Unix socket path %q is too long", addr) + } + // No need for compression in local communications. + tr.DisableCompression = true + tr.Dial = func(_, _ string) (net.Conn, error) { + return net.DialTimeout(proto, addr, 32*time.Second) + } + return nil +} + +// Client returns a new configured CRI-O client +func Client() (crioClient, error) { + tr := new(http.Transport) + configureUnixTransport(tr, "unix", CrioSocket) + c := &http.Client{ + Transport: tr, + } + return &crioClientImpl{ + client: c, + }, nil +} + +func getRequest(path string) (*http.Request, error) { + req, err := http.NewRequest("GET", path, nil) + if err != nil { + return nil, err + } + // For local communications over a unix socket, it doesn't matter what + // the host is. We just need a valid and meaningful host name. + req.Host = "crio" + req.URL.Host = CrioSocket + req.URL.Scheme = "http" + return req, nil +} + +// Info returns generic info from the CRI-O server +func (c *crioClientImpl) Info() (Info, error) { + info := Info{} + req, err := getRequest("/info") + if err != nil { + return info, err + } + resp, err := c.client.Do(req) + if err != nil { + return info, err + } + defer resp.Body.Close() + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return info, err + } + return info, nil +} + +// ContainerInfo returns information about a given container +func (c *crioClientImpl) ContainerInfo(id string) (*ContainerInfo, error) { + req, err := getRequest("/containers/" + id) + if err != nil { + return nil, err + } + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + cInfo := ContainerInfo{} + if err := json.NewDecoder(resp.Body).Decode(&cInfo); err != nil { + return nil, err + } + return &cInfo, nil +} diff --git a/container/crio/client_test.go b/container/crio/client_test.go new file mode 100644 index 0000000000..8e786454b9 --- /dev/null +++ b/container/crio/client_test.go @@ -0,0 +1,49 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crio + +import "fmt" + +type crioClientMock struct { + info Info + containersInfo map[string]*ContainerInfo + err error +} + +func (c *crioClientMock) Info() (Info, error) { + if c.err != nil { + return Info{}, c.err + } + return c.info, nil +} + +func (c *crioClientMock) ContainerInfo(id string) (*ContainerInfo, error) { + if c.err != nil { + return nil, c.err + } + cInfo, ok := c.containersInfo[id] + if !ok { + return nil, fmt.Errorf("no container with id %s", id) + } + return cInfo, nil +} + +func mockCrioClient(info Info, containersInfo map[string]*ContainerInfo, err error) crioClient { + return &crioClientMock{ + err: err, + info: info, + containersInfo: containersInfo, + } +} diff --git a/container/crio/factory.go b/container/crio/factory.go new file mode 100644 index 0000000000..b0151c7f12 --- /dev/null +++ b/container/crio/factory.go @@ -0,0 +1,170 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crio + +import ( + "fmt" + "path" + "regexp" + "strings" + + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/container/libcontainer" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/manager/watcher" + + "github.com/golang/glog" +) + +// The namespace under which crio aliases are unique. +const CrioNamespace = "crio" + +// Regexp that identifies CRI-O cgroups +var crioCgroupRegexp = regexp.MustCompile(`([a-z0-9]{64})`) + +type storageDriver string + +const ( + // TODO add full set of supported drivers in future.. + overlayStorageDriver storageDriver = "overlay" + overlay2StorageDriver storageDriver = "overlay2" +) + +type crioFactory struct { + machineInfoFactory info.MachineInfoFactory + + storageDriver storageDriver + storageDir string + + // Information about the mounted cgroup subsystems. + cgroupSubsystems libcontainer.CgroupSubsystems + + // Information about mounted filesystems. + fsInfo fs.FsInfo + + ignoreMetrics container.MetricSet + + client crioClient +} + +func (self *crioFactory) String() string { + return CrioNamespace +} + +func (self *crioFactory) NewContainerHandler(name string, inHostNamespace bool) (handler container.ContainerHandler, err error) { + client, err := Client() + if err != nil { + return + } + // TODO are there any env vars we need to white list, if so, do it here... + metadataEnvs := []string{} + handler, err = newCrioContainerHandler( + client, + name, + self.machineInfoFactory, + self.fsInfo, + self.storageDriver, + self.storageDir, + &self.cgroupSubsystems, + inHostNamespace, + metadataEnvs, + self.ignoreMetrics, + ) + return +} + +// Returns the CRIO ID from the full container name. +func ContainerNameToCrioId(name string) string { + id := path.Base(name) + + if matches := crioCgroupRegexp.FindStringSubmatch(id); matches != nil { + return matches[1] + } + + return id +} + +// isContainerName returns true if the cgroup with associated name +// corresponds to a crio container. +func isContainerName(name string) bool { + // always ignore .mount cgroup even if associated with crio and delegate to systemd + if strings.HasSuffix(name, ".mount") { + return false + } + return crioCgroupRegexp.MatchString(path.Base(name)) +} + +// crio handles all containers under /crio +func (self *crioFactory) CanHandleAndAccept(name string) (bool, bool, error) { + if strings.HasPrefix(path.Base(name), "crio-conmon") { + // TODO(runcom): should we include crio-conmon cgroups? + return false, false, nil + } + if !strings.HasPrefix(path.Base(name), CrioNamespace) { + return false, false, nil + } + // if the container is not associated with CRI-O, we can't handle it or accept it. + if !isContainerName(name) { + return false, false, nil + } + return true, true, nil +} + +func (self *crioFactory) DebugInfo() map[string][]string { + return map[string][]string{} +} + +var ( + // TODO(runcom): handle versioning in CRI-O + version_regexp_string = `(\d+)\.(\d+)\.(\d+)` + version_re = regexp.MustCompile(version_regexp_string) + apiversion_regexp_string = `(\d+)\.(\d+)` + apiversion_re = regexp.MustCompile(apiversion_regexp_string) +) + +// Register root container before running this function! +func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, ignoreMetrics container.MetricSet) error { + client, err := Client() + if err != nil { + return err + } + + info, err := client.Info() + if err != nil { + return err + } + + // TODO determine crio version so we can work differently w/ future versions if needed + + cgroupSubsystems, err := libcontainer.GetCgroupSubsystems() + if err != nil { + return fmt.Errorf("failed to get cgroup subsystems: %v", err) + } + + glog.Infof("Registering CRI-O factory") + f := &crioFactory{ + client: client, + cgroupSubsystems: cgroupSubsystems, + fsInfo: fsInfo, + machineInfoFactory: factory, + storageDriver: storageDriver(info.StorageDriver), + storageDir: info.StorageRoot, + ignoreMetrics: ignoreMetrics, + } + + container.RegisterContainerHandlerFactory(f, []watcher.ContainerWatchSource{watcher.Raw}) + return nil +} diff --git a/container/crio/factory_test.go b/container/crio/factory_test.go new file mode 100644 index 0000000000..d5d4f1f49c --- /dev/null +++ b/container/crio/factory_test.go @@ -0,0 +1,47 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crio + +import ( + "testing" + + containerlibcontainer "github.com/google/cadvisor/container/libcontainer" + "github.com/stretchr/testify/assert" +) + +func TestCanHandleAndAccept(t *testing.T) { + as := assert.New(t) + f := &crioFactory{ + client: nil, + cgroupSubsystems: containerlibcontainer.CgroupSubsystems{}, + fsInfo: nil, + machineInfoFactory: nil, + storageDriver: "", + storageDir: "", + ignoreMetrics: nil, + } + for k, v := range map[string]bool{ + "/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/crio-81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f": true, + "/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/crio-81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f.mount": false, + "/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/crio-conmon-81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f": false, + "/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/no-crio-conmon-81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f": false, + "/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/crio-990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75": false, + } { + b1, b2, err := f.CanHandleAndAccept(k) + as.Nil(err) + as.Equal(b1, v) + as.Equal(b2, v) + } +} diff --git a/container/crio/handler.go b/container/crio/handler.go new file mode 100644 index 0000000000..6e6fa58563 --- /dev/null +++ b/container/crio/handler.go @@ -0,0 +1,334 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Handler for CRI-O containers. +package crio + +import ( + "fmt" + "path" + "strconv" + "strings" + "time" + + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/container/common" + containerlibcontainer "github.com/google/cadvisor/container/libcontainer" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + + "github.com/opencontainers/runc/libcontainer/cgroups" + cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs" + libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs" +) + +type crioContainerHandler struct { + name string + id string + aliases []string + machineInfoFactory info.MachineInfoFactory + + // Absolute path to the cgroup hierarchies of this container. + // (e.g.: "cpu" -> "/sys/fs/cgroup/cpu/test") + cgroupPaths map[string]string + + // Manager of this container's cgroups. + cgroupManager cgroups.Manager + + // the CRI-O storage driver + storageDriver storageDriver + fsInfo fs.FsInfo + rootfsStorageDir string + + // Time at which this container was created. + creationTime time.Time + + // Metadata associated with the container. + labels map[string]string + envs map[string]string + + // TODO + // crio version handling... + + // The container PID used to switch namespaces as required + pid int + + // Image name used for this container. + image string + + // The host root FS to read + rootFs string + + // The network mode of the container + // TODO + + // Filesystem handler. + fsHandler common.FsHandler + + // The IP address of the container + ipAddress string + + ignoreMetrics container.MetricSet + + // container restart count + restartCount int +} + +var _ container.ContainerHandler = &crioContainerHandler{} + +// newCrioContainerHandler returns a new container.ContainerHandler +func newCrioContainerHandler( + client crioClient, + name string, + machineInfoFactory info.MachineInfoFactory, + fsInfo fs.FsInfo, + storageDriver storageDriver, + storageDir string, + cgroupSubsystems *containerlibcontainer.CgroupSubsystems, + inHostNamespace bool, + metadataEnvs []string, + ignoreMetrics container.MetricSet, +) (container.ContainerHandler, error) { + // Create the cgroup paths. + cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) + for key, val := range cgroupSubsystems.MountPoints { + cgroupPaths[key] = path.Join(val, name) + } + + // Generate the equivalent cgroup manager for this container. + cgroupManager := &cgroupfs.Manager{ + Cgroups: &libcontainerconfigs.Cgroup{ + Name: name, + }, + Paths: cgroupPaths, + } + + rootFs := "/" + if !inHostNamespace { + rootFs = "/rootfs" + storageDir = path.Join(rootFs, storageDir) + } + + id := ContainerNameToCrioId(name) + + cInfo, err := client.ContainerInfo(id) + if err != nil { + return nil, err + } + + // passed to fs handler below ... + // XXX: this is using the full container logpath, as constructed by the CRI + // /var/log/pods//container_instance.log + // It's not actually a log dir, as the CRI doesn't have per-container dirs + // under /var/log/pods// + // We can't use /var/log/pods// to count per-container log usage. + // We use the container log file directly. + storageLogDir := cInfo.LogPath + + // Determine the rootfs storage dir + rootfsStorageDir := cInfo.Root + // TODO(runcom): CRI-O doesn't strip /merged but we need to in order to + // get device ID from root, otherwise, it's going to error out as overlay + // mounts doesn't have fixed dev ids. + rootfsStorageDir = strings.TrimSuffix(rootfsStorageDir, "/merged") + + // TODO: extract object mother method + handler := &crioContainerHandler{ + id: id, + name: name, + machineInfoFactory: machineInfoFactory, + cgroupPaths: cgroupPaths, + cgroupManager: cgroupManager, + storageDriver: storageDriver, + fsInfo: fsInfo, + rootFs: rootFs, + rootfsStorageDir: rootfsStorageDir, + envs: make(map[string]string), + ignoreMetrics: ignoreMetrics, + } + + handler.creationTime = time.Unix(0, cInfo.CreatedTime) + handler.pid = cInfo.Pid + handler.aliases = append(handler.aliases, cInfo.Name, id) + handler.labels = cInfo.Labels + handler.image = cInfo.Image + // TODO: we wantd to know graph driver DeviceId (dont think this is needed now) + + // ignore err and get zero as default, this happens with sandboxes, not sure why... + // kube isn't sending restart count in labels for sandboxes. + restartCount, _ := strconv.Atoi(cInfo.Annotations["io.kubernetes.container.restartCount"]) + handler.restartCount = restartCount + + handler.ipAddress = cInfo.IP + + // we optionally collect disk usage metrics + if !ignoreMetrics.Has(container.DiskUsageMetrics) { + handler.fsHandler = common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, storageLogDir, fsInfo) + } + // TODO for env vars we wanted to show from container.Config.Env from whitelist + //for _, exposedEnv := range metadataEnvs { + //glog.Infof("TODO env whitelist: %v", exposedEnv) + //} + + return handler, nil +} + +func (self *crioContainerHandler) Start() { + if self.fsHandler != nil { + self.fsHandler.Start() + } +} + +func (self *crioContainerHandler) Cleanup() { + if self.fsHandler != nil { + self.fsHandler.Stop() + } +} + +func (self *crioContainerHandler) ContainerReference() (info.ContainerReference, error) { + return info.ContainerReference{ + Id: self.id, + Name: self.name, + Aliases: self.aliases, + Namespace: CrioNamespace, + Labels: self.labels, + }, nil +} + +func (self *crioContainerHandler) needNet() bool { + if !self.ignoreMetrics.Has(container.NetworkUsageMetrics) { + return self.labels["io.kubernetes.container.name"] == "POD" + } + return false +} + +func (self *crioContainerHandler) GetSpec() (info.ContainerSpec, error) { + hasFilesystem := !self.ignoreMetrics.Has(container.DiskUsageMetrics) + spec, err := common.GetSpec(self.cgroupPaths, self.machineInfoFactory, self.needNet(), hasFilesystem) + + spec.Labels = self.labels + // Only adds restartcount label if it's greater than 0 + if self.restartCount > 0 { + spec.Labels["restartcount"] = strconv.Itoa(self.restartCount) + } + spec.Envs = self.envs + spec.Image = self.image + + return spec, err +} + +func (self *crioContainerHandler) getFsStats(stats *info.ContainerStats) error { + mi, err := self.machineInfoFactory.GetMachineInfo() + if err != nil { + return err + } + + if !self.ignoreMetrics.Has(container.DiskIOMetrics) { + common.AssignDeviceNamesToDiskStats((*common.MachineInfoNamer)(mi), &stats.DiskIo) + } + + if self.ignoreMetrics.Has(container.DiskUsageMetrics) { + return nil + } + var device string + switch self.storageDriver { + case overlay2StorageDriver, overlayStorageDriver: + deviceInfo, err := self.fsInfo.GetDirFsDevice(self.rootfsStorageDir) + if err != nil { + return fmt.Errorf("unable to determine device info for dir: %v: %v", self.rootfsStorageDir, err) + } + device = deviceInfo.Device + default: + return nil + } + + var ( + limit uint64 + fsType string + ) + + // crio does not impose any filesystem limits for containers. So use capacity as limit. + for _, fs := range mi.Filesystems { + if fs.Device == device { + limit = fs.Capacity + fsType = fs.Type + break + } + } + + fsStat := info.FsStats{Device: device, Type: fsType, Limit: limit} + usage := self.fsHandler.Usage() + fsStat.BaseUsage = usage.BaseUsageBytes + fsStat.Usage = usage.TotalUsageBytes + fsStat.Inodes = usage.InodeUsage + + stats.Filesystem = append(stats.Filesystem, fsStat) + + return nil +} + +func (self *crioContainerHandler) GetStats() (*info.ContainerStats, error) { + stats, err := containerlibcontainer.GetStats(self.cgroupManager, self.rootFs, self.pid, self.ignoreMetrics) + if err != nil { + return stats, err + } + // Clean up stats for containers that don't have their own network - this + // includes containers running in Kubernetes pods that use the network of the + // infrastructure container. This stops metrics being reported multiple times + // for each container in a pod. + if !self.needNet() { + stats.Network = info.NetworkStats{} + } + + // Get filesystem stats. + err = self.getFsStats(stats) + if err != nil { + return stats, err + } + + return stats, nil +} + +func (self *crioContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) { + // No-op for Docker driver. + return []info.ContainerReference{}, nil +} + +func (self *crioContainerHandler) GetCgroupPath(resource string) (string, error) { + path, ok := self.cgroupPaths[resource] + if !ok { + return "", fmt.Errorf("could not find path for resource %q for container %q\n", resource, self.name) + } + return path, nil +} + +func (self *crioContainerHandler) GetContainerLabels() map[string]string { + return self.labels +} + +func (self *crioContainerHandler) GetContainerIPAddress() string { + return self.ipAddress +} + +func (self *crioContainerHandler) ListProcesses(listType container.ListType) ([]int, error) { + return containerlibcontainer.GetProcesses(self.cgroupManager) +} + +func (self *crioContainerHandler) Exists() bool { + return common.CgroupExists(self.cgroupPaths) +} + +func (self *crioContainerHandler) Type() container.ContainerType { + return container.ContainerTypeCrio +} diff --git a/container/crio/handler_test.go b/container/crio/handler_test.go new file mode 100644 index 0000000000..ed39b56569 --- /dev/null +++ b/container/crio/handler_test.go @@ -0,0 +1,119 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crio + +import ( + "fmt" + "testing" + + "github.com/google/cadvisor/container" + containerlibcontainer "github.com/google/cadvisor/container/libcontainer" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/stretchr/testify/assert" +) + +func TestHandler(t *testing.T) { + as := assert.New(t) + type testCase struct { + client crioClient + name string + machineInfoFactory info.MachineInfoFactory + fsInfo fs.FsInfo + storageDriver storageDriver + storageDir string + cgroupSubsystems *containerlibcontainer.CgroupSubsystems + inHostNamespace bool + metadataEnvs []string + ignoreMetrics container.MetricSet + + hasErr bool + errContains string + checkReference *info.ContainerReference + } + for _, ts := range []testCase{ + { + mockCrioClient(Info{}, nil, fmt.Errorf("no client returned")), + "/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/crio-81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f", + nil, + nil, + "", + "", + &containerlibcontainer.CgroupSubsystems{}, + false, + nil, + nil, + + true, + "no client returned", + nil, + }, + { + mockCrioClient(Info{}, nil, nil), + "/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/crio-81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f", + nil, + nil, + "", + "", + &containerlibcontainer.CgroupSubsystems{}, + false, + nil, + nil, + + true, + "no container with id 81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f", + nil, + }, + { + mockCrioClient( + Info{}, + map[string]*ContainerInfo{"81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f": {Name: "test", Labels: map[string]string{"io.kubernetes.container.name": "POD"}}}, + nil, + ), + "/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/crio-81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f", + nil, + nil, + "", + "", + &containerlibcontainer.CgroupSubsystems{}, + false, + nil, + nil, + + false, + "", + &info.ContainerReference{ + Id: "81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f", + Name: "/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/crio-81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f", + Aliases: []string{"test", "81e5c2990803c383229c9680ce964738d5e566d97f5bd436ac34808d2ec75d5f"}, + Namespace: CrioNamespace, + Labels: map[string]string{"io.kubernetes.container.name": "POD"}, + }, + }, + } { + handler, err := newCrioContainerHandler(ts.client, ts.name, ts.machineInfoFactory, ts.fsInfo, ts.storageDriver, ts.storageDir, ts.cgroupSubsystems, ts.inHostNamespace, ts.metadataEnvs, ts.ignoreMetrics) + if ts.hasErr { + as.NotNil(err) + if ts.errContains != "" { + as.Contains(err.Error(), ts.errContains) + } + } + if ts.checkReference != nil { + cr, err := handler.ContainerReference() + as.Nil(err) + as.Equal(*ts.checkReference, cr) + } + } +} diff --git a/fs/fs.go b/fs/fs.go index 361a0c2557..97afba8ece 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -43,6 +43,7 @@ const ( LabelSystemRoot = "root" LabelDockerImages = "docker-images" LabelRktImages = "rkt-images" + LabelCrioImages = "crio-images" ) // The maximum number of `du` and `find` tasks that can be running at once. @@ -91,6 +92,7 @@ type Context struct { // docker root directory. Docker DockerContext RktPath string + Crio CrioContext } type DockerContext struct { @@ -99,6 +101,10 @@ type DockerContext struct { DriverStatus map[string]string } +type CrioContext struct { + Root string +} + func NewFsInfo(context Context) (FsInfo, error) { mounts, err := mount.GetMounts() if err != nil { @@ -128,6 +134,7 @@ func NewFsInfo(context Context) (FsInfo, error) { // need to call this before the log line below printing out the partitions, as this function may // add a "partition" for devicemapper to fsInfo.partitions fsInfo.addDockerImagesLabel(context, mounts) + fsInfo.addCrioImagesLabel(context, mounts) glog.Infof("Filesystem UUIDs: %+v", fsInfo.fsUUIDToDeviceName) glog.Infof("Filesystem partitions: %+v", fsInfo.partitions) @@ -278,6 +285,23 @@ func (self *RealFsInfo) addDockerImagesLabel(context Context, mounts []*mount.In } } +func (self *RealFsInfo) addCrioImagesLabel(context Context, mounts []*mount.Info) { + if context.Crio.Root != "" { + crioPath := context.Crio.Root + crioImagePaths := map[string]struct{}{ + "/": {}, + } + for _, dir := range []string{"overlay", "overlay2"} { + crioImagePaths[path.Join(crioPath, dir+"-images")] = struct{}{} + } + for crioPath != "/" && crioPath != "." { + crioImagePaths[crioPath] = struct{}{} + crioPath = filepath.Dir(crioPath) + } + self.updateContainerImagesPath(LabelCrioImages, mounts, crioImagePaths) + } +} + func (self *RealFsInfo) addRktImagesLabel(context Context, mounts []*mount.Info) { if context.RktPath != "" { rktPath := context.RktPath diff --git a/manager/manager.go b/manager/manager.go index 8086c058a2..cafac0828a 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -28,6 +28,7 @@ import ( "github.com/google/cadvisor/cache/memory" "github.com/google/cadvisor/collector" "github.com/google/cadvisor/container" + "github.com/google/cadvisor/container/crio" "github.com/google/cadvisor/container/docker" "github.com/google/cadvisor/container/raw" "github.com/google/cadvisor/container/rkt" @@ -156,6 +157,15 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn glog.Warningf("unable to connect to Rkt api service: %v", err) } + crioClient, err := crio.Client() + if err != nil { + return nil, err + } + crioInfo, err := crioClient.Info() + if err != nil { + glog.Warningf("unable to connect to CRI-O api service: %v", err) + } + context := fs.Context{ Docker: fs.DockerContext{ Root: docker.RootDir(), @@ -163,6 +173,9 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn DriverStatus: dockerStatus.DriverStatus, }, RktPath: rktPath, + Crio: fs.CrioContext{ + Root: crioInfo.StorageRoot, + }, } fsInfo, err := fs.NewFsInfo(context) if err != nil { @@ -258,6 +271,11 @@ func (self *manager) Start() error { self.containerWatchers = append(self.containerWatchers, watcher) } + err = crio.Register(self, self.fsInfo, self.ignoreMetrics) + if err != nil { + glog.Warningf("Registration of the crio container factory failed: %v", err) + } + err = systemd.Register(self, self.fsInfo, self.ignoreMetrics) if err != nil { glog.Warningf("Registration of the systemd container factory failed: %v", err)