sdn: garbage-collect dead containers to recover IPAM leases

Port of kubernetes/kubernetes#35572 to openshift-sdn.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1395183

Author: dcbw

pkg/sdn/plugin/pod_linux.go

@@ -4,7 +4,10 @@ package plugin
 
 import (
 	"fmt"
+	"io/ioutil"
+	"net"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
@@ -20,6 +23,7 @@ import (
 	knetwork "k8s.io/kubernetes/pkg/kubelet/network"
 	kubehostport "k8s.io/kubernetes/pkg/kubelet/network/hostport"
 	kbandwidth "k8s.io/kubernetes/pkg/util/bandwidth"
+	ksets "k8s.io/kubernetes/pkg/util/sets"
 
 	"github.com/containernetworking/cni/pkg/invoke"
 	"github.com/containernetworking/cni/pkg/ip"
@@ -270,6 +274,93 @@ func vnidToString(vnid uint32) string {
 	return strconv.FormatUint(uint64(vnid), 10)
 }
 
+// podIsExited returns true if the pod is exited (all containers inside are exited).
+func podIsExited(p *kcontainer.Pod) bool {
+	for _, c := range p.Containers {
+		if c.State != kcontainer.ContainerStateExited {
+			return false
+		}
+	}
+	return true
+}
+
+// getNonExitedPods returns a list of pods that have at least one running container.
+func (m *podManager) getNonExitedPods() ([]*kcontainer.Pod, error) {
+	ret := []*kcontainer.Pod{}
+	pods, err := m.host.GetRuntime().GetPods(true)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to retrieve pods from runtime: %v", err)
+	}
+	for _, p := range pods {
+		if podIsExited(p) {
+			continue
+		}
+		ret = append(ret, p)
+	}
+	return ret, nil
+}
+
+// ipamGarbageCollection will release unused IPs from dead containers that
+// the CNI plugin was never notified had died.  openshift-sdn uses the CNI
+// host-local IPAM plugin, which stores allocated IPs in a file in
+// /var/lib/cni/network. Each file in this directory has as its name the
+// allocated IP address of the container, and as its contents the container ID.
+// This routine looks for container IDs that are not reported as running by the
+// container runtime, and releases each one's IPAM allocation.
+func (m *podManager) ipamGarbageCollection() {
+	glog.V(2).Infof("Starting IP garbage collection")
+
+	const ipamDir string = "/var/lib/cni/networks/openshift-sdn"
+	files, err := ioutil.ReadDir(ipamDir)
+	if err != nil {
+		glog.Errorf("Failed to list files in CNI host-local IPAM store %v: %v", ipamDir, err)
+		return
+	}
+
+	// gather containerIDs for allocated ips
+	ipContainerIdMap := make(map[string]string)
+	for _, file := range files {
+		// skip non checkpoint file
+		if ip := net.ParseIP(file.Name()); ip == nil {
+			continue
+		}
+
+		content, err := ioutil.ReadFile(filepath.Join(ipamDir, file.Name()))
+		if err != nil {
+			glog.Errorf("Failed to read file %v: %v", file, err)
+		}
+		ipContainerIdMap[file.Name()] = strings.TrimSpace(string(content))
+	}
+
+	// gather infra container IDs of current running Pods
+	runningContainerIDs := ksets.String{}
+	pods, err := m.getNonExitedPods()
+	if err != nil {
+		glog.Errorf("Failed to get pods: %v", err)
+		return
+	}
+	for _, pod := range pods {
+		containerID, err := m.host.GetRuntime().GetPodContainerID(pod)
+		if err != nil {
+			glog.Warningf("Failed to get infra containerID of %q/%q: %v", pod.Namespace, pod.Name, err)
+			continue
+		}
+
+		runningContainerIDs.Insert(strings.TrimSpace(containerID.ID))
+	}
+
+	// release leaked ips
+	for ip, containerID := range ipContainerIdMap {
+		// if the container is not running, release IP
+		if runningContainerIDs.Has(containerID) {
+			continue
+		}
+
+		glog.V(2).Infof("Releasing IP %q allocated to %q.", ip, containerID)
+		m.ipamDel(containerID)
+	}
+}
+
 // Set up all networking (host/container veth, OVS flows, IPAM, loopback, etc)
 func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *kubehostport.RunningPod, error) {
 	podConfig, pod, err := m.getPodConfig(req)
@@ -279,6 +370,15 @@ func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *kubeho
 
 	ipamResult, err := m.ipamAdd(req.Netns, req.ContainerId)
 	if err != nil {
+		// TODO: Remove this hack once we've figured out how to retrieve the netns
+		// of an exited container. Currently, restarting docker will leak a bunch of
+		// ips. This will exhaust available ip space unless we cleanup old ips. At the
+		// same time we don't want to try GC'ing them periodically as that could lead
+		// to a performance regression in starting pods. So on each setup failure, try
+		// GC on the assumption that the kubelet is going to retry pod creation, and
+		// when it does, there will be ips.
+		m.ipamGarbageCollection()
+
 		return nil, nil, fmt.Errorf("failed to run IPAM for %v: %v", req.ContainerId, err)
 	}
 	podIP := ipamResult.IP4.IP.IP