Merge pull request #17564 from knobunc/fix/proxy-ip-node-race-idler

Automatic merge from submit-queue (batch tested with PRs 17734, 17550, 17647, 17761, 17564).

Change the userspace proxy to wait for the node record

The userspaceproxy setup races with the node registration.  The userspace proxy setup tries to read the node record to retrieve the address for the node.  If the registration has not happened yet the read fails, but the code proceeds despite the failure.

The fix is to add a retry loop (with a backoff) to keep reading until the node record is present.

The bug is masked when a network plugin is used because it typically needs the node record to be present and it runs first, so by the time the proxy is set up, there is a node record.

Fixes bug 1519991 (https://bugzilla.redhat.com/show_bug.cgi?id=1519991)

@openshift/networking PTAL

Author: openshift-merge-robot

pkg/cmd/server/kubernetes/network/network.go

@@ -10,6 +10,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"k8s.io/api/core/v1"
+	kapierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
@@ -93,7 +94,11 @@ func (c *NetworkConfig) RunProxy() {
 	case kubeproxyconfig.ProxyModeIPTables:
 		glog.V(0).Info("Using iptables Proxier.")
 		if bindAddr.Equal(net.IPv4zero) {
-			bindAddr = getNodeIP(c.ExternalKubeClientset.CoreV1(), hostname)
+			var err error
+			bindAddr, err = getNodeIP(c.ExternalKubeClientset.CoreV1(), hostname)
+			if err != nil {
+				glog.Fatalf("Unable to get a bind address: %v", err)
+			}
 		}
 		if c.ProxyConfig.IPTables.MasqueradeBit == nil {
 			// IPTablesMasqueradeBit must be specified or defaulted.
@@ -233,17 +238,37 @@ func (c *NetworkConfig) RunProxy() {
 }
 
 // getNodeIP is copied from the upstream proxy config to retrieve the IP of a node.
-func getNodeIP(client kv1core.CoreV1Interface, hostname string) net.IP {
-	var nodeIP net.IP
-	node, err := client.Nodes().Get(hostname, metav1.GetOptions{})
-	if err != nil {
-		glog.Warningf("Failed to retrieve node info: %v", err)
-		return nil
+func getNodeIP(client kv1core.CoreV1Interface, hostname string) (net.IP, error) {
+	var node *v1.Node
+	var nodeErr error
+
+	// We may beat the thread that causes the node object to be created,
+	// so if we can't get it, then we need to wait.
+	// This will wait 0, 2, 4, 8, ... 64 seconds, for a total of ~2 mins
+	nodeWaitBackoff := utilwait.Backoff{
+		Duration: 2 * time.Second,
+		Factor:   2,
+		Steps:    7,
 	}
-	nodeIP, err = utilnode.GetNodeHostIP(node)
+	utilwait.ExponentialBackoff(nodeWaitBackoff, func() (bool, error) {
+		node, nodeErr = client.Nodes().Get(hostname, metav1.GetOptions{})
+		if nodeErr == nil {
+			return true, nil
+		} else if kapierrors.IsNotFound(nodeErr) {
+			glog.Warningf("waiting for node %q to be registered with master...", hostname)
+			return false, nil
+		} else {
+			return false, nodeErr
+		}
+	})
+	if nodeErr != nil {
+		return nil, fmt.Errorf("failed to retrieve node info (after waiting): %v", nodeErr)
+	}
+
+	nodeIP, err := utilnode.GetNodeHostIP(node)
 	if err != nil {
-		glog.Warningf("Failed to retrieve node IP: %v", err)
-		return nil
+		return nil, fmt.Errorf("failed to retrieve node IP: %v", err)
 	}
-	return nodeIP
+
+	return nodeIP, nil
 }