Change the userspace proxy to wait for the node record

The proxy setup races with the node registration.  The iptables proxy setup tries to read the node record to retrieve the address for the node.  If the registration has not happened yet the read fails, but the code proceeds despite the failure.  This only shows up in the testing when the unidler is used because the ip address doesn't get used until the iptables rules are set to send the traffic to the userspace proxy, and because there is a nil ip address, the rules are rejected by iptables.

The fix is to add a retry loop (with a backoff) to keep reading until the node record is present.

The bug is masked when a network plugin is used because it typically needs the node record to be present and it runs first, so by the time the proxy is set up, there is a node record.

Fixes bug 1519991 (https://bugzilla.redhat.com/show_bug.cgi?id=1519991)

Author: knobunc

pkg/cmd/server/kubernetes/network/network.go

@@ -10,6 +10,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"k8s.io/api/core/v1"
+	kapierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
@@ -92,7 +93,11 @@ func (c *NetworkConfig) RunProxy() {
 	case componentconfig.ProxyModeIPTables:
 		glog.V(0).Info("Using iptables Proxier.")
 		if bindAddr.Equal(net.IPv4zero) {
-			bindAddr = getNodeIP(c.ExternalKubeClientset.CoreV1(), hostname)
+			var err error
+			bindAddr, err = getNodeIP(c.ExternalKubeClientset.CoreV1(), hostname)
+			if err != nil {
+				glog.Fatalf("Unable to get a bind address: %v", err)
+			}
 		}
 		if c.ProxyConfig.IPTables.MasqueradeBit == nil {
 			// IPTablesMasqueradeBit must be specified or defaulted.
@@ -232,17 +237,37 @@ func (c *NetworkConfig) RunProxy() {
 }
 
 // getNodeIP is copied from the upstream proxy config to retrieve the IP of a node.
-func getNodeIP(client kv1core.CoreV1Interface, hostname string) net.IP {
-	var nodeIP net.IP
-	node, err := client.Nodes().Get(hostname, metav1.GetOptions{})
-	if err != nil {
-		glog.Warningf("Failed to retrieve node info: %v", err)
-		return nil
+func getNodeIP(client kv1core.CoreV1Interface, hostname string) (net.IP, error) {
+	var node *v1.Node
+	var nodeErr error
+
+	// We may beat the thread that causes the node object to be created,
+	// so if we can't get it, then we need to wait.
+	// This will wait 0, 2, 4, 8, ... 64 seconds, for a total of ~2 mins
+	nodeWaitBackoff := utilwait.Backoff{
+		Duration: 2 * time.Second,
+		Factor:   2,
+		Steps:    7,
 	}
-	nodeIP, err = utilnode.GetNodeHostIP(node)
+	utilwait.ExponentialBackoff(nodeWaitBackoff, func() (bool, error) {
+		node, nodeErr = client.Nodes().Get(hostname, metav1.GetOptions{})
+		if nodeErr == nil {
+			return true, nil
+		} else if kapierrors.IsNotFound(nodeErr) {
+			glog.Warningf("waiting for node %q to be registered with master...", hostname)
+			return false, nil
+		} else {
+			return false, nodeErr
+		}
+	})
+	if nodeErr != nil {
+		return nil, fmt.Errorf("failed to retrieve node info (after waiting): %v", nodeErr)
+	}
+
+	nodeIP, err := utilnode.GetNodeHostIP(node)
 	if err != nil {
-		glog.Warningf("Failed to retrieve node IP: %v", err)
-		return nil
+		return nil, fmt.Errorf("failed to retrieve node IP: %v", err)
 	}
-	return nodeIP
+
+	return nodeIP, nil
 }