Skip to content

Commit d6a7486

Browse files
smarterclaytonsmarterclayton
committed
Ensure openshift start network can run in a pod
Need to be able to take node-config from bootstrap node. For openshift start network the --kubeconfig flag from the CLI overrides the value of masterKubeConfig in the provided node config. If the value is empty (like it is by default) the in-cluster-config is used. Reorganize the node startup slightly so there is even less overlap between kubelet and network. A future change will completely separate these two initialization paths.
1 parent ce5aace commit d6a7486

File tree

9 files changed

+166
-97
lines changed

9 files changed

+166
-97
lines changed

‎contrib/completions/bash/openshift

+2
Original file line numberDiff line numberDiff line change
@@ -33708,6 +33708,8 @@ _openshift_start_network()
3370833708
local_nonpersistent_flags+=("--kubernetes=")
3370933709
flags+=("--latest-images")
3371033710
local_nonpersistent_flags+=("--latest-images")
33711+
flags+=("--listen=")
33712+
local_nonpersistent_flags+=("--listen=")
3371133713
flags+=("--network-plugin=")
3371233714
local_nonpersistent_flags+=("--network-plugin=")
3371333715
flags+=("--recursive-resolv-conf=")

‎contrib/completions/zsh/openshift

+2
Original file line numberDiff line numberDiff line change
@@ -33857,6 +33857,8 @@ _openshift_start_network()
3385733857
local_nonpersistent_flags+=("--kubernetes=")
3385833858
flags+=("--latest-images")
3385933859
local_nonpersistent_flags+=("--latest-images")
33860+
flags+=("--listen=")
33861+
local_nonpersistent_flags+=("--listen=")
3386033862
flags+=("--network-plugin=")
3386133863
local_nonpersistent_flags+=("--network-plugin=")
3386233864
flags+=("--recursive-resolv-conf=")

‎pkg/cmd/server/api/helpers.go

+21
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,27 @@ func SetProtobufClientDefaults(overrides *ClientConnectionOverrides) {
334334
overrides.Burst *= 2
335335
}
336336

337+
// GetKubeConfigOrInClusterConfig loads in-cluster config if kubeConfigFile is empty or the file if not,
338+
// then applies overrides.
339+
func GetKubeConfigOrInClusterConfig(kubeConfigFile string, overrides *ClientConnectionOverrides) (*restclient.Config, error) {
340+
var kubeConfig *restclient.Config
341+
var err error
342+
if len(kubeConfigFile) == 0 {
343+
kubeConfig, err = restclient.InClusterConfig()
344+
} else {
345+
loadingRules := &clientcmd.ClientConfigLoadingRules{}
346+
loadingRules.ExplicitPath = kubeConfigFile
347+
loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
348+
349+
kubeConfig, err = loader.ClientConfig()
350+
}
351+
if err != nil {
352+
return nil, err
353+
}
354+
applyClientConnectionOverrides(overrides, kubeConfig)
355+
return kubeConfig, nil
356+
}
357+
337358
// TODO: clients should be copied and instantiated from a common client config, tweaked, then
338359
// given to individual controllers and other infrastructure components.
339360
func GetInternalKubeClient(kubeConfigFile string, overrides *ClientConnectionOverrides) (kclientsetinternal.Interface, *restclient.Config, error) {

‎pkg/cmd/server/api/validation/validation.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ func ValidateServingInfo(info api.ServingInfo, certificatesRequired bool, fldPat
128128
validationResults.AddErrors(ValidateFile(info.ClientCA, fldPath.Child("clientCA"))...)
129129
}
130130
} else {
131-
if len(info.ClientCA) > 0 {
131+
if certificatesRequired && len(info.ClientCA) > 0 {
132132
validationResults.AddErrors(field.Invalid(fldPath.Child("clientCA"), info.ClientCA, "cannot specify a clientCA without a certFile"))
133133
}
134134
}

‎pkg/cmd/server/kubernetes/network/network_config.go

+7-3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
kclientset "k8s.io/client-go/kubernetes"
1515
"k8s.io/kubernetes/pkg/apis/componentconfig"
1616
kclientsetexternal "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
17+
kclientsetinternal "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
1718
kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
1819

1920
configapi "github.com/openshift/origin/pkg/cmd/server/api"
@@ -49,11 +50,15 @@ type NetworkConfig struct {
4950

5051
// New creates a new network config object for running the networking components of the OpenShift node.
5152
func New(options configapi.NodeConfig, clusterDomain string, proxyConfig *componentconfig.KubeProxyConfiguration, enableProxy, enableDNS bool) (*NetworkConfig, error) {
52-
internalKubeClient, kubeConfig, err := configapi.GetInternalKubeClient(options.MasterKubeConfig, options.MasterClientConnectionOverrides)
53+
kubeConfig, err := configapi.GetKubeConfigOrInClusterConfig(options.MasterKubeConfig, options.MasterClientConnectionOverrides)
5354
if err != nil {
5455
return nil, err
5556
}
56-
externalKubeClient, _, err := configapi.GetExternalKubeClient(options.MasterKubeConfig, options.MasterClientConnectionOverrides)
57+
internalKubeClient, err := kclientsetinternal.NewForConfig(kubeConfig)
58+
if err != nil {
59+
return nil, err
60+
}
61+
externalKubeClient, err := kclientsetexternal.NewForConfig(kubeConfig)
5762
if err != nil {
5863
return nil, err
5964
}
@@ -135,7 +140,6 @@ func New(options configapi.NodeConfig, clusterDomain string, proxyConfig *compon
135140

136141
// TODO: use kubeletConfig.ResolverConfig as an argument to etcd in the event the
137142
// user sets it, instead of passing it to the kubelet.
138-
glog.Infof("DNS Bind to %s", options.DNSBindAddress)
139143
config.DNSServer = dns.NewServer(
140144
dnsConfig,
141145
services,

‎pkg/cmd/server/kubernetes/network/options/options.go

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package node
2+
3+
import (
4+
"fmt"
5+
"net"
6+
"time"
7+
8+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
kerrors "k8s.io/apimachinery/pkg/util/errors"
10+
kubeproxyoptions "k8s.io/kubernetes/cmd/kube-proxy/app"
11+
"k8s.io/kubernetes/pkg/apis/componentconfig"
12+
13+
configapi "github.com/openshift/origin/pkg/cmd/server/api"
14+
cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
15+
)
16+
17+
// Build creates the network Kubernetes component configs for a given NodeConfig, or returns
18+
// an error
19+
func Build(options configapi.NodeConfig) (*componentconfig.KubeProxyConfiguration, error) {
20+
proxyOptions, err := kubeproxyoptions.NewOptions()
21+
if err != nil {
22+
return nil, err
23+
}
24+
// get default config
25+
proxyconfig := proxyOptions.GetConfig()
26+
27+
proxyconfig.HostnameOverride = options.NodeName
28+
29+
// BindAddress - Override default bind address from our config
30+
addr := options.ServingInfo.BindAddress
31+
host, _, err := net.SplitHostPort(addr)
32+
if err != nil {
33+
return nil, fmt.Errorf("The provided value to bind to must be an ip:port %q", addr)
34+
}
35+
ip := net.ParseIP(host)
36+
if ip == nil {
37+
return nil, fmt.Errorf("The provided value to bind to must be an ip:port: %q", addr)
38+
}
39+
proxyconfig.BindAddress = ip.String()
40+
// MetricsBindAddress - disable by default but allow enablement until we switch to
41+
// reading proxy config directly
42+
proxyconfig.MetricsBindAddress = ""
43+
if arg := options.ProxyArguments["metrics-bind-address"]; len(arg) > 0 {
44+
proxyconfig.MetricsBindAddress = arg[0]
45+
}
46+
delete(options.ProxyArguments, "metrics-bind-address")
47+
48+
// OOMScoreAdj, ResourceContainer - clear, we don't run in a container
49+
oomScoreAdj := int32(0)
50+
proxyconfig.OOMScoreAdj = &oomScoreAdj
51+
proxyconfig.ResourceContainer = ""
52+
53+
// use the same client as the node
54+
proxyconfig.ClientConnection.KubeConfigFile = options.MasterKubeConfig
55+
56+
// ProxyMode, set to iptables
57+
proxyconfig.Mode = "iptables"
58+
59+
// IptablesSyncPeriod, set to our config value
60+
syncPeriod, err := time.ParseDuration(options.IPTablesSyncPeriod)
61+
if err != nil {
62+
return nil, fmt.Errorf("Cannot parse the provided ip-tables sync period (%s) : %v", options.IPTablesSyncPeriod, err)
63+
}
64+
proxyconfig.IPTables.SyncPeriod = metav1.Duration{
65+
Duration: syncPeriod,
66+
}
67+
masqueradeBit := int32(0)
68+
proxyconfig.IPTables.MasqueradeBit = &masqueradeBit
69+
70+
// PortRange, use default
71+
// HostnameOverride, use default
72+
// ConfigSyncPeriod, use default
73+
// MasqueradeAll, use default
74+
// CleanupAndExit, use default
75+
// KubeAPIQPS, use default, doesn't apply until we build a separate client
76+
// KubeAPIBurst, use default, doesn't apply until we build a separate client
77+
// UDPIdleTimeout, use default
78+
79+
// Resolve cmd flags to add any user overrides
80+
if err := cmdflags.Resolve(options.ProxyArguments, proxyOptions.AddFlags); len(err) > 0 {
81+
return nil, kerrors.NewAggregate(err)
82+
}
83+
84+
if err := proxyOptions.Complete(); err != nil {
85+
return nil, err
86+
}
87+
88+
return proxyconfig, nil
89+
}

‎pkg/cmd/server/kubernetes/node/node_config.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
1313
kubeletoptions "k8s.io/kubernetes/cmd/kubelet/app/options"
1414
"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
15+
kclientsetexternal "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
1516
"k8s.io/kubernetes/pkg/cloudprovider"
1617
"k8s.io/kubernetes/pkg/kubelet"
1718
dockertools "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
@@ -57,7 +58,7 @@ func New(options configapi.NodeConfig, server *kubeletoptions.KubeletServer) (*N
5758
return nil, err
5859
}
5960
// Make a separate client for event reporting, to avoid event QPS blocking node calls
60-
eventClient, _, err := configapi.GetExternalKubeClient(options.MasterKubeConfig, options.MasterClientConnectionOverrides)
61+
eventClient, err := kclientsetexternal.NewForConfig(kubeConfig)
6162
if err != nil {
6263
return nil, err
6364
}

‎pkg/cmd/server/kubernetes/node/options/options.go

+8-79
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1010
kerrors "k8s.io/apimachinery/pkg/util/errors"
1111
utilfeature "k8s.io/apiserver/pkg/util/feature"
12-
kubeproxyoptions "k8s.io/kubernetes/cmd/kube-proxy/app"
1312
kubeletoptions "k8s.io/kubernetes/cmd/kubelet/app/options"
1413
"k8s.io/kubernetes/pkg/apis/componentconfig"
1514
"k8s.io/kubernetes/pkg/features"
@@ -25,7 +24,7 @@ import (
2524

2625
// Build creates the core Kubernetes component configs for a given NodeConfig, or returns
2726
// an error
28-
func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *componentconfig.KubeProxyConfiguration, error) {
27+
func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, error) {
2928
imageTemplate := variable.NewDefaultImageTemplate()
3029
imageTemplate.Format = options.ImageConfig.Format
3130
imageTemplate.Latest = options.ImageConfig.Latest
@@ -39,11 +38,11 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
3938

4039
kubeAddressStr, kubePortStr, err := net.SplitHostPort(options.ServingInfo.BindAddress)
4140
if err != nil {
42-
return nil, nil, fmt.Errorf("cannot parse node address: %v", err)
41+
return nil, fmt.Errorf("cannot parse node address: %v", err)
4342
}
4443
kubePort, err := strconv.Atoi(kubePortStr)
4544
if err != nil {
46-
return nil, nil, fmt.Errorf("cannot parse node port: %v", err)
45+
return nil, fmt.Errorf("cannot parse node port: %v", err)
4746
}
4847

4948
// Defaults are tested in TestKubeletDefaults
@@ -91,7 +90,7 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
9190
// Setup auth
9291
authnTTL, err := time.ParseDuration(options.AuthConfig.AuthenticationCacheTTL)
9392
if err != nil {
94-
return nil, nil, err
93+
return nil, err
9594
}
9695
server.Authentication = componentconfig.KubeletAuthentication{
9796
X509: componentconfig.KubeletX509Authentication{
@@ -107,7 +106,7 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
107106
}
108107
authzTTL, err := time.ParseDuration(options.AuthConfig.AuthorizationCacheTTL)
109108
if err != nil {
110-
return nil, nil, err
109+
return nil, err
111110
}
112111
server.Authorization = componentconfig.KubeletAuthorization{
113112
Mode: componentconfig.KubeletAuthorizationModeWebhook,
@@ -121,13 +120,13 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
121120
// TODO: this should be done in config validation (along with the above) so we can provide
122121
// proper errors
123122
if err := cmdflags.Resolve(options.KubeletArguments, server.AddFlags); len(err) > 0 {
124-
return nil, nil, kerrors.NewAggregate(err)
123+
return nil, kerrors.NewAggregate(err)
125124
}
126125

127126
// terminate early if feature gate is incorrect on the node
128127
if len(server.FeatureGates) > 0 {
129128
if err := utilfeature.DefaultFeatureGate.Set(server.FeatureGates); err != nil {
130-
return nil, nil, err
129+
return nil, err
131130
}
132131
}
133132
if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
@@ -138,11 +137,6 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
138137
}
139138
}
140139

141-
proxyconfig, err := buildKubeProxyConfig(options)
142-
if err != nil {
143-
return nil, nil, err
144-
}
145-
146140
if network.IsOpenShiftNetworkPlugin(options.NetworkConfig.NetworkPluginName) {
147141
// SDN plugin pod setup/teardown is implemented as a CNI plugin
148142
server.NetworkPluginName = kubeletcni.CNIPluginName
@@ -152,72 +146,7 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
152146
server.HairpinMode = componentconfig.HairpinNone
153147
}
154148

155-
return server, proxyconfig, nil
156-
}
157-
158-
func buildKubeProxyConfig(options configapi.NodeConfig) (*componentconfig.KubeProxyConfiguration, error) {
159-
proxyOptions, err := kubeproxyoptions.NewOptions()
160-
if err != nil {
161-
return nil, err
162-
}
163-
// get default config
164-
proxyconfig := proxyOptions.GetConfig()
165-
166-
// BindAddress - Override default bind address from our config
167-
addr := options.ServingInfo.BindAddress
168-
host, _, err := net.SplitHostPort(addr)
169-
if err != nil {
170-
return nil, fmt.Errorf("The provided value to bind to must be an ip:port %q", addr)
171-
}
172-
ip := net.ParseIP(host)
173-
if ip == nil {
174-
return nil, fmt.Errorf("The provided value to bind to must be an ip:port: %q", addr)
175-
}
176-
proxyconfig.BindAddress = ip.String()
177-
// MetricsBindAddress - disable
178-
proxyconfig.MetricsBindAddress = ""
179-
180-
// OOMScoreAdj, ResourceContainer - clear, we don't run in a container
181-
oomScoreAdj := int32(0)
182-
proxyconfig.OOMScoreAdj = &oomScoreAdj
183-
proxyconfig.ResourceContainer = ""
184-
185-
// use the same client as the node
186-
proxyconfig.ClientConnection.KubeConfigFile = options.MasterKubeConfig
187-
188-
// ProxyMode, set to iptables
189-
proxyconfig.Mode = "iptables"
190-
191-
// IptablesSyncPeriod, set to our config value
192-
syncPeriod, err := time.ParseDuration(options.IPTablesSyncPeriod)
193-
if err != nil {
194-
return nil, fmt.Errorf("Cannot parse the provided ip-tables sync period (%s) : %v", options.IPTablesSyncPeriod, err)
195-
}
196-
proxyconfig.IPTables.SyncPeriod = metav1.Duration{
197-
Duration: syncPeriod,
198-
}
199-
masqueradeBit := int32(0)
200-
proxyconfig.IPTables.MasqueradeBit = &masqueradeBit
201-
202-
// PortRange, use default
203-
// HostnameOverride, use default
204-
// ConfigSyncPeriod, use default
205-
// MasqueradeAll, use default
206-
// CleanupAndExit, use default
207-
// KubeAPIQPS, use default, doesn't apply until we build a separate client
208-
// KubeAPIBurst, use default, doesn't apply until we build a separate client
209-
// UDPIdleTimeout, use default
210-
211-
// Resolve cmd flags to add any user overrides
212-
if err := cmdflags.Resolve(options.ProxyArguments, proxyOptions.AddFlags); len(err) > 0 {
213-
return nil, kerrors.NewAggregate(err)
214-
}
215-
216-
if err := proxyOptions.Complete(); err != nil {
217-
return nil, err
218-
}
219-
220-
return proxyconfig, nil
149+
return server, nil
221150
}
222151

223152
func ToFlags(config *kubeletoptions.KubeletServer) []string {

0 commit comments

Comments
 (0)