Skip to content

Commit 6817db8

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #5936 from wangke19/backport-pr-5876-to-4.18
OCPBUGS-54411: Backport PR 5876 to release-4.18
2 parents fb635de + aa1a65a commit 6817db8

File tree

11 files changed

+225
-12
lines changed

11 files changed

+225
-12
lines changed

‎control-plane-operator/controllers/hostedcontrolplane/oapi/deployment.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package oapi
22

33
import (
44
"fmt"
5-
"net/url"
65
"path"
76
"strings"
87

@@ -138,9 +137,9 @@ func ReconcileDeployment(deployment *appsv1.Deployment,
138137
}
139138
}
140139
deployment.Spec.Template.ObjectMeta.Labels = openShiftAPIServerLabels()
141-
etcdUrlData, err := url.Parse(etcdURL)
140+
etcdHost, err := util.HostFromURL(etcdURL)
142141
if err != nil {
143-
return fmt.Errorf("failed to parse etcd url: %w", err)
142+
return err
144143
}
145144
if deployment.Spec.Template.Annotations == nil {
146145
deployment.Spec.Template.Annotations = map[string]string{}
@@ -151,7 +150,7 @@ func ReconcileDeployment(deployment *appsv1.Deployment,
151150
deployment.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(false)
152151
deployment.Spec.Template.Spec.InitContainers = []corev1.Container{util.BuildContainer(oasTrustAnchorGenerator(), buildOASTrustAnchorGenerator(image))}
153152
deployment.Spec.Template.Spec.Containers = []corev1.Container{
154-
util.BuildContainer(oasContainerMain(), buildOASContainerMain(image, strings.Split(etcdUrlData.Host, ":")[0], defaultOAPIPort, internalOAuthDisable)),
153+
util.BuildContainer(oasContainerMain(), buildOASContainerMain(image, etcdHost, defaultOAPIPort, internalOAuthDisable)),
155154
util.BuildContainer(oasKonnectivityProxyContainer(), buildOASKonnectivityProxyContainer(konnectivityHTTPSProxyImage, proxyConfig, noProxy)),
156155
}
157156
deployment.Spec.Template.Spec.Volumes = []corev1.Volume{

‎control-plane-operator/controllers/hostedcontrolplane/oapi/deployment_test.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,9 @@ func TestReconcileOpenshiftOAuthAPIServerDeployment(t *testing.T) {
236236
name: "Empty deployment config and oauth params",
237237
deploymentConfig: config.DeploymentConfig{},
238238
auditConfig: manifests.OpenShiftOAuthAPIServerAuditConfig(targetNamespace),
239-
params: OAuthDeploymentParams{},
239+
params: OAuthDeploymentParams{
240+
EtcdURL: "https://etcd-client:2379",
241+
},
240242
},
241243
}
242244
for _, tc := range testCases {

‎control-plane-operator/controllers/hostedcontrolplane/oapi/oauth_deployment.go

+85-3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package oapi
33
import (
44
"fmt"
55
"path"
6+
"strings"
67

78
configv1 "github.com/openshift/api/config/v1"
89
hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
@@ -40,6 +41,11 @@ var (
4041
oauthVolumeEtcdClientCert().Name: "/etc/kubernetes/certs/etcd-client",
4142
common.VolumeTotalClientCA().Name: "/etc/kubernetes/certs/client-ca",
4243
},
44+
oauthKonnectivityProxyContainer().Name: {
45+
oauthVolumeKubeconfig().Name: "/etc/kubernetes/secrets/kubeconfig",
46+
oauthVolumeKonnectivityProxyCert().Name: "/etc/konnectivity/proxy-client",
47+
oauthVolumeKonnectivityProxyCA().Name: "/etc/konnectivity/proxy-ca",
48+
},
4349
}
4450
oauthAuditWebhookConfigFileVolumeMount = util.PodVolumeMounts{
4551
oauthContainerMain().Name: {
@@ -55,9 +61,18 @@ func openShiftOAuthAPIServerLabels() map[string]string {
5561
}
5662
}
5763

58-
func ReconcileOAuthAPIServerDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, auditConfig *corev1.ConfigMap, p *OAuthDeploymentParams, platformType hyperv1.PlatformType) error {
64+
func ReconcileOAuthAPIServerDeployment(deployment *appsv1.Deployment,
65+
ownerRef config.OwnerRef,
66+
auditConfig *corev1.ConfigMap,
67+
p *OAuthDeploymentParams,
68+
platformType hyperv1.PlatformType) error {
5969
ownerRef.ApplyTo(deployment)
6070

71+
etcdHost, err := util.HostFromURL(p.EtcdURL)
72+
if err != nil {
73+
return err
74+
}
75+
6176
// preserve existing resource requirements for main oauth apiserver container
6277
mainContainer := util.FindContainer(oauthContainerMain().Name, deployment.Spec.Template.Spec.Containers)
6378
if mainContainer != nil {
@@ -95,7 +110,8 @@ func ReconcileOAuthAPIServerDeployment(deployment *appsv1.Deployment, ownerRef c
95110
AutomountServiceAccountToken: ptr.To(false),
96111
TerminationGracePeriodSeconds: ptr.To[int64](120),
97112
Containers: []corev1.Container{
98-
util.BuildContainer(oauthContainerMain(), buildOAuthContainerMain(p)),
113+
util.BuildContainer(oauthContainerMain(), buildOAuthContainerMain(p, etcdHost)),
114+
util.BuildContainer(oauthKonnectivityProxyContainer(), buildOAuthKonnectivityProxyContainer(p.KonnectivityProxyImage)),
99115
},
100116
Volumes: []corev1.Volume{
101117
util.BuildVolume(oauthVolumeWorkLogs(), buildOAuthVolumeWorkLogs),
@@ -106,6 +122,8 @@ func ReconcileOAuthAPIServerDeployment(deployment *appsv1.Deployment, ownerRef c
106122
util.BuildVolume(oauthVolumeServingCert(), buildOAuthVolumeServingCert),
107123
util.BuildVolume(oauthVolumeEtcdClientCert(), buildOAuthVolumeEtcdClientCert),
108124
util.BuildVolume(common.VolumeTotalClientCA(), common.BuildVolumeTotalClientCA),
125+
util.BuildVolume(oauthVolumeKonnectivityProxyCert(), buildOAuthVolumeKonnectivityProxyCert),
126+
util.BuildVolume(oauthVolumeKonnectivityProxyCA(), buildOAuthVolumeKonnectivityProxyCA),
109127
},
110128
}
111129

@@ -147,7 +165,13 @@ func oauthContainerMain() *corev1.Container {
147165
}
148166
}
149167

150-
func buildOAuthContainerMain(p *OAuthDeploymentParams) func(c *corev1.Container) {
168+
func oauthKonnectivityProxyContainer() *corev1.Container {
169+
return &corev1.Container{
170+
Name: "konnectivity-proxy",
171+
}
172+
}
173+
174+
func buildOAuthContainerMain(p *OAuthDeploymentParams, etcdHost string) func(c *corev1.Container) {
151175
return func(c *corev1.Container) {
152176
cpath := func(volume, file string) string {
153177
return path.Join(oauthVolumeMounts.Path(c.Name, volume), file)
@@ -199,6 +223,41 @@ func buildOAuthContainerMain(p *OAuthDeploymentParams) func(c *corev1.Container)
199223
corev1.ResourceCPU: resource.MustParse("10m"),
200224
},
201225
}
226+
c.Env = append(c.Env, []corev1.EnvVar{
227+
{
228+
Name: "HTTP_PROXY",
229+
Value: "socks5://127.0.0.1:8090",
230+
},
231+
{
232+
Name: "HTTPS_PROXY",
233+
Value: "socks5://127.0.0.1:8090",
234+
},
235+
{
236+
Name: "NO_PROXY",
237+
Value: strings.Join([]string{
238+
manifests.KubeAPIServerService("").Name,
239+
etcdHost,
240+
config.AuditWebhookService,
241+
}, ","),
242+
},
243+
}...)
244+
}
245+
}
246+
247+
func buildOAuthKonnectivityProxyContainer(image string) func(c *corev1.Container) {
248+
return func(c *corev1.Container) {
249+
c.Image = image
250+
c.Command = []string{"/usr/bin/control-plane-operator", "konnectivity-socks5-proxy"}
251+
c.Args = []string{"run", "--resolve-from-guest-cluster-dns=true"}
252+
c.Env = []corev1.EnvVar{{
253+
Name: "KUBECONFIG",
254+
Value: fmt.Sprintf("%s/kubeconfig", volumeMounts.Path(c.Name, oauthVolumeKubeconfig().Name)),
255+
}}
256+
c.Resources.Requests = corev1.ResourceList{
257+
corev1.ResourceCPU: resource.MustParse("10m"),
258+
corev1.ResourceMemory: resource.MustParse("10Mi"),
259+
}
260+
c.VolumeMounts = oauthVolumeMounts.ContainerMounts(c.Name)
202261
}
203262
}
204263

@@ -297,6 +356,29 @@ func oauthAuditWebhookConfigFile() string {
297356
return path.Join(cfgDir, hyperv1.AuditWebhookKubeconfigKey)
298357
}
299358

359+
func oauthVolumeKonnectivityProxyCert() *corev1.Volume {
360+
return &corev1.Volume{
361+
Name: "oauth-konnectivity-proxy-cert",
362+
}
363+
}
364+
365+
func oauthVolumeKonnectivityProxyCA() *corev1.Volume {
366+
return &corev1.Volume{
367+
Name: "oauth-konnectivity-proxy-ca",
368+
}
369+
}
370+
371+
func buildOAuthVolumeKonnectivityProxyCert(v *corev1.Volume) {
372+
v.Secret = &corev1.SecretVolumeSource{}
373+
v.Secret.SecretName = manifests.KonnectivityClientSecret("").Name
374+
v.Secret.DefaultMode = ptr.To[int32](0640)
375+
}
376+
377+
func buildOAuthVolumeKonnectivityProxyCA(v *corev1.Volume) {
378+
v.ConfigMap = &corev1.ConfigMapVolumeSource{}
379+
v.ConfigMap.Name = manifests.KonnectivityCAConfigMap("").Name
380+
}
381+
300382
func buildOAuthVolumeEtcdClientCert(v *corev1.Volume) {
301383
v.Secret = &corev1.SecretVolumeSource{}
302384
v.Secret.SecretName = manifests.EtcdClientSecret("").Name

‎control-plane-operator/controllers/hostedcontrolplane/oapi/params.go

+2
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ type OAuthDeploymentParams struct {
4444
ServiceAccountIssuerURL string
4545
DeploymentConfig config.DeploymentConfig
4646
AvailabilityProberImage string
47+
KonnectivityProxyImage string
4748
Availability hyperv1.AvailabilityPolicy
4849
OwnerRef config.OwnerRef
4950
AuditWebhookRef *corev1.LocalObjectReference
@@ -220,6 +221,7 @@ func (p *OpenShiftAPIServerParams) AuditPolicyConfig() configv1.Audit {
220221
func (p *OpenShiftAPIServerParams) OAuthAPIServerDeploymentParams(hcp *hyperv1.HostedControlPlane) *OAuthDeploymentParams {
221222
params := &OAuthDeploymentParams{
222223
Image: p.OAuthAPIServerImage,
224+
KonnectivityProxyImage: p.ProxyImage,
223225
EtcdURL: p.EtcdURL,
224226
ServiceAccountIssuerURL: p.ServiceAccountIssuerURL,
225227
DeploymentConfig: p.OpenShiftOAuthAPIServerDeploymentConfig,

‎control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/zz_fixture_TestControlPlaneComponents.yaml

+36
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,13 @@ spec:
9595
- --tls-min-version=VersionTLS12
9696
command:
9797
- /usr/bin/oauth-apiserver
98+
env:
99+
- name: HTTP_PROXY
100+
value: socks5://127.0.0.1:8090
101+
- name: HTTPS_PROXY
102+
value: socks5://127.0.0.1:8090
103+
- name: NO_PROXY
104+
value: kube-apiserver,etcd-client,audit-webhook
98105
image: oauth-apiserver
99106
imagePullPolicy: IfNotPresent
100107
livenessProbe:
@@ -167,6 +174,28 @@ spec:
167174
volumeMounts:
168175
- mountPath: /var/log/openshift-oauth-apiserver
169176
name: work-logs
177+
- args:
178+
- run
179+
- --resolve-from-guest-cluster-dns=true
180+
command:
181+
- /usr/bin/control-plane-operator
182+
- konnectivity-socks5-proxy
183+
env:
184+
- name: KUBECONFIG
185+
value: /etc/kubernetes/secrets/kubeconfig/kubeconfig
186+
image: controlplane-operator
187+
name: konnectivity-proxy-socks5
188+
resources:
189+
requests:
190+
cpu: 10m
191+
memory: 10Mi
192+
volumeMounts:
193+
- mountPath: /etc/kubernetes/secrets/kubeconfig
194+
name: kubeconfig
195+
- mountPath: /etc/konnectivity/proxy-client
196+
name: konnectivity-proxy-cert
197+
- mountPath: /etc/konnectivity/proxy-ca
198+
name: konnectivity-proxy-ca
170199
initContainers:
171200
- command:
172201
- /usr/bin/control-plane-operator
@@ -219,4 +248,11 @@ spec:
219248
defaultMode: 420
220249
name: client-ca
221250
name: client-ca
251+
- name: konnectivity-proxy-cert
252+
secret:
253+
defaultMode: 416
254+
secretName: konnectivity-client
255+
- configMap:
256+
name: konnectivity-ca-bundle
257+
name: konnectivity-proxy-ca
222258
status: {}

‎control-plane-operator/controllers/hostedcontrolplane/v2/assets/openshift-oauth-apiserver/deployment.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,13 @@ spec:
4747
- --client-ca-file=/etc/kubernetes/certs/client-ca/ca.crt
4848
command:
4949
- /usr/bin/oauth-apiserver
50+
env:
51+
- name: HTTP_PROXY
52+
value: socks5://127.0.0.1:8090
53+
- name: HTTPS_PROXY
54+
value: socks5://127.0.0.1:8090
55+
- name: NO_PROXY
56+
value: kube-apiserver,etcd-client,audit-webhook
5057
image: oauth-apiserver
5158
imagePullPolicy: IfNotPresent
5259
livenessProbe:

‎control-plane-operator/controllers/hostedcontrolplane/v2/oapi/deployment.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package oapi
22

33
import (
44
"fmt"
5-
"net/url"
65
"path"
76
"strings"
87

@@ -44,11 +43,10 @@ func adaptDeployment(cpContext component.ControlPlaneContext, deployment *appsv1
4443

4544
etcdHostname := "etcd-client"
4645
if cpContext.HCP.Spec.Etcd.ManagementType == hyperv1.Unmanaged {
47-
etcdUrl, err := url.Parse(cpContext.HCP.Spec.Etcd.Unmanaged.Endpoint)
46+
etcdHostname, err = util.HostFromURL(cpContext.HCP.Spec.Etcd.Unmanaged.Endpoint)
4847
if err != nil {
49-
return fmt.Errorf("failed to parse etcd url: %w", err)
48+
return err
5049
}
51-
etcdHostname = strings.Split(etcdUrl.Host, ":")[0]
5250
}
5351
noProxy := []string{
5452
manifests.KubeAPIServerService("").Name,

‎control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/component.go

+7
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/openshift/hypershift/support/util"
88

99
corev1 "k8s.io/api/core/v1"
10+
"k8s.io/utils/ptr"
1011
)
1112

1213
const (
@@ -48,6 +49,12 @@ func NewComponent() component.ControlPlaneComponent {
4849
WithDependencies(oapiv2.ComponentName).
4950
WatchResource(&corev1.ConfigMap{}, "openshift-oauth-apiserver-audit").
5051
InjectAvailabilityProberContainer(util.AvailabilityProberOpts{}).
52+
InjectKonnectivityContainer(component.KonnectivityContainerOptions{
53+
Mode: component.Socks5,
54+
Socks5Options: component.Socks5Options{
55+
ResolveFromGuestClusterDNS: ptr.To(true),
56+
},
57+
}).
5158
Build()
5259
}
5360

‎control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/deployment.go

+21
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@ package oapi
33
import (
44
"fmt"
55
"path"
6+
"strings"
67

78
configv1 "github.com/openshift/api/config/v1"
89
hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
10+
"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
911
appsv1 "k8s.io/api/apps/v1"
1012
corev1 "k8s.io/api/core/v1"
1113

@@ -19,6 +21,21 @@ const (
1921
)
2022

2123
func adaptDeployment(cpContext component.ControlPlaneContext, deployment *appsv1.Deployment) error {
24+
25+
var err error
26+
etcdHostname := "etcd-client"
27+
if cpContext.HCP.Spec.Etcd.ManagementType == hyperv1.Unmanaged {
28+
etcdHostname, err = util.HostFromURL(cpContext.HCP.Spec.Etcd.Unmanaged.Endpoint)
29+
if err != nil {
30+
return err
31+
}
32+
}
33+
noProxy := []string{
34+
manifests.KubeAPIServerService("").Name,
35+
etcdHostname,
36+
config.AuditWebhookService,
37+
}
38+
2239
util.UpdateContainer(ComponentName, deployment.Spec.Template.Spec.Containers, func(c *corev1.Container) {
2340
etcdURL := config.DefaultEtcdURL
2441
if cpContext.HCP.Spec.Etcd.ManagementType == hyperv1.Unmanaged {
@@ -41,6 +58,10 @@ func adaptDeployment(cpContext component.ControlPlaneContext, deployment *appsv1
4158
tokenInactivityTimeout := configuration.OAuth.TokenConfig.AccessTokenInactivityTimeout.Duration.String()
4259
c.Args = append(c.Args, fmt.Sprintf("--accesstoken-inactivity-timeout=%s", tokenInactivityTimeout))
4360
}
61+
util.UpsertEnvVar(c, corev1.EnvVar{
62+
Name: "NO_PROXY",
63+
Value: strings.Join(noProxy, ","),
64+
})
4465
})
4566

4667
if cpContext.HCP.Spec.Configuration.GetAuditPolicyConfig().Profile == configv1.NoneAuditProfileType {

‎support/util/util.go

+28
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import (
1212
"io"
1313
"net"
1414
"net/http"
15+
"net/url"
1516
"os"
17+
"regexp"
1618
"sort"
1719
"strings"
1820
"time"
@@ -538,3 +540,29 @@ func GetPullSecretBytes(ctx context.Context, c client.Client, hc *hyperv1.Hosted
538540

539541
return pullSecretBytes, nil
540542
}
543+
544+
var (
545+
hasPortRegex = regexp.MustCompile(`:\d{1,5}$`)
546+
)
547+
548+
func HostFromURL(addr string) (string, error) {
549+
parsedURL, err := url.Parse(addr)
550+
if err != nil {
551+
return "", fmt.Errorf("failed to parse URL(%s): %w", addr, err)
552+
}
553+
hostPort := parsedURL.Host
554+
if hostPort == "" {
555+
return "", fmt.Errorf("missing host/port name in URL(%s)", addr)
556+
}
557+
if !hasPortRegex.MatchString(hostPort) {
558+
return hostPort, nil
559+
}
560+
hostName, _, err := net.SplitHostPort(hostPort)
561+
if err != nil {
562+
return "", fmt.Errorf("failed to split host/port from (%s): %w", hostPort, err)
563+
}
564+
if hostName == "" {
565+
return "", fmt.Errorf("missing host name in URL(%s)", addr)
566+
}
567+
return hostName, nil
568+
}

0 commit comments

Comments
 (0)