Skip to content

Commit 890a6c8

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#118895 from RyanAoh/kep-1860
Make Kubernetes aware of the LoadBalancer behaviour
2 parents 4f60a8d + b185049 commit 890a6c8

35 files changed

+1870
-634
lines changed

‎api/openapi-spec/swagger.json

+4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/openapi-spec/v3/api__v1_openapi.json

+4
Original file line numberDiff line numberDiff line change
@@ -3102,6 +3102,10 @@
31023102
"description": "IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)",
31033103
"type": "string"
31043104
},
3105+
"ipMode": {
3106+
"description": "IPMode specifies how the load-balancer IP behaves, and may only be specified when the ip field is specified. Setting this to \"VIP\" indicates that traffic is delivered to the node with the destination set to the load-balancer's IP and port. Setting this to \"Proxy\" indicates that traffic is delivered to the node or pod with the destination set to the node's IP and node port or the pod's IP and port. Service implementations may use this information to adjust traffic routing.",
3107+
"type": "string"
3108+
},
31053109
"ports": {
31063110
"description": "Ports is a list of records of service ports If used, every port defined in the service should have an entry in it",
31073111
"items": {

‎pkg/apis/core/types.go

+21
Original file line numberDiff line numberDiff line change
@@ -4018,6 +4018,15 @@ type LoadBalancerIngress struct {
40184018
// +optional
40194019
Hostname string
40204020

4021+
// IPMode specifies how the load-balancer IP behaves, and may only be specified when the ip field is specified.
4022+
// Setting this to "VIP" indicates that traffic is delivered to the node with
4023+
// the destination set to the load-balancer's IP and port.
4024+
// Setting this to "Proxy" indicates that traffic is delivered to the node or pod with
4025+
// the destination set to the node's IP and node port or the pod's IP and port.
4026+
// Service implementations may use this information to adjust traffic routing.
4027+
// +optional
4028+
IPMode *LoadBalancerIPMode
4029+
40214030
// Ports is a list of records of service ports
40224031
// If used, every port defined in the service should have an entry in it
40234032
// +optional
@@ -6090,3 +6099,15 @@ type PortStatus struct {
60906099
// +kubebuilder:validation:MaxLength=316
60916100
Error *string
60926101
}
6102+
6103+
// LoadBalancerIPMode represents the mode of the LoadBalancer ingress IP
6104+
type LoadBalancerIPMode string
6105+
6106+
const (
6107+
// LoadBalancerIPModeVIP indicates that traffic is delivered to the node with
6108+
// the destination set to the load-balancer's IP and port.
6109+
LoadBalancerIPModeVIP LoadBalancerIPMode = "VIP"
6110+
// LoadBalancerIPModeProxy indicates that traffic is delivered to the node or pod with
6111+
// the destination set to the node's IP and port or the pod's IP and port.
6112+
LoadBalancerIPModeProxy LoadBalancerIPMode = "Proxy"
6113+
)

‎pkg/apis/core/v1/defaults.go

+13
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,19 @@ func SetDefaults_Service(obj *v1.Service) {
142142
obj.Spec.AllocateLoadBalancerNodePorts = pointer.Bool(true)
143143
}
144144
}
145+
146+
if obj.Spec.Type == v1.ServiceTypeLoadBalancer {
147+
if utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) {
148+
ipMode := v1.LoadBalancerIPModeVIP
149+
150+
for i, ing := range obj.Status.LoadBalancer.Ingress {
151+
if ing.IP != "" && ing.IPMode == nil {
152+
obj.Status.LoadBalancer.Ingress[i].IPMode = &ipMode
153+
}
154+
}
155+
}
156+
}
157+
145158
}
146159
func SetDefaults_Pod(obj *v1.Pod) {
147160
// If limits are specified, but requests are not, default requests to limits

‎pkg/apis/core/v1/defaults_test.go

+68
Original file line numberDiff line numberDiff line change
@@ -1221,6 +1221,74 @@ func TestSetDefaultServiceSessionAffinityConfig(t *testing.T) {
12211221
}
12221222
}
12231223

1224+
func TestSetDefaultServiceLoadbalancerIPMode(t *testing.T) {
1225+
modeVIP := v1.LoadBalancerIPModeVIP
1226+
modeProxy := v1.LoadBalancerIPModeProxy
1227+
testCases := []struct {
1228+
name string
1229+
ipModeEnabled bool
1230+
svc *v1.Service
1231+
expectedIPMode []*v1.LoadBalancerIPMode
1232+
}{
1233+
{
1234+
name: "Set IP but not set IPMode with LoadbalancerIPMode disabled",
1235+
ipModeEnabled: false,
1236+
svc: &v1.Service{
1237+
Spec: v1.ServiceSpec{Type: v1.ServiceTypeLoadBalancer},
1238+
Status: v1.ServiceStatus{
1239+
LoadBalancer: v1.LoadBalancerStatus{
1240+
Ingress: []v1.LoadBalancerIngress{{
1241+
IP: "1.2.3.4",
1242+
}},
1243+
},
1244+
}},
1245+
expectedIPMode: []*v1.LoadBalancerIPMode{nil},
1246+
}, {
1247+
name: "Set IP but bot set IPMode with LoadbalancerIPMode enabled",
1248+
ipModeEnabled: true,
1249+
svc: &v1.Service{
1250+
Spec: v1.ServiceSpec{Type: v1.ServiceTypeLoadBalancer},
1251+
Status: v1.ServiceStatus{
1252+
LoadBalancer: v1.LoadBalancerStatus{
1253+
Ingress: []v1.LoadBalancerIngress{{
1254+
IP: "1.2.3.4",
1255+
}},
1256+
},
1257+
}},
1258+
expectedIPMode: []*v1.LoadBalancerIPMode{&modeVIP},
1259+
}, {
1260+
name: "Both IP and IPMode are set with LoadbalancerIPMode enabled",
1261+
ipModeEnabled: true,
1262+
svc: &v1.Service{
1263+
Spec: v1.ServiceSpec{Type: v1.ServiceTypeLoadBalancer},
1264+
Status: v1.ServiceStatus{
1265+
LoadBalancer: v1.LoadBalancerStatus{
1266+
Ingress: []v1.LoadBalancerIngress{{
1267+
IP: "1.2.3.4",
1268+
IPMode: &modeProxy,
1269+
}},
1270+
},
1271+
}},
1272+
expectedIPMode: []*v1.LoadBalancerIPMode{&modeProxy},
1273+
},
1274+
}
1275+
1276+
for _, tc := range testCases {
1277+
t.Run(tc.name, func(t *testing.T) {
1278+
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled)()
1279+
obj := roundTrip(t, runtime.Object(tc.svc))
1280+
svc := obj.(*v1.Service)
1281+
for i, s := range svc.Status.LoadBalancer.Ingress {
1282+
got := s.IPMode
1283+
expected := tc.expectedIPMode[i]
1284+
if !reflect.DeepEqual(got, expected) {
1285+
t.Errorf("Expected IPMode %v, got %v", tc.expectedIPMode[i], s.IPMode)
1286+
}
1287+
}
1288+
})
1289+
}
1290+
}
1291+
12241292
func TestSetDefaultSecretVolumeSource(t *testing.T) {
12251293
s := v1.PodSpec{}
12261294
s.Volumes = []v1.Volume{

‎pkg/apis/core/v1/zz_generated.conversion.go

+2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/apis/core/validation/validation.go

+15
Original file line numberDiff line numberDiff line change
@@ -6997,6 +6997,10 @@ func ValidatePodLogOptions(opts *core.PodLogOptions) field.ErrorList {
69976997
return allErrs
69986998
}
69996999

7000+
var (
7001+
supportedLoadBalancerIPMode = sets.NewString(string(core.LoadBalancerIPModeVIP), string(core.LoadBalancerIPModeProxy))
7002+
)
7003+
70007004
// ValidateLoadBalancerStatus validates required fields on a LoadBalancerStatus
70017005
func ValidateLoadBalancerStatus(status *core.LoadBalancerStatus, fldPath *field.Path) field.ErrorList {
70027006
allErrs := field.ErrorList{}
@@ -7007,6 +7011,17 @@ func ValidateLoadBalancerStatus(status *core.LoadBalancerStatus, fldPath *field.
70077011
allErrs = append(allErrs, field.Invalid(idxPath.Child("ip"), ingress.IP, "must be a valid IP address"))
70087012
}
70097013
}
7014+
7015+
if utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) && ingress.IPMode == nil {
7016+
if len(ingress.IP) > 0 {
7017+
allErrs = append(allErrs, field.Required(idxPath.Child("ipMode"), "must be specified when `ip` is set"))
7018+
}
7019+
} else if ingress.IPMode != nil && len(ingress.IP) == 0 {
7020+
allErrs = append(allErrs, field.Forbidden(idxPath.Child("ipMode"), "may not be specified when `ip` is not set"))
7021+
} else if ingress.IPMode != nil && !supportedLoadBalancerIPMode.Has(string(*ingress.IPMode)) {
7022+
allErrs = append(allErrs, field.NotSupported(idxPath.Child("ipMode"), ingress.IPMode, supportedLoadBalancerIPMode.List()))
7023+
}
7024+
70107025
if len(ingress.Hostname) > 0 {
70117026
for _, msg := range validation.IsDNS1123Subdomain(ingress.Hostname) {
70127027
allErrs = append(allErrs, field.Invalid(idxPath.Child("hostname"), ingress.Hostname, msg))

‎pkg/apis/core/validation/validation_test.go

+84
Original file line numberDiff line numberDiff line change
@@ -22997,3 +22997,87 @@ func TestValidateDynamicResourceAllocation(t *testing.T) {
2299722997
}
2299822998
})
2299922999
}
23000+
23001+
func TestValidateLoadBalancerStatus(t *testing.T) {
23002+
ipModeVIP := core.LoadBalancerIPModeVIP
23003+
ipModeProxy := core.LoadBalancerIPModeProxy
23004+
ipModeDummy := core.LoadBalancerIPMode("dummy")
23005+
23006+
testCases := []struct {
23007+
name string
23008+
ipModeEnabled bool
23009+
tweakLBStatus func(s *core.LoadBalancerStatus)
23010+
numErrs int
23011+
}{
23012+
/* LoadBalancerIPMode*/
23013+
{
23014+
name: "valid vip ipMode",
23015+
ipModeEnabled: true,
23016+
tweakLBStatus: func(s *core.LoadBalancerStatus) {
23017+
s.Ingress = []core.LoadBalancerIngress{{
23018+
IP: "1.2.3.4",
23019+
IPMode: &ipModeVIP,
23020+
}}
23021+
},
23022+
numErrs: 0,
23023+
}, {
23024+
name: "valid proxy ipMode",
23025+
ipModeEnabled: true,
23026+
tweakLBStatus: func(s *core.LoadBalancerStatus) {
23027+
s.Ingress = []core.LoadBalancerIngress{{
23028+
IP: "1.2.3.4",
23029+
IPMode: &ipModeProxy,
23030+
}}
23031+
},
23032+
numErrs: 0,
23033+
}, {
23034+
name: "invalid ipMode",
23035+
ipModeEnabled: true,
23036+
tweakLBStatus: func(s *core.LoadBalancerStatus) {
23037+
s.Ingress = []core.LoadBalancerIngress{{
23038+
IP: "1.2.3.4",
23039+
IPMode: &ipModeDummy,
23040+
}}
23041+
},
23042+
numErrs: 1,
23043+
}, {
23044+
name: "missing ipMode with LoadbalancerIPMode enabled",
23045+
ipModeEnabled: true,
23046+
tweakLBStatus: func(s *core.LoadBalancerStatus) {
23047+
s.Ingress = []core.LoadBalancerIngress{{
23048+
IP: "1.2.3.4",
23049+
}}
23050+
},
23051+
numErrs: 1,
23052+
}, {
23053+
name: "missing ipMode with LoadbalancerIPMode disabled",
23054+
ipModeEnabled: false,
23055+
tweakLBStatus: func(s *core.LoadBalancerStatus) {
23056+
s.Ingress = []core.LoadBalancerIngress{{
23057+
IP: "1.2.3.4",
23058+
}}
23059+
},
23060+
numErrs: 0,
23061+
}, {
23062+
name: "missing ip with ipMode present",
23063+
ipModeEnabled: true,
23064+
tweakLBStatus: func(s *core.LoadBalancerStatus) {
23065+
s.Ingress = []core.LoadBalancerIngress{{
23066+
IPMode: &ipModeProxy,
23067+
}}
23068+
},
23069+
numErrs: 1,
23070+
},
23071+
}
23072+
for _, tc := range testCases {
23073+
t.Run(tc.name, func(t *testing.T) {
23074+
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled)()
23075+
s := core.LoadBalancerStatus{}
23076+
tc.tweakLBStatus(&s)
23077+
errs := ValidateLoadBalancerStatus(&s, field.NewPath("status"))
23078+
if len(errs) != tc.numErrs {
23079+
t.Errorf("Unexpected error list for case %q(expected:%v got %v) - Errors:\n %v", tc.name, tc.numErrs, len(errs), errs.ToAggregate())
23080+
}
23081+
})
23082+
}
23083+
}

‎pkg/apis/core/zz_generated.deepcopy.go

+5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/features/kube_features.go

+8
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,12 @@ const (
892892
//
893893
// Enables In-Place Pod Vertical Scaling
894894
InPlacePodVerticalScaling featuregate.Feature = "InPlacePodVerticalScaling"
895+
896+
// owner: @Sh4d1,@RyanAoh
897+
// kep: http://kep.k8s.io/1860
898+
// alpha: v1.29
899+
// LoadBalancerIPMode enables the IPMode field in the LoadBalancerIngress status of a Service
900+
LoadBalancerIPMode featuregate.Feature = "LoadBalancerIPMode"
895901
)
896902

897903
func init() {
@@ -1133,6 +1139,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
11331139

11341140
PodIndexLabel: {Default: true, PreRelease: featuregate.Beta},
11351141

1142+
LoadBalancerIPMode: {Default: false, PreRelease: featuregate.Alpha},
1143+
11361144
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
11371145
// unintentionally on either side:
11381146

‎pkg/generated/openapi/zz_generated.openapi.go

+7
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/proxy/conntrack/cleanup.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func deleteStaleServiceConntrackEntries(isIPv6 bool, exec utilexec.Interface, sv
5050
for _, extIP := range svcInfo.ExternalIPStrings() {
5151
conntrackCleanupServiceIPs.Insert(extIP)
5252
}
53-
for _, lbIP := range svcInfo.LoadBalancerIPStrings() {
53+
for _, lbIP := range svcInfo.LoadBalancerVIPStrings() {
5454
conntrackCleanupServiceIPs.Insert(lbIP)
5555
}
5656
nodePort := svcInfo.NodePort()
@@ -100,7 +100,7 @@ func deleteStaleEndpointConntrackEntries(exec utilexec.Interface, svcPortMap pro
100100
klog.ErrorS(err, "Failed to delete endpoint connections for externalIP", "servicePortName", epSvcPair.ServicePortName, "externalIP", extIP)
101101
}
102102
}
103-
for _, lbIP := range svcInfo.LoadBalancerIPStrings() {
103+
for _, lbIP := range svcInfo.LoadBalancerVIPStrings() {
104104
err := ClearEntriesForNAT(exec, lbIP, endpointIP, v1.ProtocolUDP)
105105
if err != nil {
106106
klog.ErrorS(err, "Failed to delete endpoint connections for LoadBalancerIP", "servicePortName", epSvcPair.ServicePortName, "loadBalancerIP", lbIP)

‎pkg/proxy/iptables/proxier.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -1032,7 +1032,7 @@ func (proxier *Proxier) syncProxyRules() {
10321032
// create a firewall chain.
10331033
loadBalancerTrafficChain := externalTrafficChain
10341034
fwChain := svcInfo.firewallChainName
1035-
usesFWChain := hasEndpoints && len(svcInfo.LoadBalancerIPStrings()) > 0 && len(svcInfo.LoadBalancerSourceRanges()) > 0
1035+
usesFWChain := hasEndpoints && len(svcInfo.LoadBalancerVIPStrings()) > 0 && len(svcInfo.LoadBalancerSourceRanges()) > 0
10361036
if usesFWChain {
10371037
activeNATChains[fwChain] = true
10381038
loadBalancerTrafficChain = fwChain
@@ -1124,7 +1124,7 @@ func (proxier *Proxier) syncProxyRules() {
11241124
}
11251125

11261126
// Capture load-balancer ingress.
1127-
for _, lbip := range svcInfo.LoadBalancerIPStrings() {
1127+
for _, lbip := range svcInfo.LoadBalancerVIPStrings() {
11281128
if hasEndpoints {
11291129
natRules.Write(
11301130
"-A", string(kubeServicesChain),
@@ -1149,7 +1149,7 @@ func (proxier *Proxier) syncProxyRules() {
11491149
// Either no endpoints at all (REJECT) or no endpoints for
11501150
// external traffic (DROP anything that didn't get short-circuited
11511151
// by the EXT chain.)
1152-
for _, lbip := range svcInfo.LoadBalancerIPStrings() {
1152+
for _, lbip := range svcInfo.LoadBalancerVIPStrings() {
11531153
filterRules.Write(
11541154
"-A", string(kubeExternalServicesChain),
11551155
"-m", "comment", "--comment", externalTrafficFilterComment,
@@ -1327,7 +1327,7 @@ func (proxier *Proxier) syncProxyRules() {
13271327
// will loop back with the source IP set to the VIP. We
13281328
// need the following rules to allow requests from this node.
13291329
if allowFromNode {
1330-
for _, lbip := range svcInfo.LoadBalancerIPStrings() {
1330+
for _, lbip := range svcInfo.LoadBalancerVIPStrings() {
13311331
natRules.Write(
13321332
args,
13331333
"-s", lbip,

0 commit comments

Comments
 (0)