8
8
"net"
9
9
"os"
10
10
"regexp"
11
+ "slices"
11
12
"strings"
12
13
"time"
13
14
@@ -71,6 +72,7 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro
71
72
workerNodesOrdered []corev1.Node
72
73
workerNodesOrderedNames []string
73
74
advertisedPodsNodes []string
75
+ egressIPNodes []string
74
76
externalNodeName string
75
77
targetNamespace string
76
78
snifferNamespace string
@@ -153,6 +155,7 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro
153
155
o .Expect (len (workerNodesOrderedNames )).Should (o .BeNumerically (">" , 1 ))
154
156
externalNodeName = workerNodesOrderedNames [0 ]
155
157
advertisedPodsNodes = workerNodesOrderedNames [1 :]
158
+ egressIPNodes = workerNodesOrderedNames [1 :]
156
159
157
160
g .By ("Creating a project for the prober pod" )
158
161
// Create a target project and assign source and target namespace
@@ -208,9 +211,8 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro
208
211
spawnProberSendEgressIPTrafficCheckLogs (oc , snifferNamespace , probePodName , routeName , targetProtocol , v4ExternalIP , serverPort , numberOfRequestsToSend , numberOfRequestsToSend , packetSnifferDaemonSet , v4PodIPSet )
209
212
}
210
213
if clusterIPFamily == DualStack || clusterIPFamily == IPv6 {
211
- // [TODO] enable IPv6 test once OCPBUGS-52194 is fixed
212
- // g.By("sending to IPv6 external host")
213
- // spawnProberSendEgressIPTrafficCheckLogs(oc, snifferNamespace, probePodName, routeName, targetProtocol, v6ExternalIP, serverPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, v6PodIPSet)
214
+ g .By ("sending to IPv6 external host" )
215
+ spawnProberSendEgressIPTrafficCheckLogs (oc , snifferNamespace , probePodName , routeName , targetProtocol , v6ExternalIP , serverPort , numberOfRequestsToSend , numberOfRequestsToSend , packetSnifferDaemonSet , v6PodIPSet )
214
216
}
215
217
})
216
218
@@ -229,10 +231,10 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro
229
231
230
232
if clusterIPFamily == DualStack || clusterIPFamily == IPv6 {
231
233
// [TODO] enable IPv6 test once OCPBUGS-52194 is fixed
232
- // g.By("checking the external host to pod traffic works for IPv6")
233
- // for podIP := range v6PodIPSet {
234
- // checkExternalResponse(oc, proberPod, podIP, v6ExternalIP, serverPort, packetSnifferDaemonSet, targetProtocol)
235
- // }
234
+ g .By ("checking the external host to pod traffic works for IPv6" )
235
+ for podIP := range v6PodIPSet {
236
+ checkExternalResponse (oc , proberPod , podIP , v6ExternalIP , serverPort , packetSnifferDaemonSet , targetProtocol )
237
+ }
236
238
}
237
239
})
238
240
})
@@ -349,9 +351,313 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:RouteAdvertisements][Feature:Ro
349
351
}
350
352
})
351
353
})
354
+
355
+ g .Context ("[EgressIP][apigroup:user.openshift.io][apigroup:security.openshift.io]" , func () {
356
+ var err error
357
+ var egressIPYamlPath , egressIPObjectName string
358
+
359
+ g .BeforeEach (func () {
360
+ egressIPYamlPath = tmpDirBGP + "/" + egressIPYaml
361
+
362
+ g .By ("Setup packet sniffer at nodes" )
363
+ packetSnifferDaemonSet , err = setupPacketSniffer (oc , clientset , snifferNamespace , egressIPNodes , networkPlugin )
364
+ o .Expect (err ).NotTo (o .HaveOccurred ())
365
+
366
+ g .By ("Setting the EgressIP nodes as EgressIP assignable" )
367
+ for _ , node := range egressIPNodes {
368
+ _ , err = runOcWithRetry (oc .AsAdmin (), "label" , "node" , node , "k8s.ovn.org/egress-assignable=" )
369
+ o .Expect (err ).NotTo (o .HaveOccurred ())
370
+ }
371
+ })
372
+
373
+ g .AfterEach (func () {
374
+ g .By ("Deleting the EgressIP object if it exists for OVN Kubernetes" )
375
+ if _ , err := os .Stat (egressIPYamlPath ); err == nil {
376
+ _ , _ = runOcWithRetry (oc .AsAdmin (), "delete" , "-f" , tmpDirBGP + "/" + egressIPYaml )
377
+ }
378
+
379
+ g .By ("Removing the EgressIP assignable annotation for OVN Kubernetes" )
380
+ for _ , nodeName := range egressIPNodes {
381
+ _ , _ = runOcWithRetry (oc .AsAdmin (), "label" , "node" , nodeName , "k8s.ovn.org/egress-assignable-" )
382
+ }
383
+ })
384
+
385
+ g .Context ("Advertising egressIP for default network" , func () {
386
+ g .BeforeEach (func () {
387
+ egressIPObjectName = targetNamespace
388
+
389
+ g .By ("Turn on the BGP advertisement of EgressIPs" )
390
+ _ , err = runOcWithRetry (oc .AsAdmin (), "patch" , "ra" , "default" , "--type=merge" , `-p={"spec":{"advertisements":["EgressIP","PodNetwork"]}}` )
391
+ o .Expect (err ).NotTo (o .HaveOccurred ())
392
+
393
+ g .By ("Ensure the RouteAdvertisements is accepted" )
394
+ waitForRouteAdvertisements (oc , "default" )
395
+
396
+ g .By ("Makes sure the FRR configuration is generated for each node" )
397
+ for _ , nodeName := range workerNodesOrderedNames {
398
+ frr , err := getGeneratedFrrConfigurationForNode (oc , nodeName , "default" )
399
+ o .Expect (err ).NotTo (o .HaveOccurred ())
400
+ o .Expect (frr ).NotTo (o .BeNil ())
401
+ }
402
+ })
403
+
404
+ g .AfterEach (func () {
405
+ g .By ("Turn off the BGP advertisement of EgressIPs" )
406
+ _ , err := runOcWithRetry (oc .AsAdmin (), "patch" , "ra" , "default" , "--type=merge" , `-p={"spec":{"advertisements":["PodNetwork"]}}` )
407
+ o .Expect (err ).NotTo (o .HaveOccurred ())
408
+ })
409
+
410
+ g .DescribeTable ("pods should have the assigned EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io]" ,
411
+ func (ipFamily IPFamily , externalIP string ) {
412
+ if clusterIPFamily != ipFamily && clusterIPFamily != DualStack {
413
+ skipper .Skipf ("Skipping test for IPFamily: %s" , ipFamily )
414
+ return
415
+ }
416
+ g .By ("Selecte EgressIP set for the test" )
417
+ egressIPSet , newEgressIPSet := getEgressIPSet (ipFamily , egressIPNodes )
418
+
419
+ g .By ("Deploy the test pods" )
420
+ deployName , routeName , podList , err = setupTestDeployment (oc , clientset , targetNamespace , advertisedPodsNodes )
421
+ o .Expect (err ).NotTo (o .HaveOccurred ())
422
+ o .Expect (len (podList .Items )).To (o .Equal (len (advertisedPodsNodes )))
423
+
424
+ numberOfRequestsToSend := 10
425
+ // Run this twice to make sure that repeated EgressIP creation, update and deletion works.
426
+ for i := 0 ; i < 2 ; i ++ {
427
+ g .By ("Creating the EgressIP object" )
428
+ ovnKubernetesCreateEgressIPObject (oc , egressIPYamlPath , egressIPObjectName , targetNamespace , "" , egressIPSet )
429
+
430
+ g .By ("Applying the EgressIP object" )
431
+ applyEgressIPObject (oc , nil , egressIPYamlPath , targetNamespace , egressIPSet , egressUpdateTimeout )
432
+
433
+ g .By ("Makes sure the EgressIP is advertised by FRR" )
434
+ for eip , nodeName := range egressIPSet {
435
+ o .Expect (nodeName ).ShouldNot (o .BeEmpty ())
436
+ o .Eventually (func () bool {
437
+ return isRouteToEgressIPPresent (oc , eip , "default" , nodeName )
438
+ }).WithTimeout (timeOut ).WithPolling (interval ).Should (o .BeTrue ())
439
+ }
440
+
441
+ g .By (fmt .Sprintf ("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen" , numberOfRequestsToSend , egressIPSet ))
442
+ spawnProberSendEgressIPTrafficCheckLogs (oc , snifferNamespace , probePodName , routeName , targetProtocol , externalIP , serverPort , numberOfRequestsToSend , numberOfRequestsToSend , packetSnifferDaemonSet , egressIPSet )
443
+
444
+ g .By ("Updating the EgressIP object" )
445
+ ovnKubernetesCreateEgressIPObject (oc , egressIPYamlPath , egressIPObjectName , targetNamespace , "" , newEgressIPSet )
446
+
447
+ g .By ("Applying the updated EgressIP object" )
448
+ applyEgressIPObject (oc , nil , egressIPYamlPath , targetNamespace , newEgressIPSet , egressUpdateTimeout )
449
+
450
+ g .By ("Makes sure the updated EgressIP is advertised by FRR " )
451
+ for eip , nodeName := range newEgressIPSet {
452
+ o .Expect (nodeName ).ShouldNot (o .BeEmpty ())
453
+ o .Eventually (func () bool {
454
+ return isRouteToEgressIPPresent (oc , eip , "default" , nodeName )
455
+ }).WithTimeout (timeOut ).WithPolling (interval ).Should (o .BeTrue ())
456
+ }
457
+
458
+ g .By (fmt .Sprintf ("Sending requests from prober and making sure that %d requests with search string and updated EgressIPs %v were seen" , numberOfRequestsToSend , newEgressIPSet ))
459
+ spawnProberSendEgressIPTrafficCheckLogs (oc , snifferNamespace , probePodName , routeName , targetProtocol , externalIP , serverPort , numberOfRequestsToSend , numberOfRequestsToSend , packetSnifferDaemonSet , newEgressIPSet )
460
+
461
+ g .By ("Deleting the EgressIP object" )
462
+ // Use cascading foreground deletion to make sure that the EgressIP object and its dependencies are gone.
463
+ _ , err = runOcWithRetry (oc .AsAdmin (), "delete" , "egressip" , egressIPObjectName , "--cascade=foreground" )
464
+ o .Expect (err ).NotTo (o .HaveOccurred ())
465
+
466
+ g .By ("Makes sure the EgressIP is not advertised by FRR" )
467
+ for eip , nodeName := range newEgressIPSet {
468
+ o .Eventually (func () bool {
469
+ return isRouteToEgressIPPresent (oc , eip , "default" , nodeName )
470
+ }).WithTimeout (timeOut ).WithPolling (interval ).Should (o .BeFalse ())
471
+ }
472
+
473
+ g .By (fmt .Sprintf ("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen" , 0 , newEgressIPSet ))
474
+ spawnProberSendEgressIPTrafficCheckLogs (oc , snifferNamespace , probePodName , routeName , targetProtocol , externalIP , serverPort , numberOfRequestsToSend , 0 , packetSnifferDaemonSet , newEgressIPSet )
475
+ }
476
+ },
477
+ g .Entry ("IPv4" , IPv4 , v4ExternalIP ),
478
+ g .Entry ("IPv6" , IPv6 , v6ExternalIP ),
479
+ )
480
+ })
481
+
482
+ g .Context ("Advertising egressIP for user defined network" , func () {
483
+ g .BeforeEach (func () {
484
+ g .By ("Create a namespace with UDPN" )
485
+ ns , err := f .CreateNamespace (context .TODO (), f .BaseName , map [string ]string {
486
+ "e2e-framework" : f .BaseName ,
487
+ RequiredUDNNamespaceLabel : "" ,
488
+ })
489
+ o .Expect (err ).NotTo (o .HaveOccurred ())
490
+ err = udnWaitForOpenShift (oc , ns .Name )
491
+ o .Expect (err ).NotTo (o .HaveOccurred ())
492
+ targetNamespace = ns .Name
493
+ f .Namespace = ns
494
+ egressIPObjectName = targetNamespace
495
+
496
+ g .By ("Creating a cluster user defined network" )
497
+ nc := & networkAttachmentConfigParams {
498
+ name : cudnName ,
499
+ topology : "layer3" ,
500
+ role : "primary" ,
501
+ namespace : targetNamespace ,
502
+ }
503
+ nc .cidr = correctCIDRFamily (oc , userDefinedNetworkIPv4Subnet , userDefinedNetworkIPv6Subnet )
504
+ cudnManifest := generateClusterUserDefinedNetworkManifest (nc )
505
+ cleanup , err := createManifest (targetNamespace , cudnManifest )
506
+
507
+ o .Expect (err ).NotTo (o .HaveOccurred ())
508
+ o .Eventually (clusterUserDefinedNetworkReadyFunc (oc .AdminDynamicClient (), cudnName ), 60 * time .Second , time .Second ).Should (o .Succeed ())
509
+ g .By ("Labeling the UDN for advertisement" )
510
+ _ , err = runOcWithRetry (oc .AsAdmin (), "label" , "clusteruserdefinednetworks" , "-n" , targetNamespace , cudnName , "advertise=true" )
511
+ o .Expect (err ).NotTo (o .HaveOccurred ())
512
+
513
+ g .By ("Create the route advertisement for UDN" )
514
+ raManifest := newRouteAdvertisementsManifest (cudnName , true , true )
515
+ err = applyManifest (targetNamespace , raManifest )
516
+ o .Expect (err ).NotTo (o .HaveOccurred ())
517
+
518
+ g .By ("Ensure the RouteAdvertisements is accepted" )
519
+ waitForRouteAdvertisements (oc , cudnName )
520
+
521
+ g .DeferCleanup (func () {
522
+ runOcWithRetry (oc .AsAdmin (), "delete" , "deploy" , deployName )
523
+ runOcWithRetry (oc .AsAdmin (), "delete" , "ra" , cudnName )
524
+ runOcWithRetry (oc .AsAdmin (), "delete" , "pod" , "--all" )
525
+ runOcWithRetry (oc .AsAdmin (), "delete" , "clusteruserdefinednetwork" , cudnName )
526
+ cleanup ()
527
+ })
528
+ })
529
+
530
+ g .AfterEach (func () {
531
+ })
532
+
533
+ g .DescribeTable ("UDN pods should have the assigned EgressIPs and EgressIPs can be created, updated and deleted [apigroup:route.openshift.io]" ,
534
+ func (ipFamily IPFamily , externalIP string ) {
535
+ if clusterIPFamily != ipFamily && clusterIPFamily != DualStack {
536
+ skipper .Skipf ("Skipping test for IPFamily: %s" , ipFamily )
537
+ return
538
+ }
539
+
540
+ g .By ("Selecte EgressIP set for the test" )
541
+ egressIPSet , newEgressIPSet := getEgressIPSet (ipFamily , egressIPNodes )
542
+
543
+ g .By ("Deploy the test pods" )
544
+ deployName , routeName , podList , err = setupTestDeployment (oc , clientset , targetNamespace , advertisedPodsNodes )
545
+ o .Expect (err ).NotTo (o .HaveOccurred ())
546
+ o .Expect (len (podList .Items )).To (o .Equal (len (advertisedPodsNodes )))
547
+
548
+ numberOfRequestsToSend := 10
549
+ // Run this twice to make sure that repeated EgressIP creation and deletion works.
550
+ for i := 0 ; i < 2 ; i ++ {
551
+ g .By ("Creating the EgressIP object" )
552
+ ovnKubernetesCreateEgressIPObject (oc , egressIPYamlPath , egressIPObjectName , targetNamespace , "" , egressIPSet )
553
+
554
+ g .By ("Applying the EgressIP object" )
555
+ applyEgressIPObject (oc , nil , egressIPYamlPath , targetNamespace , egressIPSet , egressUpdateTimeout )
556
+
557
+ g .By ("Makes sure the EgressIP is advertised by FRR" )
558
+ for eip , nodeName := range egressIPSet {
559
+ o .Expect (nodeName ).ShouldNot (o .BeEmpty ())
560
+ o .Eventually (func () bool {
561
+ return isRouteToEgressIPPresent (oc , eip , cudnName , nodeName )
562
+ }).WithTimeout (timeOut ).WithPolling (interval ).Should (o .BeTrue ())
563
+ }
564
+
565
+ svcUrl := fmt .Sprintf ("%s-0-service:%d" , targetNamespace , serverPort )
566
+ g .By (fmt .Sprintf ("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen" , numberOfRequestsToSend , egressIPSet ))
567
+ spawnProberSendEgressIPTrafficCheckLogs (oc , targetNamespace , probePodName , svcUrl , targetProtocol , externalIP , serverPort , numberOfRequestsToSend , numberOfRequestsToSend , packetSnifferDaemonSet , egressIPSet )
568
+
569
+ g .By ("Updating the EgressIP object" )
570
+ ovnKubernetesCreateEgressIPObject (oc , egressIPYamlPath , egressIPObjectName , targetNamespace , "" , newEgressIPSet )
571
+
572
+ g .By ("Applying the updated EgressIP object" )
573
+ applyEgressIPObject (oc , nil , egressIPYamlPath , targetNamespace , newEgressIPSet , egressUpdateTimeout )
574
+
575
+ g .By ("Makes sure the updated EgressIP is advertised by FRR " )
576
+ for eip , nodeName := range newEgressIPSet {
577
+ o .Expect (nodeName ).ShouldNot (o .BeEmpty ())
578
+ o .Eventually (func () bool {
579
+ return isRouteToEgressIPPresent (oc , eip , cudnName , nodeName )
580
+ }).WithTimeout (timeOut ).WithPolling (interval ).Should (o .BeTrue ())
581
+ }
582
+
583
+ g .By (fmt .Sprintf ("Sending requests from prober and making sure that %d requests with search string and updated EgressIPs %v were seen" , numberOfRequestsToSend , newEgressIPSet ))
584
+ spawnProberSendEgressIPTrafficCheckLogs (oc , targetNamespace , probePodName , svcUrl , targetProtocol , externalIP , serverPort , numberOfRequestsToSend , numberOfRequestsToSend , packetSnifferDaemonSet , newEgressIPSet )
585
+
586
+ g .By ("Deleting the EgressIP object" )
587
+ // Use cascading foreground deletion to make sure that the EgressIP object and its dependencies are gone.
588
+ _ , err = runOcWithRetry (oc .AsAdmin (), "delete" , "egressip" , egressIPObjectName , "--cascade=foreground" )
589
+ o .Expect (err ).NotTo (o .HaveOccurred ())
590
+
591
+ g .By ("Makes sure the EgressIP is not advertised by FRR" )
592
+ for eip , nodeName := range newEgressIPSet {
593
+ o .Eventually (func () bool {
594
+ return isRouteToEgressIPPresent (oc , eip , cudnName , nodeName )
595
+ }).WithTimeout (timeOut ).WithPolling (interval ).Should (o .BeFalse ())
596
+ }
597
+
598
+ g .By (fmt .Sprintf ("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen" , 0 , newEgressIPSet ))
599
+ spawnProberSendEgressIPTrafficCheckLogs (oc , targetNamespace , probePodName , svcUrl , targetProtocol , externalIP , serverPort , numberOfRequestsToSend , 0 , packetSnifferDaemonSet , newEgressIPSet )
600
+ }
601
+ },
602
+ g .Entry ("IPv4" , IPv4 , v4ExternalIP ),
603
+ g .Entry ("IPv6" , IPv6 , v6ExternalIP ),
604
+ )
605
+ })
606
+ })
352
607
})
353
608
})
354
609
610
+ func getEgressIPSet (ipFamily IPFamily , eipNodes []string ) (map [string ]string , map [string ]string ) {
611
+ egressIPSet := make (map [string ]string )
612
+ newEgressIPSet := make (map [string ]string )
613
+ for i := range eipNodes {
614
+ switch ipFamily {
615
+ case IPv4 :
616
+ eip := fmt .Sprintf ("192.168.111.%d" , i + 30 )
617
+ egressIPSet [eip ] = ""
618
+ neip := fmt .Sprintf ("192.168.111.%d" , i + 40 )
619
+ newEgressIPSet [neip ] = ""
620
+ case IPv6 :
621
+ eip := fmt .Sprintf ("fd2e:6f44:5dd8:c956::%d" , i + 30 )
622
+ egressIPSet [eip ] = ""
623
+ neip := fmt .Sprintf ("fd2e:6f44:5dd8:c956::%d" , i + 40 )
624
+ newEgressIPSet [neip ] = ""
625
+ default :
626
+ o .Expect (false ).To (o .BeTrue ())
627
+ }
628
+ }
629
+ return egressIPSet , newEgressIPSet
630
+ }
631
+
632
+ // isRouteToEgressIPPresent checks that routes to the egress IPs are being advertised by FRR.
633
+ func isRouteToEgressIPPresent (oc * exutil.CLI , eip , netName , nodeName string ) bool {
634
+ advertised := false
635
+ frr , err := getGeneratedFrrConfigurationForNode (oc , nodeName , netName )
636
+ if err != nil && err .Error () == "FRR configuration for node " + nodeName + " not found" {
637
+ return advertised
638
+ }
639
+ o .Expect (err ).NotTo (o .HaveOccurred ())
640
+ if len (frr .Spec .BGP .Routers ) == 0 {
641
+ return advertised
642
+ }
643
+
644
+ // Parse IP to determine if it's IPv4 or IPv6
645
+ ip := net .ParseIP (eip )
646
+ o .Expect (ip ).NotTo (o .BeNil ())
647
+
648
+ var prefix string
649
+ if ip .To4 () != nil {
650
+ prefix = fmt .Sprintf ("%s/32" , eip )
651
+ } else {
652
+ prefix = fmt .Sprintf ("%s/128" , eip )
653
+ }
654
+
655
+ if slices .Contains (frr .Spec .BGP .Routers [0 ].Prefixes , prefix ) {
656
+ advertised = true
657
+ }
658
+ return advertised
659
+ }
660
+
355
661
// getRouteAdvertisements uses the dynamic admin client to return a pointer to
356
662
// an existing RouteAdvertisements, or error.
357
663
func getRouteAdvertisements (oc * exutil.CLI , name string ) (* ratypes.RouteAdvertisements , error ) {
0 commit comments