diff --git a/docs/tutorial-ssl-backendset.md b/docs/tutorial-ssl-backendset.md new file mode 100644 index 0000000000..f0aa5ee6d9 --- /dev/null +++ b/docs/tutorial-ssl-backendset.md @@ -0,0 +1,105 @@ +# Tutorial + +This example will show you how to use the CCM to create a load balancer with SSL +termination for BackendSets. + +### Load balancer with SSL termination for BackendSets example + +When you create a service with --type=LoadBalancer a OCI load balancer will be +created. + +The example below will create an NGINX deployment and expose it via a load +balancer serving http on port 80, and https on 443. Note that the service +**type** is set to **LoadBalancer**. + +```yaml +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: nginx-deployment +spec: + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 +--- +kind: Service +apiVersion: v1 +metadata: + name: nginx-service + annotations: + service.beta.kubernetes.io/oci-load-balancer-ssl-ports: "443" + service.beta.kubernetes.io/oci-load-balancer-tls-secret: ssl-certificate-secret + service.beta.kubernetes.io/oci-load-balancer-tls-backendset-secret: ssl-certificate-secret +spec: + selector: + app: nginx + type: LoadBalancer + ports: + - name: http + port: 80 + targetPort: 80 + - name: https + port: 443 + targetPort: 80 +``` + +First, the required Secret needs to be created in Kubernetes. For the purposes +of this example, we will create a self-signed certificate. However, in +production you would most likely use a public certificate signed by a +certificate authority. + +Below is an example of a secret configuration file required to be uploaded as a Kubernetes +generic Secret. The CA certificate, the public certificate and the private key need to be base64 encoded: + +***Note: Certificates for BackendSets require a CA certificate to be provided.*** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: ssl-certificate-secret +type: Opaque +data: + ca.crt: LS0tLS1CRUdJTiBDRV(...) + tls.crt: LS0tLS1CRUdJTi(...) + tls.key: LS0tLS1CRUdJTi(...) +``` + +``` +kubectl create -f ssl-certificate-secret.yaml +``` + +Create the service: + +``` +$ kubectl create -f manifests/demo/nginx-demo-svc-ssl.yaml +``` + +Watch the service and await a public IP address. This will be the load balancer +IP which you can use to connect to your service. + +``` +$ kubectl get svc --watch +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +nginx-service 10.96.97.137 129.213.12.174 80:30274/TCP 5m +``` + +You can now access your service via the provisioned load balancer using either +http or https: + +``` +curl http://129.213.12.174 +curl --insecure https://129.213.12.174 +``` + +Note: The `--insecure` flag above is only required due to our use of self-signed +certificates in this example. diff --git a/pkg/oci/client/load_balancer.go b/pkg/oci/client/load_balancer.go index b06a755d4c..b897fe6c74 100644 --- a/pkg/oci/client/load_balancer.go +++ b/pkg/oci/client/load_balancer.go @@ -35,7 +35,7 @@ type LoadBalancerInterface interface { DeleteLoadBalancer(ctx context.Context, id string) (string, error) GetCertificateByName(ctx context.Context, lbID, name string) (*loadbalancer.Certificate, error) - CreateCertificate(ctx context.Context, lbID, certificate, key string) (string, error) + CreateCertificate(ctx context.Context, lbID string, cert loadbalancer.CertificateDetails) (string, error) CreateBackendSet(ctx context.Context, lbID, name string, details loadbalancer.BackendSetDetails) (string, error) UpdateBackendSet(ctx context.Context, lbID, name string, details loadbalancer.BackendSetDetails) (string, error) @@ -150,18 +150,19 @@ func (c *client) GetCertificateByName(ctx context.Context, lbID, name string) (* return nil, errors.WithStack(errNotFound) } -func (c *client) CreateCertificate(ctx context.Context, lbID, certificate, key string) (string, error) { +func (c *client) CreateCertificate(ctx context.Context, lbID string, cert loadbalancer.CertificateDetails) (string, error) { if !c.rateLimiter.Writer.TryAccept() { return "", RateLimitError(true, "CreateCertificate") } - // TODO(apryde): We currently don't have a mechanism for supplying - // CreateCertificateDetails.CaCertificate. resp, err := c.loadbalancer.CreateCertificate(ctx, loadbalancer.CreateCertificateRequest{ LoadBalancerId: &lbID, CreateCertificateDetails: loadbalancer.CreateCertificateDetails{ - PublicCertificate: &certificate, - PrivateKey: &key, + CertificateName: cert.CertificateName, + CaCertificate: cert.CaCertificate, + PublicCertificate: cert.PublicCertificate, + PrivateKey: cert.PrivateKey, + Passphrase: cert.Passphrase, }, }) incRequestCounter(err, createVerb, certificateResource) diff --git a/pkg/oci/load_balancer.go b/pkg/oci/load_balancer.go index 9eac454e54..002e8b80d6 100644 --- a/pkg/oci/load_balancer.go +++ b/pkg/oci/load_balancer.go @@ -63,6 +63,12 @@ const ( // See: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls ServiceAnnotationLoadBalancerTLSSecret = "service.beta.kubernetes.io/oci-load-balancer-tls-secret" + // ServiceAnnotationLoadBalancerTLSBackendSetSecret is a Service annotation for + // specifying the generic secret to install on the load balancer listeners which + // have SSL enabled. + // See: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls + ServiceAnnotationLoadBalancerTLSBackendSetSecret = "service.beta.kubernetes.io/oci-load-balancer-tls-backendset-secret" + // ServiceAnnotationLoadBalancerConnectionIdleTimeout is the annotation used // on the service to specify the idle connection timeout. ServiceAnnotationLoadBalancerConnectionIdleTimeout = "service.beta.kubernetes.io/oci-load-balancer-connection-idle-timeout" @@ -89,9 +95,10 @@ const ( // Fallback value if annotation on service is not set lbDefaultShape = "100Mbps" - lbNodesHealthCheckPath = "/healthz" - lbNodesHealthCheckPort = k8sports.ProxyHealthzPort - lbNodesHealthCheckProto = "HTTP" + lbNodesHealthCheckPath = "/healthz" + lbNodesHealthCheckPort = k8sports.ProxyHealthzPort + lbNodesHealthCheckProtoHTTP = "HTTP" + lbNodesHealthCheckProtoTCP = "TCP" ) // GetLoadBalancer returns whether the specified load balancer exists, and if @@ -189,62 +196,48 @@ func getSubnetsForNodes(ctx context.Context, nodes []*v1.Node, client client.Int // readSSLSecret returns the certificate and private key from a Kubernetes TLS // private key Secret. -func (cp *CloudProvider) readSSLSecret(svc *v1.Service) (string, string, error) { - secretString, ok := svc.Annotations[ServiceAnnotationLoadBalancerTLSSecret] - if !ok { - return "", "", errors.Errorf("no %q annotation found", ServiceAnnotationLoadBalancerTLSSecret) - } - - ns, name := parseSecretString(secretString) - if ns == "" { - ns = svc.Namespace - } +func (cp *CloudProvider) readSSLSecret(ns, name string) (*certificateData, error) { secret, err := cp.kubeclient.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{}) if err != nil { - return "", "", err + return nil, err } - - var cert, key []byte - if cert, ok = secret.Data[sslCertificateFileName]; !ok { - return "", "", errors.Errorf("%s not found in secret %s/%s", sslCertificateFileName, ns, name) + var ok bool + var cacert, cert, key, pass []byte + cacert = secret.Data[SSLCAFileName] + if cert, ok = secret.Data[SSLCertificateFileName]; !ok { + return nil, errors.Errorf("%s not found in secret %s/%s", SSLCertificateFileName, ns, name) } - if key, ok = secret.Data[sslPrivateKeyFileName]; !ok { - return "", "", errors.Errorf("%s not found in secret %s/%s", sslPrivateKeyFileName, ns, name) + if key, ok = secret.Data[SSLPrivateKeyFileName]; !ok { + return nil, errors.Errorf("%s not found in secret %s/%s", SSLPrivateKeyFileName, ns, name) } - - return string(cert), string(key), nil + pass = secret.Data[SSLPassphrase] + return &certificateData{CACert: cacert, PublicCert: cert, PrivateKey: key, Passphrase: pass}, nil } // ensureSSLCertificate creates a OCI SSL certificate to the given load // balancer, if it doesn't already exist. -func (cp *CloudProvider) ensureSSLCertificate(ctx context.Context, lb *loadbalancer.LoadBalancer, spec *LBSpec) error { - name := spec.SSLConfig.Name - logger := cp.logger.With("loadBalancerID", *lb.Id, "certificateName", name) - _, err := cp.client.LoadBalancer().GetCertificateByName(ctx, *lb.Id, name) - if err == nil { - logger.Debug("Certificate already exists on load balancer. Nothing to do.") - return nil - } - if !client.IsNotFound(err) { - return err - } - - // Although we iterate here only one certificate is supported at the moment. +func (cp *CloudProvider) ensureSSLCertificates(ctx context.Context, lb *loadbalancer.LoadBalancer, spec *LBSpec) error { + logger := cp.logger.With("loadBalancerID", *lb.Id) + // Get all required certificates certs, err := spec.Certificates() if err != nil { return err } + for _, cert := range certs { - wrID, err := cp.client.LoadBalancer().CreateCertificate(ctx, *lb.Id, *cert.PublicCertificate, *cert.PrivateKey) - if err != nil { - return err - } - _, err = cp.client.LoadBalancer().AwaitWorkRequest(ctx, wrID) - if err != nil { - return err - } + if _, ok := lb.Certificates[*cert.CertificateName]; !ok { + logger = cp.logger.With("certificateName", *cert.CertificateName) + wrID, err := cp.client.LoadBalancer().CreateCertificate(ctx, *lb.Id, cert) + if err != nil { + return err + } + _, err = cp.client.LoadBalancer().AwaitWorkRequest(ctx, wrID) + if err != nil { + return err + } - logger.Info("Certificate created") + logger.Info("Certificate created") + } } return nil } @@ -323,16 +316,18 @@ func (cp *CloudProvider) EnsureLoadBalancer(ctx context.Context, clusterName str } exists := !client.IsNotFound(err) - var ssl *SSLConfig + var sslConfig *SSLConfig if requiresCertificate(service) { ports, err := getSSLEnabledPorts(service) if err != nil { return nil, err } - ssl = NewSSLConfig(lbName, ports, cp) + secretListenerString := service.Annotations[ServiceAnnotationLoadBalancerTLSSecret] + secretBackendSetString := service.Annotations[ServiceAnnotationLoadBalancerTLSBackendSetSecret] + sslConfig = NewSSLConfig(secretListenerString, secretBackendSetString, ports, cp) } subnets := []string{cp.config.LoadBalancer.Subnet1, cp.config.LoadBalancer.Subnet2} - spec, err := NewLBSpec(service, nodes, subnets, ssl, cp.securityListManagerFactory) + spec, err := NewLBSpec(service, nodes, subnets, sslConfig, cp.securityListManagerFactory) if err != nil { logger.With(zap.Error(err)).Error("Failed to derive LBSpec") return nil, err @@ -351,8 +346,8 @@ func (cp *CloudProvider) EnsureLoadBalancer(ctx context.Context, clusterName str // If the load balancer needs an SSL cert ensure it is present. if requiresCertificate(service) { - if err := cp.ensureSSLCertificate(ctx, lb, spec); err != nil { - return nil, errors.Wrap(err, "ensuring ssl certificate") + if err := cp.ensureSSLCertificates(ctx, lb, spec); err != nil { + return nil, errors.Wrap(err, "ensuring ssl certificates") } } diff --git a/pkg/oci/load_balancer_spec.go b/pkg/oci/load_balancer_spec.go index 41f5b40cb7..32c0f7ef89 100644 --- a/pkg/oci/load_balancer_spec.go +++ b/pkg/oci/load_balancer_spec.go @@ -30,20 +30,31 @@ import ( "github.com/oracle/oci-cloud-controller-manager/pkg/oci/util" ) +// certificateData is a structure containing the data about a K8S secret required +// to store SSL information required for BackendSets and Listeners +type certificateData struct { + Name string + CACert []byte + PublicCert []byte + PrivateKey []byte + Passphrase []byte +} + type sslSecretReader interface { - readSSLSecret(svc *v1.Service) (cert, key string, err error) + readSSLSecret(ns, name string) (sslSecret *certificateData, err error) } type noopSSLSecretReader struct{} -func (ssr noopSSLSecretReader) readSSLSecret(svc *v1.Service) (cert, key string, err error) { - return "", "", nil +func (ssr noopSSLSecretReader) readSSLSecret(ns, name string) (sslSecret *certificateData, err error) { + return nil, nil } // SSLConfig is a description of a SSL certificate. type SSLConfig struct { - Name string - Ports sets.Int + Ports sets.Int + ListenerSSLSecretName string + BackendSetSSLSecretName string sslSecretReader } @@ -54,14 +65,15 @@ func requiresCertificate(svc *v1.Service) bool { } // NewSSLConfig constructs a new SSLConfig. -func NewSSLConfig(name string, ports []int, ssr sslSecretReader) *SSLConfig { +func NewSSLConfig(listenerSecretName, backendSetSecretName string, ports []int, ssr sslSecretReader) *SSLConfig { if ssr == nil { ssr = noopSSLSecretReader{} } return &SSLConfig{ - Name: name, - Ports: sets.NewInt(ports...), - sslSecretReader: ssr, + Ports: sets.NewInt(ports...), + ListenerSSLSecretName: listenerSecretName, + BackendSetSSLSecretName: backendSetSecretName, + sslSecretReader: ssr, } } @@ -85,7 +97,7 @@ type LBSpec struct { } // NewLBSpec creates a LB Spec from a Kubernetes service and a slice of nodes. -func NewLBSpec(svc *v1.Service, nodes []*v1.Node, defaultSubnets []string, sslCfg *SSLConfig, secListFactory securityListManagerFactory) (*LBSpec, error) { +func NewLBSpec(svc *v1.Service, nodes []*v1.Node, defaultSubnets []string, sslConfig *SSLConfig, secListFactory securityListManagerFactory) (*LBSpec, error) { if len(defaultSubnets) != 2 { return nil, errors.New("default subnets incorrectly configured") } @@ -131,7 +143,7 @@ func NewLBSpec(svc *v1.Service, nodes []*v1.Node, defaultSubnets []string, sslCf } } - listeners, err := getListeners(svc, sslCfg) + listeners, err := getListeners(svc, sslConfig) if err != nil { return nil, err } @@ -142,10 +154,10 @@ func NewLBSpec(svc *v1.Service, nodes []*v1.Node, defaultSubnets []string, sslCf Internal: internal, Subnets: subnets, Listeners: listeners, - BackendSets: getBackendSets(svc, nodes), + BackendSets: getBackendSets(svc, nodes, sslConfig), Ports: getPorts(svc), - SSLConfig: sslCfg, + SSLConfig: sslConfig, SourceCIDRs: sourceCIDRs, service: svc, @@ -161,21 +173,28 @@ func (s *LBSpec) Certificates() (map[string]loadbalancer.CertificateDetails, err if s.SSLConfig == nil { return certs, nil } - - cert, key, err := s.SSLConfig.readSSLSecret(s.service) - if err != nil { - return nil, errors.Wrap(err, "reading SSL Secret") + secrets := make([]string, 0, 2) + if s.SSLConfig.ListenerSSLSecretName != "" { + secrets = append(secrets, s.SSLConfig.ListenerSSLSecretName) } - if cert == "" || key == "" { - return certs, nil + if s.SSLConfig.BackendSetSSLSecretName != "" { + secrets = append(secrets, s.SSLConfig.BackendSetSSLSecretName) } - certs[s.SSLConfig.Name] = loadbalancer.CertificateDetails{ - CertificateName: &s.SSLConfig.Name, - PublicCertificate: &cert, - PrivateKey: &key, - } + for idx, name := range secrets { + cert, err := s.SSLConfig.readSSLSecret(s.service.Namespace, name) + if err != nil { + return nil, errors.Wrap(err, "reading SSL BackendSet Secret") + } + certs[name] = loadbalancer.CertificateDetails{ + CertificateName: &secrets[idx], + CaCertificate: common.String(string(cert.CACert)), + PublicCertificate: common.String(string(cert.PublicCert)), + PrivateKey: common.String(string(cert.PrivateKey)), + Passphrase: common.String(string(cert.Passphrase)), + } + } return certs, nil } @@ -225,7 +244,7 @@ func getPorts(svc *v1.Service) map[string]portSpec { ports[name] = portSpec{ BackendPort: int(servicePort.NodePort), ListenerPort: int(servicePort.Port), - HealthCheckerPort: *getHealthChecker(svc).Port, + HealthCheckerPort: *getHealthChecker(nil, int(servicePort.Port), svc).Port, } } @@ -244,43 +263,53 @@ func getBackends(nodes []*v1.Node, nodePort int32) []loadbalancer.BackendDetails return backends } -func getBackendSets(svc *v1.Service, nodes []*v1.Node) map[string]loadbalancer.BackendSetDetails { +func getBackendSets(svc *v1.Service, nodes []*v1.Node, sslCfg *SSLConfig) map[string]loadbalancer.BackendSetDetails { backendSets := make(map[string]loadbalancer.BackendSetDetails) for _, servicePort := range svc.Spec.Ports { name := getBackendSetName(string(servicePort.Protocol), int(servicePort.Port)) + port := int(servicePort.Port) + var secretName string + if sslCfg != nil && len(sslCfg.BackendSetSSLSecretName) != 0 { + secretName = sslCfg.BackendSetSSLSecretName + } backendSets[name] = loadbalancer.BackendSetDetails{ - Policy: common.String(DefaultLoadBalancerPolicy), - Backends: getBackends(nodes, servicePort.NodePort), - HealthChecker: getHealthChecker(svc), + Policy: common.String(DefaultLoadBalancerPolicy), + Backends: getBackends(nodes, servicePort.NodePort), + HealthChecker: getHealthChecker(sslCfg, port, svc), + SslConfiguration: getSSLConfiguration(sslCfg, secretName, port), } - } return backendSets } -func getHealthChecker(svc *v1.Service) *loadbalancer.HealthCheckerDetails { - path, port := apiservice.GetServiceHealthCheckPathPort(svc) - if path != "" { +func getHealthChecker(cfg *SSLConfig, port int, svc *v1.Service) *loadbalancer.HealthCheckerDetails { + // If the health-check has SSL enabled use TCP rather than HTTP. + protocol := lbNodesHealthCheckProtoHTTP + if cfg != nil && cfg.Ports.Has(port) { + protocol = lbNodesHealthCheckProtoTCP + } + checkPath, checkPort := apiservice.GetServiceHealthCheckPathPort(svc) + if checkPath != "" { return &loadbalancer.HealthCheckerDetails{ - Protocol: common.String(lbNodesHealthCheckProto), - UrlPath: &path, - Port: common.Int(int(port)), + Protocol: &protocol, + UrlPath: &checkPath, + Port: common.Int(int(checkPort)), } } return &loadbalancer.HealthCheckerDetails{ - Protocol: common.String(lbNodesHealthCheckProto), + Protocol: &protocol, UrlPath: common.String(lbNodesHealthCheckPath), Port: common.Int(lbNodesHealthCheckPort), } } -func getSSLConfiguration(cfg *SSLConfig, port int) *loadbalancer.SslConfigurationDetails { - if cfg == nil || !cfg.Ports.Has(port) { +func getSSLConfiguration(cfg *SSLConfig, name string, port int) *loadbalancer.SslConfigurationDetails { + if cfg == nil || !cfg.Ports.Has(port) || len(name) == 0 { return nil } return &loadbalancer.SslConfigurationDetails{ - CertificateName: &cfg.Name, + CertificateName: &name, VerifyDepth: common.Int(0), VerifyPeerCertificate: common.Bool(false), } @@ -318,7 +347,11 @@ func getListeners(svc *v1.Service, sslCfg *SSLConfig) (map[string]loadbalancer.L } } port := int(servicePort.Port) - sslConfiguration := getSSLConfiguration(sslCfg, port) + var secretName string + if sslCfg != nil && len(sslCfg.ListenerSSLSecretName) != 0 { + secretName = sslCfg.ListenerSSLSecretName + } + sslConfiguration := getSSLConfiguration(sslCfg, secretName, port) name := getListenerName(protocol, port, sslConfiguration) listener := loadbalancer.ListenerDetails{ diff --git a/pkg/oci/load_balancer_spec_test.go b/pkg/oci/load_balancer_spec_test.go index 26be6824ff..e4f426e9a9 100644 --- a/pkg/oci/load_balancer_spec_test.go +++ b/pkg/oci/load_balancer_spec_test.go @@ -20,7 +20,6 @@ import ( "github.com/oracle/oci-go-sdk/common" "github.com/oracle/oci-go-sdk/loadbalancer" - "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/oci/load_balancer_util.go b/pkg/oci/load_balancer_util.go index 0b87b55aea..4c5c28f38b 100644 --- a/pkg/oci/load_balancer_util.go +++ b/pkg/oci/load_balancer_util.go @@ -29,8 +29,14 @@ import ( ) const ( - sslCertificateFileName = "tls.crt" - sslPrivateKeyFileName = "tls.key" + // SSLCAFileName is a key name for ca data in the secrets config. + SSLCAFileName = "ca.crt" + // SSLCertificateFileName is a key name for certificate data in the secrets config. + SSLCertificateFileName = "tls.crt" + // SSLPrivateKeyFileName is a key name for cartificate private key in the secrets config. + SSLPrivateKeyFileName = "tls.key" + // SSLPassphrase is a key name for certificate passphrase in the secrets config. + SSLPassphrase = "passphrase" ) const lbNamePrefixEnvVar = "LOAD_BALANCER_PREFIX" diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index e07ed66f57..c237e5ec89 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -84,6 +84,7 @@ type Framework struct { SkipNamespaceCreation bool // Whether to skip creating a namespace Namespace *v1.Namespace // Every test has at least one namespace unless creation is skipped + Secret *v1.Secret // Every test has at least one namespace unless creation is skipped namespacesToDelete []*v1.Namespace // Some tests have more than one. // To make sure that this framework cleans up after itself, no matter what, diff --git a/test/e2e/framework/networking_util.go b/test/e2e/framework/networking_util.go index 20ba93e700..097c23288f 100644 --- a/test/e2e/framework/networking_util.go +++ b/test/e2e/framework/networking_util.go @@ -36,6 +36,7 @@ import ( const ( EndpointHttpPort = 8080 + EndpointHttpsPort = 443 EndpointUdpPort = 8081 TestContainerHttpPort = 8080 ClusterHttpPort = 80 @@ -84,26 +85,29 @@ func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Re return client.Get(url) } -func TestReachableHTTP(ip string, port int, request string, expect string) (bool, error) { - return TestReachableHTTPWithContent(ip, port, request, expect, nil) +func TestReachableHTTP(secure bool, ip string, port int, request string, expect string) (bool, error) { + return TestReachableHTTPWithContent(secure, ip, port, request, expect, nil) } -func TestReachableHTTPWithRetriableErrorCodes(ip string, port int, request string, expect string, retriableErrCodes []int) (bool, error) { - return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, nil, retriableErrCodes, time.Second*5) +func TestReachableHTTPWithRetriableErrorCodes(secure bool, ip string, port int, request string, expect string, retriableErrCodes []int) (bool, error) { + return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(secure, ip, port, request, expect, nil, retriableErrCodes, time.Second*5) } -func TestReachableHTTPWithContent(ip string, port int, request string, expect string, content *bytes.Buffer) (bool, error) { - return TestReachableHTTPWithContentTimeout(ip, port, request, expect, content, 5*time.Second) +func TestReachableHTTPWithContent(secure bool, ip string, port int, request string, expect string, content *bytes.Buffer) (bool, error) { + return TestReachableHTTPWithContentTimeout(secure, ip, port, request, expect, content, 5*time.Second) } -func TestReachableHTTPWithContentTimeout(ip string, port int, request string, expect string, content *bytes.Buffer, timeout time.Duration) (bool, error) { - return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, content, []int{}, timeout) +func TestReachableHTTPWithContentTimeout(secure bool, ip string, port int, request string, expect string, content *bytes.Buffer, timeout time.Duration) (bool, error) { + return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(secure, ip, port, request, expect, content, []int{}, timeout) } -func TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip string, port int, request string, expect string, content *bytes.Buffer, retriableErrCodes []int, timeout time.Duration) (bool, error) { +func TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(secure bool, ip string, port int, request string, expect string, content *bytes.Buffer, retriableErrCodes []int, timeout time.Duration) (bool, error) { ipPort := net.JoinHostPort(ip, strconv.Itoa(port)) url := fmt.Sprintf("http://%s%s", ipPort, request) + if secure { + url = fmt.Sprintf("https://%s%s", ipPort, request) + } if ip == "" { Failf("Got empty IP for reachability check (%s)", url) return false, nil @@ -271,7 +275,7 @@ func TestHitNodesFromOutsideWithCount(externalIP string, httpPort int32, timeout count := 0 condition := func() (bool, error) { var respBody bytes.Buffer - reached, err := TestReachableHTTPWithContentTimeout(externalIP, int(httpPort), "/hostname", "", &respBody, + reached, err := TestReachableHTTPWithContentTimeout(false, externalIP, int(httpPort), "/hostname", "", &respBody, 1*time.Second) if err != nil || !reached { return false, nil diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index 4880ffa246..25adc63427 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -740,13 +740,13 @@ func (j *ServiceTestJig) waitForPodsReady(namespace string, pods []string) error return nil } -func (j *ServiceTestJig) TestReachableHTTP(host string, port int, timeout time.Duration) { - j.TestReachableHTTPWithRetriableErrorCodes(host, port, []int{}, timeout) +func (j *ServiceTestJig) TestReachableHTTP(secure bool, host string, port int, timeout time.Duration) { + j.TestReachableHTTPWithRetriableErrorCodes(secure, host, port, []int{}, timeout) } -func (j *ServiceTestJig) TestReachableHTTPWithRetriableErrorCodes(host string, port int, retriableErrCodes []int, timeout time.Duration) { +func (j *ServiceTestJig) TestReachableHTTPWithRetriableErrorCodes(secure bool, host string, port int, retriableErrCodes []int, timeout time.Duration) { if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { - return TestReachableHTTPWithRetriableErrorCodes(host, port, "/echo?msg=hello", "hello", retriableErrCodes) + return TestReachableHTTPWithRetriableErrorCodes(secure, host, port, "/echo?msg=hello", "hello", retriableErrCodes) }); err != nil { if err == wait.ErrWaitTimeout { Failf("Could not reach HTTP service through %v:%v after %v", host, port, timeout) @@ -774,12 +774,12 @@ func (j *ServiceTestJig) TestNotReachableUDP(host string, port int, timeout time } } -func (j *ServiceTestJig) GetHTTPContent(host string, port int, timeout time.Duration, url string) bytes.Buffer { +func (j *ServiceTestJig) GetHTTPContent(secure bool, host string, port int, timeout time.Duration, url string) bytes.Buffer { var body bytes.Buffer var err error if pollErr := wait.PollImmediate(Poll, timeout, func() (bool, error) { var result bool - result, err = TestReachableHTTPWithContent(host, port, url, "", &body) + result, err = TestReachableHTTPWithContent(secure, host, port, url, "", &body) if err != nil { Logf("Error hitting %v:%v%v, retrying: %v", host, port, url, err) return false, nil diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 8b22532dc0..1867d12399 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -71,6 +71,90 @@ const ( var ( BusyBoxImage = "busybox" + // SSL CAData is a CA certificate not being used anywhere else; it only is utilised to check the load + // balancer SSL connection during tests + SSLCAData = `-----BEGIN CERTIFICATE----- +MIIE5jCCAs6gAwIBAgIBATANBgkqhkiG9w0BAQsFADATMREwDwYDVQQDEwhzc2wt +dGVzdDAeFw0xODA5MDYxNTQzNTlaFw0yMDAzMDYxNTQzNTlaMBMxETAPBgNVBAMT +CHNzbC10ZXN0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuZkDWwTh +2/Oq00ViJw8yXUqAppGgH81QaHbSkGp1W4TbpieFufheWYkxjBksqynph/RD1JX/ +J+k7WLha6QcAEFsDB+b9b+suQS8MPoLxE7Rw95p6PW9TlXYOh7SQSjOOFSA/SDKh +ewt7PIaGtvn0Qmda8mOTC/Bi/XbnF+ZdM39/WsHYGdqHfEsT40L8Kz3BvQaaLL6Q +ocJEMKmBCVUN6bLhc8GhFhM1mOn9fvM5wKNzLV07VeEWogH3Zz48qiCK1gosE3c8 +oHynHxbEtnsjawIeYsy01A/i434i1dksqA0bRaNJu5YCQAf5o9JhyPJb16neUag8 +wEuczF7x5Q0V/G5oU7k4Hh1Zcl1kIG7hHvgcpV6uX6+i5H/0Pmxa+9TVP2x3axwH +0XDZyis8aH0aLLndtH/EqIUJfYq9Q3N871cBYueY9/rqL5sinXgrH0Ud5WKJeREL +ilFWwFthafj6DMe00S1lKCNIGtRLKt9B9GGhuRdRyOdw4ZWOrxYc66uoeyaQowvi +iNJnjZ0yQruk3uLkTOMrKo6xXkJKuHyoIWHL6uNFMcVBLBeA9ERdjV36wCJx+zh7 +y2DGydQjSbcGKTYHQr1z21E1n+QRMEnqxQ5cW1kWxhYecfrOja4zh/iGrXOA9ffO +BEHn0Fb/fA/Pijr0t7mvH4VdaICx8G3X2TUCAwEAAaNFMEMwDgYDVR0PAQH/BAQD +AgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFDEbBw9aIWhE9MHxc+Cm +4xHNnmu8MA0GCSqGSIb3DQEBCwUAA4ICAQCLtbLoPS/jLWA1zgbw2wQA2rcA7kpB +D3iKKzA+6FnJlWSbMi9u4Ui/e3Z98P0KLafHZ+E/0jzMM4gq/yEMbWTP4KzY5aCG +sRJMtawSf3hCxK/1uRmdFbEiygQh2BzjmDb2TYXOeLMMSif+5LzXHD9Vzq4Rnzbf +Qqf0TnKh7EHIjptkEWgsIX0afBPfXSs0EzVQ8dJNioy/NqkuUoStCdB2OaKcFt/9 +YoCuaWf/4hGvOY59WpUI1i1MXuig2s3o1cpkwu4jB08la2kPFCinv4Cpsv4s5BvY +zsxPFeMFjjMNLaZnp/LruvbvMq08D58Qvqsn4vpxrYPiRPNvpO3Dh1OeA6KppTNV +15NtOG0H0K2Wc7iJCMV+vnGBIWUZ9lh57RcKb+pf9E5F5vWBO2MdKUugrUQ60NH9 +cubjZ7moO9OgjNLqe24PdEtx3CK933bwvTn2fT5pR1yYVOGVI0vbmFDgwD9Tg6+X +Q3xRbI67X9qqGWj1Nxl+WAOCFdzvQWaI6+f4Byhf1xwQ07CFSykUA1JpurXfemkT +R1F56uxXc5pWQFYU38r8ROKfrzD857kIGoMQhHbf3KBTavR1fCQdun3991wfkKsd +7Quf316BO4qqn1G3e5vIaRLAqePgvSdm7CRzOtja1RXvqA/o52/9zVourhZinK0x +UV5xGY0MDAKR0g== +-----END CERTIFICATE-----` + SSLCertificateData = `-----BEGIN CERTIFICATE----- +MIIEJzCCAg+gAwIBAgIRAPtkjNA2AzZwFxKV/en8uDIwDQYJKoZIhvcNAQELBQAw +EzERMA8GA1UEAxMIc3NsLXRlc3QwHhcNMTgwOTA2MTU0NTI1WhcNMjAwMzA2MTU0 +MzU4WjAYMRYwFAYDVQQDEw1zc2wtdGVzdC1jZXJ0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAlFq7tzGl8ljlzyrddPI5WWs6BfHTwi0WOkhWCOMT2jzo +qfjZJ4WXCKIn0V3fmsG9MkmDkCRmoBwrwY8axUqwKopF5MQp39rF1W2L7EYaMPiH +Ry71gPwbfyzWfRcOCG0rlNuyZOkWh9AYKJQAlDPDZrQ9X6fzj7/FhqJLPqHbSgBT +PCQ7GnPUITYEGcND7d0t4KcojwJHog+7WRuChwq28MVM5gOoONl2eI/C3q6CeXSK +dlXBjZbBj5pGV+7lqDoSvzSrJYhucruNKGAT3loMvJicRih5+kS1auOOeWLhqupM +UyLgDSHTLoWzS+gY29UiK3Au/qc72igby/xHaMsZ3QIDAQABo3EwbzAOBgNVHQ8B +Af8EBAMCA7gwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQW +BBSvDgYQAAyY+0OKaCspaI/157dYJjAfBgNVHSMEGDAWgBQxGwcPWiFoRPTB8XPg +puMRzZ5rvDANBgkqhkiG9w0BAQsFAAOCAgEAH7wRpPqp+M6UCeH9PYuKYml0eqFc +mhV67Qu/wDUYT8yqy3IoM5fKVyarxK2PbiHZxv+BMDnDMvIFi+8o2DxQC/2mvkgE +1tQsMIDPphFN/UIAEnBEdjFrnyQljsbjYpazcGr77oM0B5gjzPHc6pyCrgfjfN7g +Qb8qgTv33feLagGuZ8wV+LQcQxRL+hBqWnl6NivFeKwEHLG/pMKJmsxufTtVDVXt +GPJBoi6jnOKluIyU+XZAbRl+Iz5lUeEjr24D0+JJlOleqate4uIm8L11AEjQ5NcX +lgz1Fkr/V5mZIS/CnzxWNdJpCVV39VT5KLAYzXNsnE3kutwW0ERvANz2MqmOW3oW +hayT/qUlqvJQwghMWPRwAbU0l9TQvSzLWS9t/wLjb9G9xz4UxIrVqUHJ4BhAQsFF +pYQ6heCqW0mkmgN3Tz/3pgUk9DZvrDPVB6r431geeoFtRfeJMjGtbkjvNufo9Xp9 +sQDNlhZ5X8ZFRKjU4VedYqYrcfya8sjAJnvdktzLozgU41Op0zkykNUogPFK1Wma +330oZwh9LsEJ3bQnNOQd5P5hERmP4CXDl43qdmnjInp9Fn1kzJFWKzho68tmbdo7 +9rm7AXKM1hM5GcTxKcalQQa92UQCCPbbjQKNUj/dpGHHpNQtEYXlLcvq6OOBfegU +VcU/2vioT6/IBgM= +-----END CERTIFICATE-----` + SSLPrivateData = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAlFq7tzGl8ljlzyrddPI5WWs6BfHTwi0WOkhWCOMT2jzoqfjZ +J4WXCKIn0V3fmsG9MkmDkCRmoBwrwY8axUqwKopF5MQp39rF1W2L7EYaMPiHRy71 +gPwbfyzWfRcOCG0rlNuyZOkWh9AYKJQAlDPDZrQ9X6fzj7/FhqJLPqHbSgBTPCQ7 +GnPUITYEGcND7d0t4KcojwJHog+7WRuChwq28MVM5gOoONl2eI/C3q6CeXSKdlXB +jZbBj5pGV+7lqDoSvzSrJYhucruNKGAT3loMvJicRih5+kS1auOOeWLhqupMUyLg +DSHTLoWzS+gY29UiK3Au/qc72igby/xHaMsZ3QIDAQABAoIBAQCIZTs83Jqzy+PN +qp1dDe6INbttB78nQYi0IRlopk30UhefXjlYysvnlh6BsYsJn6Iw/8HD3pO0FPJt +zQJR7a82DVbhUzEkrOFCrqr8mh8ucvD9E0GIS0GElVbGP1IZkTWGcM79VAfw6fOs +Kxa1Kzo1zPS4Y5Pcz/XtKH2BohIZX/jEBM2bLaX1l+/kwZbFwUjSByy6EPl3cDMG +6FeYMm7mtyR/nS7LvtyQf+d8BwEEoD23OX+TUNdf7Bc72C+5Xm8NNFaI6b/ocy1M +9h5Y2KGg8opYbonY8EM6cqeUBYoxgd4azOxKehyIs6EqKfi3EkGHMWr9BAapqWKL +KaVytYiJAoGBAMWyb1ZA4MGJCtKt42XHs7QlYwpzfmY4JSTYrF7C6kk2+55HR6Wf +GR6u788Rs5t7WP1dss2LUMF8ZIYf8O2Xx5cJ75UrT6Yfj2A2mAjOPNEGqZyrJ3M8 +q4bqEqwyQizsxwNuWTfE8uT5Sv3EU1THXM0nd1xc5GgfrRzxpylFYglzAoGBAMAb +EqNv8mzZHU9/xN17BmED/rmWQJeHBXAFXG0n0U4VzjkZLJthE00ko1fFvyt/+dc2 +ZctUHBjMktwqtGRmJHVr7FlZAjesaEDCfNGq7GfHn9cNTFqwMpNHZCsl+YydHehV +2WglfKSgNd29cOluD+WgPi+bsIrHqN801B3flrtvAoGAHxkLZHD+KdNpzWwm+gqo +3OyIoMs8Fc49IYenzZwxiGTKvcIOpiHZrLbt6A+rxghoHirQBn8kq9rqYSLDHkyi +y8J3Wmqes84BGqxby/7NGEBJC+jsYrcncCh/2XBqcnCoeYSxKhGj95qzTinExfA6 +S9cqcm/2Sd65t3TXy6krW7sCgYAUoVOpFZaVNeO969y6ZEHVHVa3m3koTKm60/iP +CF2j4xeYbimqLgyiljKsdGIJS98Ky5627TtvNlj3J+bjeUylB4gEOFjSncM8YSaC +ZbaplniF4bm3a4Ci7GHHeHaKMT1K/B0y4AO4sjPWskdz6gvu/vxupGubG8H45nV5 +F8/aJQKBgHwkkWYhysyy1F+SyWOkcpFa5ORYapYvVQhnVjgMB8SSs8KZquMV5ixX +imlfs6WRsA4mr20dMC9a4Lb51lET2XPvc5Tef/kESpGfc7Zbgx5/Sm5Z2IpNlhgp +rTMMobnwm2ixgvgHMC4Uq9U0HVM9hyQ7l88m5QD+02Qr0vV/ZCMi +-----END RSA PRIVATE KEY-----` + SSLPassphrase = "" ) func nowStamp() string { @@ -526,6 +610,8 @@ func getRuntimeObjectForKind(c clientset.Interface, kind schema.GroupKind, ns, n switch kind { case api.Kind("ReplicationController"): return c.CoreV1().ReplicationControllers(ns).Get(name, metav1.GetOptions{}) + case api.Kind("Secrets"): + return c.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{}) case extensionsinternal.Kind("ReplicaSet"), appsinternal.Kind("ReplicaSet"): return c.ExtensionsV1beta1().ReplicaSets(ns).Get(name, metav1.GetOptions{}) case extensionsinternal.Kind("Deployment"), appsinternal.Kind("Deployment"): @@ -543,6 +629,8 @@ func deleteResource(c clientset.Interface, kind schema.GroupKind, ns, name strin switch kind { case api.Kind("ReplicationController"): return c.CoreV1().ReplicationControllers(ns).Delete(name, deleteOption) + case api.Kind("Secrets"): + return c.CoreV1().Secrets(ns).Delete(name, deleteOption) case extensionsinternal.Kind("ReplicaSet"), appsinternal.Kind("ReplicaSet"): return c.ExtensionsV1beta1().ReplicaSets(ns).Delete(name, deleteOption) case extensionsinternal.Kind("Deployment"), appsinternal.Kind("Deployment"): diff --git a/test/e2e/load_balancer.go b/test/e2e/load_balancer.go index 2868db3c3e..1346faddfe 100644 --- a/test/e2e/load_balancer.go +++ b/test/e2e/load_balancer.go @@ -85,11 +85,11 @@ var _ = Describe("Service [Slow]", func() { if f.NodePortTest { By("hitting the TCP service's NodePort") - jig.TestReachableHTTP(nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) + jig.TestReachableHTTP(false, nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) } By("hitting the TCP service's LoadBalancer") - jig.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) + jig.TestReachableHTTP(false, tcpIngressIP, svcPort, loadBalancerLagTimeout) // Change the services' node ports. @@ -117,21 +117,20 @@ var _ = Describe("Service [Slow]", func() { if f.NodePortTest { By("hitting the TCP service's new NodePort") - jig.TestReachableHTTP(nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) + jig.TestReachableHTTP(false, nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) By("checking the old TCP NodePort is closed") jig.TestNotReachableHTTP(nodeIP, tcpNodePortOld, framework.KubeProxyLagTimeout) } By("hitting the TCP service's LoadBalancer") - jig.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) + jig.TestReachableHTTP(false, tcpIngressIP, svcPort, loadBalancerLagTimeout) // Change the services' main ports. By("changing the TCP service's port") tcpService = jig.UpdateServiceOrFail(ns, tcpService.Name, func(s *v1.Service) { s.Spec.Ports[0].Port++ - framework.Logf("Updating port to: %d", tcpNodePort) }) jig.SanityCheckService(tcpService, v1.ServiceTypeLoadBalancer) svcPortOld := svcPort @@ -147,14 +146,13 @@ var _ = Describe("Service [Slow]", func() { } framework.Logf("service port (TCP): %d", svcPort) - if f.NodePortTest { By("hitting the TCP service's NodePort") - jig.TestReachableHTTP(nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) + jig.TestReachableHTTP(false, nodeIP, tcpNodePort, framework.KubeProxyLagTimeout) } By("hitting the TCP service's LoadBalancer") - jig.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) // this may actually recreate the LB + jig.TestReachableHTTP(false, tcpIngressIP, svcPort, loadBalancerCreateTimeout) // this may actually recreate the LB // Change the services back to ClusterIP. @@ -244,7 +242,7 @@ var _ = Describe("ESIPP [Slow]", func() { for n, publicIP := range ips { // Make sure the loadbalancer picked up the health check change. // Confirm traffic can reach backend through LB before checking healthcheck nodeport. - jig.TestReachableHTTP(ingressIP, svcTCPPort, framework.KubeProxyLagTimeout) + jig.TestReachableHTTP(false, ingressIP, svcTCPPort, framework.KubeProxyLagTimeout) expectedSuccess := nodes.Items[n].Name == endpointNodeName port := strconv.Itoa(healthCheckNodePort) ipPort := net.JoinHostPort(publicIP, port) @@ -304,3 +302,323 @@ var _ = Describe("ESIPP [Slow]", func() { } }) }) + +var _ = Describe("End to end TLS", func() { + f := framework.NewDefaultFramework("service") + + It("should be possible to create and mutate a Service type:LoadBalancer [Canary]", func() { + serviceName := "e2e-tls-lb-test" + ns := f.Namespace.Name + + jig := framework.NewServiceTestJig(f.ClientSet, serviceName) + + sslSecretName := "ssl-certificate-secret" + _, err := f.ClientSet.CoreV1().Secrets(ns).Create(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: sslSecretName, + }, + Data: map[string][]byte{ + cloudprovider.SSLCAFileName: []byte(framework.SSLCAData), + cloudprovider.SSLCertificateFileName: []byte(framework.SSLCertificateData), + cloudprovider.SSLPrivateKeyFileName: []byte(framework.SSLPrivateData), + cloudprovider.SSLPassphrase: []byte(framework.SSLPassphrase), + }, + }) + framework.ExpectNoError(err) + loadBalancerCreateTimeout := framework.LoadBalancerCreateTimeoutDefault + if nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet); len(nodes.Items) > framework.LargeClusterMinNodesNumber { + loadBalancerCreateTimeout = framework.LoadBalancerCreateTimeoutLarge + } + + requestedIP := "" + + tcpService := jig.CreateTCPServiceOrFail(ns, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeLoadBalancer + s.Spec.LoadBalancerIP = requestedIP + s.Spec.Ports = []v1.ServicePort{v1.ServicePort{Name: "http", Port: 80, TargetPort: intstr.FromInt(80)}, + v1.ServicePort{Name: "https", Port: 443, TargetPort: intstr.FromInt(80)}} + s.ObjectMeta.Annotations = map[string]string{cloudprovider.ServiceAnnotationLoadBalancerSSLPorts: "443", + cloudprovider.ServiceAnnotationLoadBalancerTLSSecret: sslSecretName, + cloudprovider.ServiceAnnotationLoadBalancerTLSBackendSetSecret: sslSecretName} + + }) + + svcPort := int(tcpService.Spec.Ports[0].Port) + + By("creating a pod to be part of the TCP service " + serviceName) + jig.RunOrFail(ns, nil) + + // TODO(apryde): Test UDP service. OCI does not currently support this. + + By("waiting for the TCP service to have a load balancer") + // Wait for the load balancer to be created asynchronously + tcpService = jig.WaitForLoadBalancerOrFail(ns, tcpService.Name, loadBalancerCreateTimeout) + jig.SanityCheckService(tcpService, v1.ServiceTypeLoadBalancer) + + tcpNodePort := int(tcpService.Spec.Ports[0].NodePort) + framework.Logf("TCP node port: %d", tcpNodePort) + + if requestedIP != "" && framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP { + framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) + } + tcpIngressIP := framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) + framework.Logf("TCP load balancer: %s", tcpIngressIP) + + By("changing TCP service back to type=ClusterIP") + tcpService = jig.UpdateServiceOrFail(ns, tcpService.Name, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeClusterIP + s.Spec.Ports[0].NodePort = 0 + s.Spec.Ports[1].NodePort = 0 + }) + + // Wait for the load balancer to be destroyed asynchronously + tcpService = jig.WaitForLoadBalancerDestroyOrFail(ns, tcpService.Name, tcpIngressIP, svcPort, loadBalancerCreateTimeout) + jig.SanityCheckService(tcpService, v1.ServiceTypeClusterIP) + + err = f.ClientSet.CoreV1().Secrets(ns).Delete(sslSecretName, nil) + framework.ExpectNoError(err) + }) +}) + +var _ = Describe("BackendSet only enabled TLS", func() { + f := framework.NewDefaultFramework("backendset-service") + + It("should be possible to create and mutate a Service type:LoadBalancer [Canary]", func() { + serviceName := "backendset-tls-lb-test" + ns := f.Namespace.Name + + jig := framework.NewServiceTestJig(f.ClientSet, serviceName) + + sslSecretName := "ssl-certificate-secret" + _, err := f.ClientSet.CoreV1().Secrets(ns).Create(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: sslSecretName, + }, + Data: map[string][]byte{ + cloudprovider.SSLCAFileName: []byte(framework.SSLCAData), + cloudprovider.SSLCertificateFileName: []byte(framework.SSLCertificateData), + cloudprovider.SSLPrivateKeyFileName: []byte(framework.SSLPrivateData), + cloudprovider.SSLPassphrase: []byte(framework.SSLPassphrase), + }, + }) + framework.ExpectNoError(err) + loadBalancerCreateTimeout := framework.LoadBalancerCreateTimeoutDefault + if nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet); len(nodes.Items) > framework.LargeClusterMinNodesNumber { + loadBalancerCreateTimeout = framework.LoadBalancerCreateTimeoutLarge + } + + requestedIP := "" + + tcpService := jig.CreateTCPServiceOrFail(ns, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeLoadBalancer + s.Spec.LoadBalancerIP = requestedIP + s.Spec.Ports = []v1.ServicePort{v1.ServicePort{Name: "http", Port: 80, TargetPort: intstr.FromInt(80)}, + v1.ServicePort{Name: "https", Port: 443, TargetPort: intstr.FromInt(80)}} + s.ObjectMeta.Annotations = map[string]string{cloudprovider.ServiceAnnotationLoadBalancerSSLPorts: "443", + cloudprovider.ServiceAnnotationLoadBalancerTLSBackendSetSecret: sslSecretName} + + }) + + svcPort := int(tcpService.Spec.Ports[0].Port) + + By("creating a pod to be part of the TCP service " + serviceName) + jig.RunOrFail(ns, nil) + + By("waiting for the TCP service to have a load balancer") + // Wait for the load balancer to be created asynchronously + tcpService = jig.WaitForLoadBalancerOrFail(ns, tcpService.Name, loadBalancerCreateTimeout) + jig.SanityCheckService(tcpService, v1.ServiceTypeLoadBalancer) + + tcpNodePort := int(tcpService.Spec.Ports[0].NodePort) + framework.Logf("TCP node port: %d", tcpNodePort) + + if requestedIP != "" && framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP { + framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) + } + tcpIngressIP := framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) + framework.Logf("TCP load balancer: %s", tcpIngressIP) + + By("changing TCP service back to type=ClusterIP") + tcpService = jig.UpdateServiceOrFail(ns, tcpService.Name, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeClusterIP + s.Spec.Ports[0].NodePort = 0 + s.Spec.Ports[1].NodePort = 0 + }) + + // Wait for the load balancer to be destroyed asynchronously + tcpService = jig.WaitForLoadBalancerDestroyOrFail(ns, tcpService.Name, tcpIngressIP, svcPort, loadBalancerCreateTimeout) + jig.SanityCheckService(tcpService, v1.ServiceTypeClusterIP) + + err = f.ClientSet.CoreV1().Secrets(ns).Delete(sslSecretName, nil) + framework.ExpectNoError(err) + }) +}) + +var _ = Describe("Listener only enabled TLS", func() { + f := framework.NewDefaultFramework("listener-service") + + It("should be possible to create and mutate a Service type:LoadBalancer [Canary]", func() { + serviceName := "listener-tls-lb-test" + ns := f.Namespace.Name + + jig := framework.NewServiceTestJig(f.ClientSet, serviceName) + + sslSecretName := "ssl-certificate-secret" + _, err := f.ClientSet.CoreV1().Secrets(ns).Create(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: sslSecretName, + }, + Data: map[string][]byte{ + cloudprovider.SSLCAFileName: []byte(framework.SSLCAData), + cloudprovider.SSLCertificateFileName: []byte(framework.SSLCertificateData), + cloudprovider.SSLPrivateKeyFileName: []byte(framework.SSLPrivateData), + cloudprovider.SSLPassphrase: []byte(framework.SSLPassphrase), + }, + }) + framework.ExpectNoError(err) + loadBalancerCreateTimeout := framework.LoadBalancerCreateTimeoutDefault + if nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet); len(nodes.Items) > framework.LargeClusterMinNodesNumber { + loadBalancerCreateTimeout = framework.LoadBalancerCreateTimeoutLarge + } + + requestedIP := "" + + tcpService := jig.CreateTCPServiceOrFail(ns, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeLoadBalancer + s.Spec.LoadBalancerIP = requestedIP + s.Spec.Ports = []v1.ServicePort{v1.ServicePort{Name: "http", Port: 80, TargetPort: intstr.FromInt(80)}, + v1.ServicePort{Name: "https", Port: 443, TargetPort: intstr.FromInt(80)}} + s.ObjectMeta.Annotations = map[string]string{cloudprovider.ServiceAnnotationLoadBalancerSSLPorts: "443", + cloudprovider.ServiceAnnotationLoadBalancerTLSSecret: sslSecretName} + + }) + + svcPort := int(tcpService.Spec.Ports[0].Port) + + By("creating a pod to be part of the TCP service " + serviceName) + jig.RunOrFail(ns, nil) + + By("waiting for the TCP service to have a load balancer") + // Wait for the load balancer to be created asynchronously + tcpService = jig.WaitForLoadBalancerOrFail(ns, tcpService.Name, loadBalancerCreateTimeout) + jig.SanityCheckService(tcpService, v1.ServiceTypeLoadBalancer) + + tcpNodePort := int(tcpService.Spec.Ports[0].NodePort) + framework.Logf("TCP node port: %d", tcpNodePort) + + if requestedIP != "" && framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP { + framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) + } + tcpIngressIP := framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) + framework.Logf("TCP load balancer: %s", tcpIngressIP) + + By("changing TCP service back to type=ClusterIP") + tcpService = jig.UpdateServiceOrFail(ns, tcpService.Name, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeClusterIP + s.Spec.Ports[0].NodePort = 0 + s.Spec.Ports[1].NodePort = 0 + }) + + // Wait for the load balancer to be destroyed asynchronously + tcpService = jig.WaitForLoadBalancerDestroyOrFail(ns, tcpService.Name, tcpIngressIP, svcPort, loadBalancerCreateTimeout) + jig.SanityCheckService(tcpService, v1.ServiceTypeClusterIP) + + err = f.ClientSet.CoreV1().Secrets(ns).Delete(sslSecretName, nil) + framework.ExpectNoError(err) + }) +}) + +var _ = Describe("End to end enabled TLS - different certificates", func() { + f := framework.NewDefaultFramework("e2e-diff-certs") + + It("should be possible to create and mutate a Service type:LoadBalancer [Canary]", func() { + serviceName := "e2e-diff-certs-service" + ns := f.Namespace.Name + + jig := framework.NewServiceTestJig(f.ClientSet, serviceName) + + sslListenerSecretName := "ssl-certificate-secret-lis" + sslBackendSetSecretName := "ssl-certificate-secret-backendset" + _, err := f.ClientSet.CoreV1().Secrets(ns).Create(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: sslListenerSecretName, + }, + Data: map[string][]byte{ + cloudprovider.SSLCAFileName: []byte(framework.SSLCAData), + cloudprovider.SSLCertificateFileName: []byte(framework.SSLCertificateData), + cloudprovider.SSLPrivateKeyFileName: []byte(framework.SSLPrivateData), + cloudprovider.SSLPassphrase: []byte(framework.SSLPassphrase), + }, + }) + framework.ExpectNoError(err) + _, err = f.ClientSet.CoreV1().Secrets(ns).Create(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: sslBackendSetSecretName, + }, + Data: map[string][]byte{ + cloudprovider.SSLCAFileName: []byte(framework.SSLCAData), + cloudprovider.SSLCertificateFileName: []byte(framework.SSLCertificateData), + cloudprovider.SSLPrivateKeyFileName: []byte(framework.SSLPrivateData), + cloudprovider.SSLPassphrase: []byte(framework.SSLPassphrase), + }, + }) + framework.ExpectNoError(err) + loadBalancerCreateTimeout := framework.LoadBalancerCreateTimeoutDefault + if nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet); len(nodes.Items) > framework.LargeClusterMinNodesNumber { + loadBalancerCreateTimeout = framework.LoadBalancerCreateTimeoutLarge + } + + requestedIP := "" + + tcpService := jig.CreateTCPServiceOrFail(ns, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeLoadBalancer + s.Spec.LoadBalancerIP = requestedIP + s.Spec.Ports = []v1.ServicePort{v1.ServicePort{Name: "http", Port: 80, TargetPort: intstr.FromInt(80)}, + v1.ServicePort{Name: "https", Port: 443, TargetPort: intstr.FromInt(80)}} + s.ObjectMeta.Annotations = map[string]string{cloudprovider.ServiceAnnotationLoadBalancerSSLPorts: "443", + cloudprovider.ServiceAnnotationLoadBalancerTLSSecret: sslListenerSecretName, + cloudprovider.ServiceAnnotationLoadBalancerTLSBackendSetSecret: sslBackendSetSecretName} + + }) + + svcPort := int(tcpService.Spec.Ports[0].Port) + + By("creating a pod to be part of the TCP service " + serviceName) + jig.RunOrFail(ns, nil) + + By("waiting for the TCP service to have a load balancer") + // Wait for the load balancer to be created asynchronously + tcpService = jig.WaitForLoadBalancerOrFail(ns, tcpService.Name, loadBalancerCreateTimeout) + jig.SanityCheckService(tcpService, v1.ServiceTypeLoadBalancer) + + tcpNodePort := int(tcpService.Spec.Ports[0].NodePort) + framework.Logf("TCP node port: %d", tcpNodePort) + + if requestedIP != "" && framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP { + framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) + } + tcpIngressIP := framework.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) + framework.Logf("TCP load balancer: %s", tcpIngressIP) + + By("changing TCP service back to type=ClusterIP") + tcpService = jig.UpdateServiceOrFail(ns, tcpService.Name, func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeClusterIP + s.Spec.Ports[0].NodePort = 0 + s.Spec.Ports[1].NodePort = 0 + }) + + // Wait for the load balancer to be destroyed asynchronously + tcpService = jig.WaitForLoadBalancerDestroyOrFail(ns, tcpService.Name, tcpIngressIP, svcPort, loadBalancerCreateTimeout) + jig.SanityCheckService(tcpService, v1.ServiceTypeClusterIP) + + err = f.ClientSet.CoreV1().Secrets(ns).Delete(sslListenerSecretName, nil) + framework.ExpectNoError(err) + err = f.ClientSet.CoreV1().Secrets(ns).Delete(sslBackendSetSecretName, nil) + framework.ExpectNoError(err) + }) +})