Port Service E2E tests from core

Gopkg.toml

@@ -36,10 +36,6 @@
   name = "github.com/spf13/pflag"
   version = "1.0.0"
 
-[[constraint]]
-  name = "gopkg.in/gcfg.v1"
-  version = "1.2.0"
-
 [[constraint]]
   branch = "release-1.8"
   name = "k8s.io/api"
@@ -63,3 +59,11 @@
 [[constraint]]
   branch = "v2"
   name = "gopkg.in/yaml.v2"
+
+[[constraint]]
+  name = "github.com/onsi/ginkgo"
+  version = "1.4.0"
+
+[[constraint]]
+  name = "github.com/onsi/gomega"
+  version = "1.3.0"

docs/development.md

@@ -82,6 +82,16 @@ $ OCI_CONFIG_FILE=cloud-provider.yaml \
    go test -timeout 45m -v ./test/integration/loadbalancer
 ```
 
+
+## Running the e2e tests
+
+The e2e test suite requires the version of the CCM under test to be installed in
+the cluster referenced via the `--kubeconfig` flag.
+
+```
+$ go test -timeout 45m -v ./test/e2e/ --kubeconfig="$HOME/.kube/config"
+```
+
 [1]: https://www.docker.com/
 [2]: https://github.com/golang/dep
 [3]: https://golang.org/

hack/boilerplate/boilerplate.py

@@ -141,6 +141,15 @@ def file_extension(filename):
     "hack/boilerplate/boilerplate_test.py",
     "hack/boilerplate/test",
     "hack/verify-boilerplate.sh",
+
+    "test/e2e/e2e_test.go",
+    "test/e2e/load_balancer.go",
+    "test/e2e/framework/cleanup.go",
+    "test/e2e/framework/framework.go",
+    "test/e2e/framework/service_util.go",
+    "test/e2e/framework/util.go",
+    "test/e2e/framework/networking_utils.go",
+    "test/e2e/framework/ginkgowrapper/wrapper.go",
 ]
 
 

test/e2e/README.md

@@ -0,0 +1,17 @@
+# E2E Tests
+
+These tests are adapted from the [Service test suite][1] in the Kubernetes core
+E2E tests.
+
+## Running
+
+
+```bash
+$ ginkgo -v -progress test/e2e -- --kubeconfig=${HOME}/.kube/config --delete-namespace=false
+```
+
+NOTE: Test suite will fail if executed behind a `$HTTP_PROXY` that returns a
+200 OK response upon failure to connect.
+
+
+[1]: https://github.com/kubernetes/kubernetes/blob/0cb15453dae92d8be66cf42e6c1b04e21a2d0fb6/test/e2e/network/service.go

test/e2e/e2e_test.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	"k8s.io/apiserver/pkg/util/logs"
+
+	"github.com/oracle/oci-cloud-controller-manager/test/e2e/framework"
+	"github.com/oracle/oci-cloud-controller-manager/test/e2e/framework/ginkgowrapper"
+)
+
+func TestE2E(t *testing.T) {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	gomega.RegisterFailHandler(ginkgowrapper.Fail)
+	ginkgo.RunSpecs(t, "CCM E2E suite")
+}
+
+var _ = ginkgo.SynchronizedAfterSuite(func() {
+	framework.Logf("Running AfterSuite actions on all node")
+	framework.RunCleanupActions()
+}, func() {})

test/e2e/framework/cleanup.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import "sync"
+
+type CleanupActionHandle *int
+
+var cleanupActionsLock sync.Mutex
+var cleanupActions = map[CleanupActionHandle]func(){}
+
+// AddCleanupAction installs a function that will be called in the event of the
+// whole test being terminated.  This allows arbitrary pieces of the overall
+// test to hook into SynchronizedAfterSuite().
+func AddCleanupAction(fn func()) CleanupActionHandle {
+	p := CleanupActionHandle(new(int))
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	cleanupActions[p] = fn
+	return p
+}
+
+// RemoveCleanupAction removes a function that was installed by
+// AddCleanupAction.
+func RemoveCleanupAction(p CleanupActionHandle) {
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	delete(cleanupActions, p)
+}
+
+// RunCleanupActions runs all functions installed by AddCleanupAction.  It does
+// not remove them (see RemoveCleanupAction) but it does run unlocked, so they
+// may remove themselves.
+func RunCleanupActions() {
+	list := []func(){}
+	func() {
+		cleanupActionsLock.Lock()
+		defer cleanupActionsLock.Unlock()
+		for _, fn := range cleanupActions {
+			list = append(list, fn)
+		}
+	}()
+	// Run unlocked.
+	for _, fn := range list {
+		fn()
+	}
+}

test/e2e/framework/framework.go

@@ -0,0 +1,211 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	wait "k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	clientcmd "k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+)
+
+const (
+	// Poll defines how regularly to poll kubernetes resources.
+	Poll = 2 * time.Second
+)
+
+// path to kubeconfig on disk.
+var (
+	kubeconfig      string
+	deleteNamespace bool
+)
+
+func init() {
+	flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to Kubeconfig file with authorization and master location information.")
+	flag.BoolVar(&deleteNamespace, "delete-namespace", true, "If true tests will delete namespace after completion. It is only designed to make debugging easier, DO NOT turn it off by default.")
+}
+
+// Framework is used in the execution of e2e tests.
+type Framework struct {
+	BaseName string
+
+	ClientSet         clientset.Interface
+	InternalClientset internalclientset.Interface
+
+	// Namespace in which test resources are created.
+	Namespace          *v1.Namespace
+	namespacesToDelete []*v1.Namespace // Some tests have more than one.
+
+	// To make sure that this framework cleans up after itself, no matter what,
+	// we install a Cleanup action before each test and clear it after.  If we
+	// should abort, the AfterSuite hook should run all Cleanup actions.
+	cleanupHandle CleanupActionHandle
+}
+
+// NewDefaultFramework constructs a new e2e test Framework with default options.
+func NewDefaultFramework(baseName string) *Framework {
+	return NewFramework(baseName, nil)
+}
+
+// NewFramework constructs a new e2e test Framework.
+func NewFramework(baseName string, client clientset.Interface) *Framework {
+	f := &Framework{
+		BaseName:  baseName,
+		ClientSet: client,
+	}
+
+	BeforeEach(f.BeforeEach)
+	AfterEach(f.AfterEach)
+
+	return f
+}
+
+// CreateNamespace creates a e2e test namespace.
+func (f *Framework) CreateNamespace(baseName string, labels map[string]string) (*v1.Namespace, error) {
+	if labels == nil {
+		labels = map[string]string{}
+	}
+
+	namespaceObj := &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: fmt.Sprintf("ccm-e2e-tests-%v-", baseName),
+			Namespace:    "",
+			Labels:       labels,
+		},
+		Status: v1.NamespaceStatus{},
+	}
+
+	// Be robust about making the namespace creation call.
+	var got *v1.Namespace
+	if err := wait.PollImmediate(Poll, 30*time.Second, func() (bool, error) {
+		var err error
+		got, err = f.ClientSet.CoreV1().Namespaces().Create(namespaceObj)
+		if err != nil {
+			Logf("Unexpected error while creating namespace: %v", err)
+			return false, nil
+		}
+		return true, nil
+	}); err != nil {
+		return nil, err
+	}
+
+	if got != nil {
+		f.namespacesToDelete = append(f.namespacesToDelete, got)
+	}
+
+	return got, nil
+}
+
+// DeleteNamespace deletes a given namespace and waits until its contents are
+// deleted.
+func (f *Framework) DeleteNamespace(namespace string, timeout time.Duration) error {
+	startTime := time.Now()
+	if err := f.ClientSet.CoreV1().Namespaces().Delete(namespace, nil); err != nil {
+		if apierrors.IsNotFound(err) {
+			Logf("Namespace %v was already deleted", namespace)
+			return nil
+		}
+		return err
+	}
+
+	// wait for namespace to delete or timeout.
+	err := wait.PollImmediate(Poll, timeout, func() (bool, error) {
+		if _, err := f.ClientSet.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{}); err != nil {
+			if apierrors.IsNotFound(err) {
+				return true, nil
+			}
+			Logf("Error while waiting for namespace to be terminated: %v", err)
+			return false, nil
+		}
+		return false, nil
+	})
+
+	// Namespace deletion timed out.
+	if err != nil {
+		return fmt.Errorf("namespace %v was not deleted with limit: %v", namespace, err)
+	}
+
+	Logf("namespace %v deletion completed in %s", namespace, time.Now().Sub(startTime))
+	return nil
+}
+
+// BeforeEach gets a client and makes a namespace.
+func (f *Framework) BeforeEach() {
+	// The fact that we need this feels like a bug in ginkgo.
+	// https://github.com/onsi/ginkgo/issues/222
+	f.cleanupHandle = AddCleanupAction(f.AfterEach)
+	if f.ClientSet == nil {
+		By("Creating a kubernetes client")
+		config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+		Expect(err).NotTo(HaveOccurred())
+		f.ClientSet, err = clientset.NewForConfig(config)
+		Expect(err).NotTo(HaveOccurred())
+		f.InternalClientset, err = internalclientset.NewForConfig(config)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	By("Building a namespace api object")
+	namespace, err := f.CreateNamespace(f.BaseName, map[string]string{
+		"e2e-framework": f.BaseName,
+	})
+	Expect(err).NotTo(HaveOccurred())
+
+	f.Namespace = namespace
+}
+
+// AfterEach deletes the namespace(s).
+func (f *Framework) AfterEach() {
+	RemoveCleanupAction(f.cleanupHandle)
+
+	// DeleteNamespace at the very end in defer, to avoid any expectation
+	// failures preventing deleting the namespace.
+	nsDeletionErrors := map[string]error{}
+
+	if deleteNamespace {
+		for _, ns := range f.namespacesToDelete {
+			By(fmt.Sprintf("Destroying namespace %q for this suite.", ns.Name))
+			if err := f.DeleteNamespace(ns.Name, 5*time.Minute); err != nil {
+				nsDeletionErrors[ns.Name] = err
+			}
+		}
+	}
+
+	// Paranoia -- prevent reuse!
+	f.Namespace = nil
+	f.ClientSet = nil
+	f.namespacesToDelete = nil
+
+	// if we had errors deleting, report them now.
+	if len(nsDeletionErrors) != 0 {
+		messages := []string{}
+		for namespaceKey, namespaceErr := range nsDeletionErrors {
+			messages = append(messages, fmt.Sprintf("Couldn't delete ns: %q: %s (%#v)", namespaceKey, namespaceErr, namespaceErr))
+		}
+		Failf(strings.Join(messages, ","))
+	}
+}