🌱 Envtest exports secure config, helpers for simplify users adding

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-logr/logr v0.4.0
 	github.com/go-logr/zapr v0.4.0
+	github.com/google/uuid v1.1.2
 	github.com/googleapis/gnostic v0.5.4 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/imdario/mergo v0.3.11 // indirect

pkg/envtest/envtest_test.go

@@ -23,11 +23,16 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	authv1 "k8s.io/api/authentication/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	kubeclient "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -845,4 +850,106 @@ var _ = Describe("Test", func() {
 			close(done)
 		}, 30)
 	})
+
+	Describe("SecureConfig", func() {
+
+		It("should be populated during envtest start", func(done Done) {
+			Expect(env.Config.CAData).To(BeNil())
+			Expect(env.Config.BearerToken).To(BeEmpty())
+
+			Expect(env.SecureConfig).NotTo(BeNil())
+			Expect(env.SecureConfig.BearerToken).NotTo(BeEmpty())
+			Expect(env.SecureConfig.CAData).NotTo(BeNil())
+			close(done)
+		})
+
+		It("client should work with credentials and authorize successfully", func(done Done) {
+			c, err := kubeclient.NewForConfig(env.SecureConfig)
+			Expect(err).NotTo(HaveOccurred())
+
+			tokenReview := &authv1.TokenReview{Spec: authv1.TokenReviewSpec{Token: env.SecureConfig.BearerToken}}
+			result, err := c.AuthenticationV1().TokenReviews().Create(context.TODO(), tokenReview, metav1.CreateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			user := result.Status.User
+			Expect(user.Username).To(Equal("kubeadmin"))
+			Expect(user.Groups).To(ContainElement("system:masters"))
+			Expect(user.Groups).To(ContainElement("system:authenticated"))
+			close(done)
+		})
+
+		It("should be refreshed after ApiServer restart", func(done Done) {
+			env := &Environment{}
+			_, err := env.Start()
+			Expect(err).NotTo(HaveOccurred())
+			copiedConfig := rest.CopyConfig(env.SecureConfig)
+			Expect(env.Stop()).To(Succeed())
+			_, err = env.Start()
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(copiedConfig).NotTo(Equal(env.SecureConfig.CAData))
+			Expect(copiedConfig).NotTo(Equal(env.SecureConfig.BearerToken))
+
+			close(done)
+		}, 30)
+	})
+
+	Describe("User related helper methods", func() {
+		It("should throw error on attempt to create user while apiserver running", func(done Done) {
+			_, err := env.AddAPIServerUser("test", []string{"test", "test2"})
+			Expect(err).Should(HaveOccurred())
+			Expect(err.Error()).Should(Equal("users cannot be added while ControlPlane running"))
+			close(done)
+		})
+
+		It("should throw error on attempt get config for non existent user", func(done Done) {
+			_, err := env.GetConfigForUser("test")
+			Expect(err).Should(HaveOccurred())
+			Expect(err.Error()).Should(Equal("user test not found"))
+			close(done)
+		})
+
+		It("should throw error on attempt get config on stopped ControlPlane", func(done Done) {
+			env := &Environment{}
+			_, err := env.GetConfigForUser("test")
+			Expect(err).Should(HaveOccurred())
+			Expect(err.Error()).Should(Equal("control plane not started"))
+			close(done)
+		})
+
+		It("should successfully add user and be able to get valid config if ControlPlane not yet running", func(done Done) {
+			env := &Environment{}
+			_, err := env.AddAPIServerUser("test", []string{"one", "two", "system:masters"})
+			Expect(err).ShouldNot(HaveOccurred())
+
+			_, err = env.AddAPIServerUser("secondtest", []string{"three", "four"})
+			Expect(err).ShouldNot(HaveOccurred())
+
+			_, err = env.Start()
+			Expect(err).ShouldNot(HaveOccurred())
+
+			clConfig1, err := env.GetConfigForUser("test")
+			Expect(err).ShouldNot(HaveOccurred())
+			testC1, err := kubeclient.NewForConfig(clConfig1)
+			Expect(err).NotTo(HaveOccurred())
+			tr := &authv1.TokenReview{Spec: authv1.TokenReviewSpec{Token: clConfig1.BearerToken}}
+			result, err := testC1.AuthenticationV1().TokenReviews().Create(context.TODO(), tr, metav1.CreateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			user := result.Status.User
+			Expect(user.Username).To(Equal("test"))
+			Expect(user.Groups).To(ContainElement("one"))
+			Expect(user.Groups).To(ContainElement("two"))
+			Expect(user.Groups).To(ContainElement("system:masters"))
+
+			clConfig2, err := env.GetConfigForUser("secondtest")
+			Expect(err).ShouldNot(HaveOccurred())
+			testC2, err := kubeclient.NewForConfig(clConfig2)
+			Expect(err).NotTo(HaveOccurred())
+			tr = &authv1.TokenReview{Spec: authv1.TokenReviewSpec{Token: clConfig2.BearerToken}}
+			_, err = testC2.AuthenticationV1().TokenReviews().Create(context.TODO(), tr, metav1.CreateOptions{})
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("User \"secondtest\" cannot create resource \"tokenreviews\" in API group \"authentication.k8s.io\" at the cluster scope"))
+
+			close(done)
+		}, 20)
+	})
 })

pkg/envtest/server.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/uuid"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -86,6 +87,9 @@ type ControlPlane = integration.ControlPlane
 // APIServer is the re-exported APIServer type from the internal integration package
 type APIServer = integration.APIServer
 
+// APIServerUser is the re-exported User type from the internal integration package
+type APIServerUser = integration.User
+
 // Etcd is the re-exported Etcd type from the internal integration package
 type Etcd = integration.Etcd
 
@@ -95,11 +99,16 @@ type Environment struct {
 	// ControlPlane is the ControlPlane including the apiserver and etcd
 	ControlPlane integration.ControlPlane
 
-	// Config can be used to talk to the apiserver.  It's automatically
-	// populated if not set using the standard controller-runtime config
+	// Config can be used to talk to the apiserver (insecure endpoint).
+	// It's automatically populated if not set using the standard controller-runtime config
 	// loading.
 	Config *rest.Config
 
+	// SecureConfig can be used to talk to the apiserver (secure endpoint).
+	// It's automatically populated if not set using the standard controller-runtime config
+	// loading. Using `kubeadmin` user BearerToken for auth (populating automatically during controlplane startup)
+	SecureConfig *rest.Config
+
 	// CRDInstallOptions are the options for installing CRDs.
 	CRDInstallOptions CRDInstallOptions
 
@@ -143,6 +152,9 @@ type Environment struct {
 	// KubeAPIServerFlags is the set of flags passed while starting the api server.
 	KubeAPIServerFlags []string
 
+	// KubeAPIServerUsers is the set of users for populate while starting the api server.
+	KubeAPIServerUsers map[string]*integration.User
+
 	// AttachControlPlaneOutput indicates if control plane output will be attached to os.Stdout and os.Stderr.
 	// Enable this to get more visibility of the testing control plane.
 	// It respect KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT environment variable.
@@ -206,7 +218,10 @@ func (te *Environment) Start() (*rest.Config, error) {
 		}
 	} else {
 		if te.ControlPlane.APIServer == nil {
-			te.ControlPlane.APIServer = &integration.APIServer{Args: te.getAPIServerFlags()}
+			te.ControlPlane.APIServer = &integration.APIServer{
+				Args:  te.getAPIServerFlags(),
+				Users: te.KubeAPIServerUsers,
+			}
 		}
 		if te.ControlPlane.Etcd == nil {
 			te.ControlPlane.Etcd = &integration.Etcd{}
@@ -261,6 +276,14 @@ func (te *Environment) Start() (*rest.Config, error) {
 			QPS:   1000.0,
 			Burst: 2000.0,
 		}
+		te.SecureConfig = &rest.Config{
+			Host:            fmt.Sprintf("%s:%d", te.ControlPlane.APIURL().Hostname(), te.ControlPlane.APIServer.SecurePort),
+			TLSClientConfig: te.ControlPlane.APIServer.TLSClientConfig,
+			BearerToken:     te.ControlPlane.APIServer.Users[integration.DefaultUserName].Token,
+			// gotta go fast during tests -- we don't really care about overwhelming our test API server
+			QPS:   1000.0,
+			Burst: 2000.0,
+		}
 	}
 
 	log.V(1).Info("installing CRDs")
@@ -279,6 +302,39 @@ func (te *Environment) Start() (*rest.Config, error) {
 	return te.Config, err
 }
 
+//AddAPIServerUser helper function for adding APIServer user. Should be used while ControlPlane NOT running
+func (te *Environment) AddAPIServerUser(username string, groups []string) (*integration.User, error) {
+	if te.ControlPlane.IsStarted() {
+		return nil, fmt.Errorf("users cannot be added while ControlPlane running")
+	}
+
+	if te.KubeAPIServerUsers == nil {
+		te.KubeAPIServerUsers = make(map[string]*integration.User)
+	}
+	newUser := integration.User{
+		Token:  uuid.New().String(),
+		Name:   username,
+		UID:    username,
+		Groups: groups,
+	}
+	te.KubeAPIServerUsers[username] = &newUser
+	return &newUser, nil
+}
+
+// GetConfigForUser returns *rest.Config for given username if user exists. Should be used while ControlPlane IS running
+func (te *Environment) GetConfigForUser(username string) (*rest.Config, error) {
+	if !te.ControlPlane.IsStarted() {
+		return nil, fmt.Errorf("control plane not started")
+	}
+	copiedConfig := rest.CopyConfig(te.SecureConfig)
+	user, ok := te.KubeAPIServerUsers[username]
+	if !ok {
+		return nil, fmt.Errorf("user %s not found", username)
+	}
+	copiedConfig.BearerToken = user.Token
+	return copiedConfig, nil
+}
+
 func (te *Environment) startControlPlane() error {
 	numTries, maxRetries := 0, 5
 	var err error

pkg/internal/testing/integration/apiserver.go

@@ -9,10 +9,24 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/google/uuid"
+	"k8s.io/client-go/rest"
+
 	"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr"
 	"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal"
 )
 
+// DefaultUserName username for user which populated by default while APIServer starting
+const DefaultUserName = "kubeadmin"
+
+// User represents user entry in token-auth-file
+type User struct {
+	Token  string
+	Name   string
+	UID    string
+	Groups []string
+}
+
 // APIServer knows how to run a kubernetes apiserver.
 type APIServer struct {
 	// URL is the address the ApiServer should listen on for client connections.
@@ -23,6 +37,9 @@ type APIServer struct {
 	// SecurePort is the additional secure port that the APIServer should listen on.
 	SecurePort int
 
+	// TLSconfig is tls configuration to connect to its secure endpoint.
+	TLSClientConfig rest.TLSClientConfig
+
 	// Path is the path to the apiserver binary.
 	//
 	// If this is left as the empty string, we will attempt to locate a binary,
@@ -50,6 +67,16 @@ type APIServer struct {
 	// directory, and the Stop() method will clean it up.
 	CertDir string
 
+	// AuthFiletDir is a path to a directory containing token-auth-file which is needed to setup
+	// authentication for APIServer
+	//
+	// If left unspecified, then the Start() method will create a fresh temporary
+	// directory, and the Stop() method will clean it up.
+	AuthFileDir string
+
+	// Map with users to be populated on APIServer startup. User's name using as key there.
+	Users map[string]*User
+
 	// EtcdURL is the URL of the Etcd the APIServer should use.
 	//
 	// If this is not specified, the Start() method will return an error.
@@ -79,9 +106,21 @@ func (s *APIServer) Start() error {
 			return err
 		}
 	}
+	if !s.dirExists() {
+		s.processState.Dir = ""
+		if err := s.setProcessState(); err != nil {
+			return err
+		}
+	}
+
 	return s.processState.Start(s.Out, s.Err)
 }
 
+func (s *APIServer) dirExists() bool {
+	_, err := os.Stat(s.processState.Dir)
+	return os.IsExist(err)
+}
+
 func (s *APIServer) setProcessState() error {
 	if s.EtcdURL == nil {
 		return fmt.Errorf("expected EtcdURL to be configured")
@@ -115,6 +154,7 @@ func (s *APIServer) setProcessState() error {
 
 	s.URL = &s.processState.URL
 	s.CertDir = s.processState.Dir
+	s.AuthFileDir = s.processState.Dir
 	s.Path = s.processState.Path
 	s.StartTimeout = s.processState.StartTimeout
 	s.StopTimeout = s.processState.StopTimeout
@@ -123,6 +163,10 @@ func (s *APIServer) setProcessState() error {
 		return err
 	}
 
+	if err := s.populateTokenAuthFile(); err != nil {
+		return err
+	}
+
 	s.processState.Args, err = internal.RenderTemplates(
 		internal.DoAPIServerArgDefaulting(s.Args), s,
 	)
@@ -157,6 +201,35 @@ func (s *APIServer) populateAPIServerCerts() error {
 		return err
 	}
 
+	s.TLSClientConfig = rest.TLSClientConfig{
+		CAData: ca.CA.CertBytes(),
+	}
+
+	return nil
+}
+
+func (s *APIServer) populateTokenAuthFile() error {
+	_, statErr := os.Stat(filepath.Join(s.CertDir, "token-auth-file"))
+	if !os.IsNotExist(statErr) {
+		return statErr
+	}
+	if s.Users == nil {
+		s.Users = make(map[string]*User)
+	}
+	defaultUser := User{
+		Token:  uuid.New().String(),
+		Name:   DefaultUserName,
+		UID:    DefaultUserName,
+		Groups: []string{"system:masters"},
+	}
+	s.Users[DefaultUserName] = &defaultUser
+	authTokenFileContent, err := internal.RenderTemplate(internal.TokenAuthFileTemplate, s)
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(filepath.Join(s.AuthFileDir, "token-auth-file"), []byte(authTokenFileContent), 0640); err != nil {
+		return err
+	}
 	return nil
 }