diff --git a/go.mod b/go.mod index 346f99faf5..5616fe3a1f 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/fsnotify/fsnotify v1.4.9 github.com/go-logr/logr v0.4.0 github.com/go-logr/zapr v0.4.0 + github.com/google/uuid v1.1.2 github.com/googleapis/gnostic v0.5.4 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.11 // indirect diff --git a/pkg/envtest/envtest_test.go b/pkg/envtest/envtest_test.go index 18ed4c22c8..a0b2b632f9 100644 --- a/pkg/envtest/envtest_test.go +++ b/pkg/envtest/envtest_test.go @@ -23,11 +23,16 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + authv1 "k8s.io/api/authentication/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -845,4 +850,106 @@ var _ = Describe("Test", func() { close(done) }, 30) }) + + Describe("SecureConfig", func() { + + It("should be populated during envtest start", func(done Done) { + Expect(env.Config.CAData).To(BeNil()) + Expect(env.Config.BearerToken).To(BeEmpty()) + + Expect(env.SecureConfig).NotTo(BeNil()) + Expect(env.SecureConfig.BearerToken).NotTo(BeEmpty()) + Expect(env.SecureConfig.CAData).NotTo(BeNil()) + close(done) + }) + + It("client should work with credentials and authorize successfully", func(done Done) { + c, err := kubeclient.NewForConfig(env.SecureConfig) + Expect(err).NotTo(HaveOccurred()) + + tokenReview := &authv1.TokenReview{Spec: authv1.TokenReviewSpec{Token: env.SecureConfig.BearerToken}} + result, err := c.AuthenticationV1().TokenReviews().Create(context.TODO(), tokenReview, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + user := result.Status.User + Expect(user.Username).To(Equal("kubeadmin")) + Expect(user.Groups).To(ContainElement("system:masters")) + Expect(user.Groups).To(ContainElement("system:authenticated")) + close(done) + }) + + It("should be refreshed after ApiServer restart", func(done Done) { + env := &Environment{} + _, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + copiedConfig := rest.CopyConfig(env.SecureConfig) + Expect(env.Stop()).To(Succeed()) + _, err = env.Start() + Expect(err).NotTo(HaveOccurred()) + + Expect(copiedConfig).NotTo(Equal(env.SecureConfig.CAData)) + Expect(copiedConfig).NotTo(Equal(env.SecureConfig.BearerToken)) + + close(done) + }, 30) + }) + + Describe("User related helper methods", func() { + It("should throw error on attempt to create user while apiserver running", func(done Done) { + _, err := env.AddAPIServerUser("test", []string{"test", "test2"}) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(Equal("users cannot be added while ControlPlane running")) + close(done) + }) + + It("should throw error on attempt get config for non existent user", func(done Done) { + _, err := env.GetConfigForUser("test") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(Equal("user test not found")) + close(done) + }) + + It("should throw error on attempt get config on stopped ControlPlane", func(done Done) { + env := &Environment{} + _, err := env.GetConfigForUser("test") + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(Equal("control plane not started")) + close(done) + }) + + It("should successfully add user and be able to get valid config if ControlPlane not yet running", func(done Done) { + env := &Environment{} + _, err := env.AddAPIServerUser("test", []string{"one", "two", "system:masters"}) + Expect(err).ShouldNot(HaveOccurred()) + + _, err = env.AddAPIServerUser("secondtest", []string{"three", "four"}) + Expect(err).ShouldNot(HaveOccurred()) + + _, err = env.Start() + Expect(err).ShouldNot(HaveOccurred()) + + clConfig1, err := env.GetConfigForUser("test") + Expect(err).ShouldNot(HaveOccurred()) + testC1, err := kubeclient.NewForConfig(clConfig1) + Expect(err).NotTo(HaveOccurred()) + tr := &authv1.TokenReview{Spec: authv1.TokenReviewSpec{Token: clConfig1.BearerToken}} + result, err := testC1.AuthenticationV1().TokenReviews().Create(context.TODO(), tr, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + user := result.Status.User + Expect(user.Username).To(Equal("test")) + Expect(user.Groups).To(ContainElement("one")) + Expect(user.Groups).To(ContainElement("two")) + Expect(user.Groups).To(ContainElement("system:masters")) + + clConfig2, err := env.GetConfigForUser("secondtest") + Expect(err).ShouldNot(HaveOccurred()) + testC2, err := kubeclient.NewForConfig(clConfig2) + Expect(err).NotTo(HaveOccurred()) + tr = &authv1.TokenReview{Spec: authv1.TokenReviewSpec{Token: clConfig2.BearerToken}} + _, err = testC2.AuthenticationV1().TokenReviews().Create(context.TODO(), tr, metav1.CreateOptions{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("User \"secondtest\" cannot create resource \"tokenreviews\" in API group \"authentication.k8s.io\" at the cluster scope")) + + close(done) + }, 20) + }) }) diff --git a/pkg/envtest/server.go b/pkg/envtest/server.go index 614d861fde..6a21154a26 100644 --- a/pkg/envtest/server.go +++ b/pkg/envtest/server.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/google/uuid" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -86,6 +87,9 @@ type ControlPlane = integration.ControlPlane // APIServer is the re-exported APIServer type from the internal integration package type APIServer = integration.APIServer +// APIServerUser is the re-exported User type from the internal integration package +type APIServerUser = integration.User + // Etcd is the re-exported Etcd type from the internal integration package type Etcd = integration.Etcd @@ -95,11 +99,16 @@ type Environment struct { // ControlPlane is the ControlPlane including the apiserver and etcd ControlPlane integration.ControlPlane - // Config can be used to talk to the apiserver. It's automatically - // populated if not set using the standard controller-runtime config + // Config can be used to talk to the apiserver (insecure endpoint). + // It's automatically populated if not set using the standard controller-runtime config // loading. Config *rest.Config + // SecureConfig can be used to talk to the apiserver (secure endpoint). + // It's automatically populated if not set using the standard controller-runtime config + // loading. Using `kubeadmin` user BearerToken for auth (populating automatically during controlplane startup) + SecureConfig *rest.Config + // CRDInstallOptions are the options for installing CRDs. CRDInstallOptions CRDInstallOptions @@ -143,6 +152,9 @@ type Environment struct { // KubeAPIServerFlags is the set of flags passed while starting the api server. KubeAPIServerFlags []string + // KubeAPIServerUsers is the set of users for populate while starting the api server. + KubeAPIServerUsers map[string]*integration.User + // AttachControlPlaneOutput indicates if control plane output will be attached to os.Stdout and os.Stderr. // Enable this to get more visibility of the testing control plane. // It respect KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT environment variable. @@ -206,7 +218,10 @@ func (te *Environment) Start() (*rest.Config, error) { } } else { if te.ControlPlane.APIServer == nil { - te.ControlPlane.APIServer = &integration.APIServer{Args: te.getAPIServerFlags()} + te.ControlPlane.APIServer = &integration.APIServer{ + Args: te.getAPIServerFlags(), + Users: te.KubeAPIServerUsers, + } } if te.ControlPlane.Etcd == nil { te.ControlPlane.Etcd = &integration.Etcd{} @@ -261,6 +276,14 @@ func (te *Environment) Start() (*rest.Config, error) { QPS: 1000.0, Burst: 2000.0, } + te.SecureConfig = &rest.Config{ + Host: fmt.Sprintf("%s:%d", te.ControlPlane.APIURL().Hostname(), te.ControlPlane.APIServer.SecurePort), + TLSClientConfig: te.ControlPlane.APIServer.TLSClientConfig, + BearerToken: te.ControlPlane.APIServer.Users[integration.DefaultUserName].Token, + // gotta go fast during tests -- we don't really care about overwhelming our test API server + QPS: 1000.0, + Burst: 2000.0, + } } log.V(1).Info("installing CRDs") @@ -279,6 +302,39 @@ func (te *Environment) Start() (*rest.Config, error) { return te.Config, err } +//AddAPIServerUser helper function for adding APIServer user. Should be used while ControlPlane NOT running +func (te *Environment) AddAPIServerUser(username string, groups []string) (*integration.User, error) { + if te.ControlPlane.IsStarted() { + return nil, fmt.Errorf("users cannot be added while ControlPlane running") + } + + if te.KubeAPIServerUsers == nil { + te.KubeAPIServerUsers = make(map[string]*integration.User) + } + newUser := integration.User{ + Token: uuid.New().String(), + Name: username, + UID: username, + Groups: groups, + } + te.KubeAPIServerUsers[username] = &newUser + return &newUser, nil +} + +// GetConfigForUser returns *rest.Config for given username if user exists. Should be used while ControlPlane IS running +func (te *Environment) GetConfigForUser(username string) (*rest.Config, error) { + if !te.ControlPlane.IsStarted() { + return nil, fmt.Errorf("control plane not started") + } + copiedConfig := rest.CopyConfig(te.SecureConfig) + user, ok := te.KubeAPIServerUsers[username] + if !ok { + return nil, fmt.Errorf("user %s not found", username) + } + copiedConfig.BearerToken = user.Token + return copiedConfig, nil +} + func (te *Environment) startControlPlane() error { numTries, maxRetries := 0, 5 var err error diff --git a/pkg/internal/testing/integration/apiserver.go b/pkg/internal/testing/integration/apiserver.go index 119657875e..789e1fe13b 100644 --- a/pkg/internal/testing/integration/apiserver.go +++ b/pkg/internal/testing/integration/apiserver.go @@ -9,10 +9,24 @@ import ( "path/filepath" "time" + "github.com/google/uuid" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr" "sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal" ) +// DefaultUserName username for user which populated by default while APIServer starting +const DefaultUserName = "kubeadmin" + +// User represents user entry in token-auth-file +type User struct { + Token string + Name string + UID string + Groups []string +} + // APIServer knows how to run a kubernetes apiserver. type APIServer struct { // URL is the address the ApiServer should listen on for client connections. @@ -23,6 +37,9 @@ type APIServer struct { // SecurePort is the additional secure port that the APIServer should listen on. SecurePort int + // TLSconfig is tls configuration to connect to its secure endpoint. + TLSClientConfig rest.TLSClientConfig + // Path is the path to the apiserver binary. // // If this is left as the empty string, we will attempt to locate a binary, @@ -50,6 +67,16 @@ type APIServer struct { // directory, and the Stop() method will clean it up. CertDir string + // AuthFiletDir is a path to a directory containing token-auth-file which is needed to setup + // authentication for APIServer + // + // If left unspecified, then the Start() method will create a fresh temporary + // directory, and the Stop() method will clean it up. + AuthFileDir string + + // Map with users to be populated on APIServer startup. User's name using as key there. + Users map[string]*User + // EtcdURL is the URL of the Etcd the APIServer should use. // // If this is not specified, the Start() method will return an error. @@ -79,9 +106,21 @@ func (s *APIServer) Start() error { return err } } + if !s.dirExists() { + s.processState.Dir = "" + if err := s.setProcessState(); err != nil { + return err + } + } + return s.processState.Start(s.Out, s.Err) } +func (s *APIServer) dirExists() bool { + _, err := os.Stat(s.processState.Dir) + return os.IsExist(err) +} + func (s *APIServer) setProcessState() error { if s.EtcdURL == nil { return fmt.Errorf("expected EtcdURL to be configured") @@ -115,6 +154,7 @@ func (s *APIServer) setProcessState() error { s.URL = &s.processState.URL s.CertDir = s.processState.Dir + s.AuthFileDir = s.processState.Dir s.Path = s.processState.Path s.StartTimeout = s.processState.StartTimeout s.StopTimeout = s.processState.StopTimeout @@ -123,6 +163,10 @@ func (s *APIServer) setProcessState() error { return err } + if err := s.populateTokenAuthFile(); err != nil { + return err + } + s.processState.Args, err = internal.RenderTemplates( internal.DoAPIServerArgDefaulting(s.Args), s, ) @@ -157,6 +201,35 @@ func (s *APIServer) populateAPIServerCerts() error { return err } + s.TLSClientConfig = rest.TLSClientConfig{ + CAData: ca.CA.CertBytes(), + } + + return nil +} + +func (s *APIServer) populateTokenAuthFile() error { + _, statErr := os.Stat(filepath.Join(s.CertDir, "token-auth-file")) + if !os.IsNotExist(statErr) { + return statErr + } + if s.Users == nil { + s.Users = make(map[string]*User) + } + defaultUser := User{ + Token: uuid.New().String(), + Name: DefaultUserName, + UID: DefaultUserName, + Groups: []string{"system:masters"}, + } + s.Users[DefaultUserName] = &defaultUser + authTokenFileContent, err := internal.RenderTemplate(internal.TokenAuthFileTemplate, s) + if err != nil { + return err + } + if err := ioutil.WriteFile(filepath.Join(s.AuthFileDir, "token-auth-file"), []byte(authTokenFileContent), 0640); err != nil { + return err + } return nil } diff --git a/pkg/internal/testing/integration/control_plane.go b/pkg/internal/testing/integration/control_plane.go index bab0fb20e0..040af0f6af 100644 --- a/pkg/internal/testing/integration/control_plane.go +++ b/pkg/internal/testing/integration/control_plane.go @@ -23,6 +23,7 @@ var NewTinyCA = internal.NewTinyCA type ControlPlane struct { APIServer *APIServer Etcd *Etcd + started bool } // Start will start your control plane processes. To stop them, call Stop(). @@ -38,7 +39,12 @@ func (f *ControlPlane) Start() error { f.APIServer = &APIServer{} } f.APIServer.EtcdURL = f.Etcd.URL - return f.APIServer.Start() + + if err := f.APIServer.Start(); err != nil { + return err + } + f.started = true + return nil } // Stop will stop your control plane processes, and clean up their data. @@ -55,10 +61,17 @@ func (f *ControlPlane) Stop() error { errList = append(errList, err) } } - + if len(errList) == 0 { + f.started = false + } return utilerrors.NewAggregate(errList) } +// IsStarted returns controlplane running status, true - if controlplane running, false otherwise +func (f *ControlPlane) IsStarted() bool { + return f.started +} + // APIURL returns the URL you should connect to to talk to your API. func (f *ControlPlane) APIURL() *url.URL { return f.APIServer.URL diff --git a/pkg/internal/testing/integration/internal/apiserver.go b/pkg/internal/testing/integration/internal/apiserver.go index 5c0435fa14..607167ba34 100644 --- a/pkg/internal/testing/integration/internal/apiserver.go +++ b/pkg/internal/testing/integration/internal/apiserver.go @@ -5,10 +5,13 @@ package internal var APIServerDefaultArgs = []string{ "--advertise-address=127.0.0.1", "--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}", - "--cert-dir={{ .CertDir }}", + "--tls-cert-file={{ .CertDir }}/apiserver.crt", + "--tls-private-key-file={{ .CertDir }}/apiserver.key", "--insecure-port={{ if .URL }}{{ .URL.Port }}{{ end }}", "--insecure-bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}", "--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}", + "--token-auth-file={{ .AuthFileDir }}/token-auth-file", + "--authorization-mode=RBAC", // we're keeping this disabled because if enabled, default SA is missing which would force all tests to create one // in normal apiserver operation this SA is created by controller, but that is not run in integration environment "--disable-admission-plugins=ServiceAccount", @@ -16,6 +19,9 @@ var APIServerDefaultArgs = []string{ "--allow-privileged=true", } +// TokenAuthFileTemplate template for populating auth-token-file for API server +var TokenAuthFileTemplate = "{{ range .Users }}{{.Token}},{{.Name}},{{.UID}},\"{{range .Groups}}{{.}},{{end}}\"\n{{end}}" + // DoAPIServerArgDefaulting will set default values to allow tests to run offline when the args are not informed. Otherwise, // it will return the same []string arg passed as param. func DoAPIServerArgDefaulting(args []string) []string { diff --git a/pkg/internal/testing/integration/internal/arguments.go b/pkg/internal/testing/integration/internal/arguments.go index 573295d904..0210d01b98 100644 --- a/pkg/internal/testing/integration/internal/arguments.go +++ b/pkg/internal/testing/integration/internal/arguments.go @@ -5,6 +5,15 @@ import ( "html/template" ) +// RenderTemplate renders single string template +func RenderTemplate(tmplStr string, data interface{}) (rendered string, err error) { + renderedArray, err := RenderTemplates([]string{tmplStr}, data) + if err != nil { + return "", err + } + return renderedArray[0], nil +} + // RenderTemplates returns an []string to render the templates func RenderTemplates(argTemplates []string, data interface{}) (args []string, err error) { var t *template.Template diff --git a/pkg/internal/testing/integration/internal/arguments_test.go b/pkg/internal/testing/integration/internal/arguments_test.go index f35a410ae4..ebcf6ca702 100644 --- a/pkg/internal/testing/integration/internal/arguments_test.go +++ b/pkg/internal/testing/integration/internal/arguments_test.go @@ -82,6 +82,18 @@ var _ = Describe("Arguments", func() { )) }) + It("templating single string", func() { + template := "one: {{ .One }}, two: {{.Two}}" + data := struct { + One string + Two string + }{"one", "two"} + + out, err := RenderTemplate(template, data) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(BeEquivalentTo("one: one, two: two")) + }) + Context("When overriding external default args", func() { It("does not change the internal default args for APIServer", func() { integration.APIServerDefaultArgs[0] = "oh no!"