diff --git a/cmd/operator-sdk/internal/genutil/genutil.go b/cmd/operator-sdk/internal/genutil/genutil.go index 970aa4925d7..89518c4a074 100644 --- a/cmd/operator-sdk/internal/genutil/genutil.go +++ b/cmd/operator-sdk/internal/genutil/genutil.go @@ -15,72 +15,14 @@ package genutil import ( - "fmt" "io/ioutil" "os" - "path" - "path/filepath" "github.com/operator-framework/operator-sdk/internal/pkg/scaffold" log "github.com/sirupsen/logrus" ) -// ParseGroupVersions parses the layout of pkg/apis to return a map of -// API groups to versions. -func parseGroupVersions() (map[string][]string, error) { - gvs := make(map[string][]string) - groups, err := ioutil.ReadDir(scaffold.ApisDir) - if err != nil { - return nil, fmt.Errorf("could not read pkg/apis directory to find api Versions: %v", err) - } - - for _, g := range groups { - if g.IsDir() { - groupDir := filepath.Join(scaffold.ApisDir, g.Name()) - versions, err := ioutil.ReadDir(groupDir) - if err != nil { - return nil, fmt.Errorf("could not read %s directory to find api Versions: %v", groupDir, err) - } - - gvs[g.Name()] = make([]string, 0) - for _, v := range versions { - if v.IsDir() { - // Ignore directories that do not contain any files, so generators - // do not get empty directories as arguments. - verDir := filepath.Join(groupDir, v.Name()) - files, err := ioutil.ReadDir(verDir) - if err != nil { - return nil, fmt.Errorf("could not read %s directory to find api Versions: %v", verDir, err) - } - for _, f := range files { - if !f.IsDir() && filepath.Ext(f.Name()) == ".go" { - gvs[g.Name()] = append(gvs[g.Name()], filepath.ToSlash(v.Name())) - break - } - } - } - } - } - } - - if len(gvs) == 0 { - return nil, fmt.Errorf("no groups or versions found in %s", scaffold.ApisDir) - } - return gvs, nil -} - -// createFQAPIs return a slice of all fully qualified pkg + groups + versions -// of pkg and gvs in the format "pkg/groupA/v1". -func createFQAPIs(pkg string, gvs map[string][]string) (apis []string) { - for g, vs := range gvs { - for _, v := range vs { - apis = append(apis, path.Join(pkg, g, v)) - } - } - return apis -} - // generateWithHeaderFile runs f with a header file path as an arguemnt. // If there is no project boilerplate.go.txt file, an empty header file is // created and its path passed as the argument. diff --git a/cmd/operator-sdk/internal/genutil/k8s.go b/cmd/operator-sdk/internal/genutil/k8s.go index 496aeff9dc9..4f103a00bb1 100644 --- a/cmd/operator-sdk/internal/genutil/k8s.go +++ b/cmd/operator-sdk/internal/genutil/k8s.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/operator-framework/operator-sdk/internal/pkg/scaffold" + "github.com/operator-framework/operator-sdk/internal/util/k8sutil" "github.com/operator-framework/operator-sdk/internal/util/projutil" "github.com/pkg/errors" @@ -37,7 +38,7 @@ func K8sCodegen() error { repoPkg := projutil.GetGoPkg() - gvMap, err := parseGroupVersions() + gvMap, err := k8sutil.ParseGroupSubpackages(scaffold.ApisDir) if err != nil { return fmt.Errorf("failed to parse group versions: (%v)", err) } @@ -49,7 +50,7 @@ func K8sCodegen() error { log.Infof("Running deepcopy code-generation for Custom Resource group versions: [%v]\n", gvb.String()) apisPkg := filepath.Join(repoPkg, scaffold.ApisDir) - fqApis := createFQAPIs(apisPkg, gvMap) + fqApis := k8sutil.CreateFQAPIs(apisPkg, gvMap) f := func(a string) error { return deepcopyGen(a, fqApis) } if err = generateWithHeaderFile(f); err != nil { return err diff --git a/cmd/operator-sdk/internal/genutil/openapi.go b/cmd/operator-sdk/internal/genutil/openapi.go index 48940f6ed07..1d0db3d726f 100644 --- a/cmd/operator-sdk/internal/genutil/openapi.go +++ b/cmd/operator-sdk/internal/genutil/openapi.go @@ -39,7 +39,7 @@ func OpenAPIGen() error { absProjectPath := projutil.MustGetwd() repoPkg := projutil.GetGoPkg() - gvMap, err := parseGroupVersions() + gvMap, err := k8sutil.ParseGroupSubpackages(scaffold.ApisDir) if err != nil { return fmt.Errorf("failed to parse group versions: (%v)", err) } @@ -51,7 +51,7 @@ func OpenAPIGen() error { log.Infof("Running OpenAPI code-generation for Custom Resource group versions: [%v]\n", gvb.String()) apisPkg := filepath.Join(repoPkg, scaffold.ApisDir) - fqApis := createFQAPIs(apisPkg, gvMap) + fqApis := k8sutil.CreateFQAPIs(apisPkg, gvMap) f := func(a string) error { return openAPIGen(a, fqApis) } if err = generateWithHeaderFile(f); err != nil { return err diff --git a/internal/pkg/scaffold/olm-catalog/csv.go b/internal/pkg/scaffold/olm-catalog/csv.go index 465ae213155..9166fe89838 100644 --- a/internal/pkg/scaffold/olm-catalog/csv.go +++ b/internal/pkg/scaffold/olm-catalog/csv.go @@ -336,16 +336,16 @@ func (s *CSV) updateCSVFromManifestFiles(cfg *CSVConfig, csv *olmapiv1alpha1.Clu scanner := yamlutil.NewYAMLScanner(yamlData) for scanner.Scan() { yamlSpec := scanner.Bytes() - typemeta, err := k8sutil.GetTypeMetaFromBytes(yamlSpec) + typeMeta, err := k8sutil.GetTypeMetaFromBytes(yamlSpec) if err != nil { return errors.Wrapf(err, "error getting type metadata from manifest %s", f) } - found, err := store.AddToUpdater(yamlSpec, typemeta.Kind) + found, err := store.AddToUpdater(yamlSpec, typeMeta.Kind) if err != nil { return errors.Wrapf(err, "error adding manifest %s to CSV updaters", f) } if !found { - id := gvkID(typemeta.GroupVersionKind()) + id := gvkID(typeMeta.GroupVersionKind()) if _, ok := otherSpecs[id]; !ok { otherSpecs[id] = make([][]byte, 0) } diff --git a/internal/util/k8sutil/crd.go b/internal/util/k8sutil/crd.go index d5a30c1fcd0..ddd2957383d 100644 --- a/internal/util/k8sutil/crd.go +++ b/internal/util/k8sutil/crd.go @@ -18,13 +18,16 @@ import ( "fmt" "io/ioutil" "os" + "path" "path/filepath" - "strings" + "regexp" yaml "github.com/ghodss/yaml" + "github.com/pkg/errors" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" ) +// GetCRDs parses all CRD manifests in the directory crdsDir and all of its subdirectories. func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) { manifests, err := GetCRDManifestPaths(crdsDir) if err != nil { @@ -45,6 +48,7 @@ func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) return crds, nil } +// GetCRDManifestPaths gets all CRD manifest paths in crdsDir and subdirs. func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) { err = filepath.Walk(crdsDir, func(path string, info os.FileInfo, werr error) error { if werr != nil { @@ -53,10 +57,98 @@ func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) { if info == nil { return nil } - if !info.IsDir() && strings.HasSuffix(path, "_crd.yaml") { - crdPaths = append(crdPaths, path) + if !info.IsDir() { + b, err := ioutil.ReadFile(path) + if err != nil { + return errors.Wrapf(err, "error reading manifest %s", path) + } + typeMeta, err := GetTypeMetaFromBytes(b) + if err != nil { + return errors.Wrapf(err, "error getting kind from manifest %s", path) + } + if typeMeta.Kind == "CustomResourceDefinition" { + crdPaths = append(crdPaths, path) + } } return nil }) return crdPaths, err } + +// ParseGroupSubpackages parses the apisDir directory tree and returns a map of +// all found API groups to subpackages. +func ParseGroupSubpackages(apisDir string) (map[string][]string, error) { + return parseGroupSubdirs(apisDir, false) +} + +// ParseGroupVersions parses the apisDir directory tree and returns a map of +// all found API groups to versions. +func ParseGroupVersions(apisDir string) (map[string][]string, error) { + return parseGroupSubdirs(apisDir, true) +} + +// versionRegexp defines a kube-like version: +// https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-versioning +var versionRegexp = regexp.MustCompile("^v[1-9][0-9]*((alpha|beta)[1-9][0-9]*)?$") + +// parseGroupSubdirs searches apisDir for all groups and potential version +// subdirs directly beneath each group dir, and returns a map of each group +// dir name to all children version dir names. If strictVersionMatch is true, +// all potential version dir names must strictly match versionRegexp. If +// false, all subdir names are considered valid. +func parseGroupSubdirs(apisDir string, strictVersionMatch bool) (map[string][]string, error) { + gvs := make(map[string][]string) + groups, err := ioutil.ReadDir(apisDir) + if err != nil { + return nil, errors.Wrapf(err, "error reading directory %q to find API groups", apisDir) + } + + for _, g := range groups { + if g.IsDir() { + groupDir := filepath.Join(apisDir, g.Name()) + versions, err := ioutil.ReadDir(groupDir) + if err != nil { + return nil, errors.Wrapf(err, "error reading directory %q to find API versions", groupDir) + } + + gvs[g.Name()] = make([]string, 0) + for _, v := range versions { + if v.IsDir() { + // Ignore directories that do not contain any files, so generators + // do not get empty directories as arguments. + verDir := filepath.Join(groupDir, v.Name()) + files, err := ioutil.ReadDir(verDir) + if err != nil { + return nil, errors.Wrapf(err, "error reading directory %q to find API source files", verDir) + } + for _, f := range files { + if !f.IsDir() && filepath.Ext(f.Name()) == ".go" { + // If strictVersionMatch is true, strictly check if v.Name() + // is a Kubernetes API version. + if !strictVersionMatch || versionRegexp.MatchString(v.Name()) { + gvs[g.Name()] = append(gvs[g.Name()], v.Name()) + } + break + } + } + } + } + } + } + + if len(gvs) == 0 { + return nil, fmt.Errorf("no groups or versions found in %s", apisDir) + } + return gvs, nil +} + +// CreateFQAPIs return a slice of all fully qualified pkg + groups + versions +// of pkg and gvs in the format "pkg/groupA/v1". +func CreateFQAPIs(pkg string, gvs map[string][]string) (apis []string) { + for g, vs := range gvs { + for _, v := range vs { + apis = append(apis, path.Join(pkg, g, v)) + } + } + return apis +} diff --git a/internal/util/yamlutil/manifest.go b/internal/util/yamlutil/manifest.go index 0644811017f..9b50ee5008a 100644 --- a/internal/util/yamlutil/manifest.go +++ b/internal/util/yamlutil/manifest.go @@ -20,11 +20,13 @@ import ( "io/ioutil" "os" "path/filepath" - "strings" "github.com/operator-framework/operator-sdk/internal/pkg/scaffold" "github.com/operator-framework/operator-sdk/internal/util/fileutil" + "github.com/operator-framework/operator-sdk/internal/util/k8sutil" + "github.com/ghodss/yaml" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -110,19 +112,17 @@ func GenerateCombinedGlobalManifest(crdsDir string) (*os.File, error) { } }() - files, err := ioutil.ReadDir(crdsDir) + crds, err := k8sutil.GetCRDs(crdsDir) if err != nil { - return nil, fmt.Errorf("could not read deploy directory: (%v)", err) + return nil, errors.Wrapf(err, "error getting CRD's from %s", crdsDir) } combined := []byte{} - for _, file := range files { - if strings.HasSuffix(file.Name(), "crd.yaml") { - fileBytes, err := ioutil.ReadFile(filepath.Join(crdsDir, file.Name())) - if err != nil { - return nil, fmt.Errorf("could not read file %s: (%v)", filepath.Join(crdsDir, file.Name()), err) - } - combined = CombineManifests(combined, fileBytes) + for _, crd := range crds { + b, err := yaml.Marshal(crd) + if err != nil { + return nil, errors.Wrapf(err, "error marshalling CRD %s bytes", crd.GetName()) } + combined = CombineManifests(combined, b) } if err := file.Chmod(os.FileMode(fileutil.DefaultFileMode)); err != nil {