*: refactor CRD/API getters

cmd/operator-sdk/internal/genutil/genutil.go

@@ -15,72 +15,14 @@
 package genutil
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
-	"path"
-	"path/filepath"
 
 	"github.com/operator-framework/operator-sdk/internal/pkg/scaffold"
 
 	log "github.com/sirupsen/logrus"
 )
 
-// ParseGroupVersions parses the layout of pkg/apis to return a map of
-// API groups to versions.
-func parseGroupVersions() (map[string][]string, error) {
-	gvs := make(map[string][]string)
-	groups, err := ioutil.ReadDir(scaffold.ApisDir)
-	if err != nil {
-		return nil, fmt.Errorf("could not read pkg/apis directory to find api Versions: %v", err)
-	}
-
-	for _, g := range groups {
-		if g.IsDir() {
-			groupDir := filepath.Join(scaffold.ApisDir, g.Name())
-			versions, err := ioutil.ReadDir(groupDir)
-			if err != nil {
-				return nil, fmt.Errorf("could not read %s directory to find api Versions: %v", groupDir, err)
-			}
-
-			gvs[g.Name()] = make([]string, 0)
-			for _, v := range versions {
-				if v.IsDir() {
-					// Ignore directories that do not contain any files, so generators
-					// do not get empty directories as arguments.
-					verDir := filepath.Join(groupDir, v.Name())
-					files, err := ioutil.ReadDir(verDir)
-					if err != nil {
-						return nil, fmt.Errorf("could not read %s directory to find api Versions: %v", verDir, err)
-					}
-					for _, f := range files {
-						if !f.IsDir() && filepath.Ext(f.Name()) == ".go" {
-							gvs[g.Name()] = append(gvs[g.Name()], filepath.ToSlash(v.Name()))
-							break
-						}
-					}
-				}
-			}
-		}
-	}
-
-	if len(gvs) == 0 {
-		return nil, fmt.Errorf("no groups or versions found in %s", scaffold.ApisDir)
-	}
-	return gvs, nil
-}
-
-// createFQAPIs return a slice of all fully qualified pkg + groups + versions
-// of pkg and gvs in the format "pkg/groupA/v1".
-func createFQAPIs(pkg string, gvs map[string][]string) (apis []string) {
-	for g, vs := range gvs {
-		for _, v := range vs {
-			apis = append(apis, path.Join(pkg, g, v))
-		}
-	}
-	return apis
-}
-
 // generateWithHeaderFile runs f with a header file path as an arguemnt.
 // If there is no project boilerplate.go.txt file, an empty header file is
 // created and its path passed as the argument.

cmd/operator-sdk/internal/genutil/k8s.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/operator-framework/operator-sdk/internal/pkg/scaffold"
+	"github.com/operator-framework/operator-sdk/internal/util/k8sutil"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
 
 	"github.com/pkg/errors"
@@ -37,7 +38,7 @@ func K8sCodegen() error {
 
 	repoPkg := projutil.GetGoPkg()
 
-	gvMap, err := parseGroupVersions()
+	gvMap, err := k8sutil.ParseGroupSubpackages(scaffold.ApisDir)
 	if err != nil {
 		return fmt.Errorf("failed to parse group versions: (%v)", err)
 	}
@@ -49,7 +50,7 @@ func K8sCodegen() error {
 	log.Infof("Running deepcopy code-generation for Custom Resource group versions: [%v]\n", gvb.String())
 
 	apisPkg := filepath.Join(repoPkg, scaffold.ApisDir)
-	fqApis := createFQAPIs(apisPkg, gvMap)
+	fqApis := k8sutil.CreateFQAPIs(apisPkg, gvMap)
 	f := func(a string) error { return deepcopyGen(a, fqApis) }
 	if err = generateWithHeaderFile(f); err != nil {
 		return err

cmd/operator-sdk/internal/genutil/openapi.go

@@ -39,7 +39,7 @@ func OpenAPIGen() error {
 	absProjectPath := projutil.MustGetwd()
 	repoPkg := projutil.GetGoPkg()
 
-	gvMap, err := parseGroupVersions()
+	gvMap, err := k8sutil.ParseGroupSubpackages(scaffold.ApisDir)
 	if err != nil {
 		return fmt.Errorf("failed to parse group versions: (%v)", err)
 	}
@@ -51,7 +51,7 @@ func OpenAPIGen() error {
 	log.Infof("Running OpenAPI code-generation for Custom Resource group versions: [%v]\n", gvb.String())
 
 	apisPkg := filepath.Join(repoPkg, scaffold.ApisDir)
-	fqApis := createFQAPIs(apisPkg, gvMap)
+	fqApis := k8sutil.CreateFQAPIs(apisPkg, gvMap)
 	f := func(a string) error { return openAPIGen(a, fqApis) }
 	if err = generateWithHeaderFile(f); err != nil {
 		return err

internal/pkg/scaffold/olm-catalog/csv.go

@@ -336,16 +336,16 @@ func (s *CSV) updateCSVFromManifestFiles(cfg *CSVConfig, csv *olmapiv1alpha1.Clu
 		scanner := yamlutil.NewYAMLScanner(yamlData)
 		for scanner.Scan() {
 			yamlSpec := scanner.Bytes()
-			typemeta, err := k8sutil.GetTypeMetaFromBytes(yamlSpec)
+			typeMeta, err := k8sutil.GetTypeMetaFromBytes(yamlSpec)
 			if err != nil {
 				return errors.Wrapf(err, "error getting type metadata from manifest %s", f)
 			}
-			found, err := store.AddToUpdater(yamlSpec, typemeta.Kind)
+			found, err := store.AddToUpdater(yamlSpec, typeMeta.Kind)
 			if err != nil {
 				return errors.Wrapf(err, "error adding manifest %s to CSV updaters", f)
 			}
 			if !found {
-				id := gvkID(typemeta.GroupVersionKind())
+				id := gvkID(typeMeta.GroupVersionKind())
 				if _, ok := otherSpecs[id]; !ok {
 					otherSpecs[id] = make([][]byte, 0)
 				}

internal/util/k8sutil/crd.go

@@ -18,13 +18,17 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	yaml "github.com/ghodss/yaml"
+	"github.com/pkg/errors"
 	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 )
 
+// GetCRDManifestPaths gets all CRD manifests in crdsDir and subdirs.
 func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) {
 	manifests, err := GetCRDManifestPaths(crdsDir)
 	if err != nil {
@@ -45,6 +49,7 @@ func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error)
 	return crds, nil
 }
 
+// GetCRDManifestPaths gets all CRD manifest paths in crdsDir and subdirs.
 func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) {
 	err = filepath.Walk(crdsDir, func(path string, info os.FileInfo, werr error) error {
 		if werr != nil {
@@ -53,10 +58,96 @@ func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) {
 		if info == nil {
 			return nil
 		}
-		if !info.IsDir() && strings.HasSuffix(path, "_crd.yaml") {
-			crdPaths = append(crdPaths, path)
+		if !info.IsDir() {
+			b, err := ioutil.ReadFile(path)
+			if err != nil {
+				return errors.Wrapf(err, "error reading manifest %s", path)
+			}
+			typeMeta, err := GetTypeMetaFromBytes(b)
+			if err != nil {
+				return errors.Wrapf(err, "error getting kind from manifest %s", path)
+			}
+			if typeMeta.Kind == "CustomResourceDefinition" {
+				crdPaths = append(crdPaths, path)
+			}
 		}
 		return nil
 	})
 	return crdPaths, err
 }
+
+// ParseGroupSubpackages parses the apisDir directory tree and returns a map of
+// all found API groups to subpackages.
+func ParseGroupSubpackages(apisDir string) (map[string][]string, error) {
+	return parseGroupSubdirs(apisDir, false)
+}
+
+// ParseGroupSubdirs parses the apisDir directory tree and returns a map of
+// all found API groups to versions.
+func ParseGroupVersions(apisDir string) (map[string][]string, error) {
+	return parseGroupSubdirs(apisDir, true)
+}
+
+var versionRegexp = regexp.MustCompile("^v[1-9][0-9]*((alpha|beta)[1-9][0-9]*)?$")
+
+func parseGroupSubdirs(apisDir string, strict bool) (map[string][]string, error) {
+	gvs := make(map[string][]string)
+	groups, err := ioutil.ReadDir(apisDir)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error reading directory %q to find API groups", apisDir)
+	}
+
+	for _, g := range groups {
+		if g.IsDir() {
+			groupDir := filepath.Join(apisDir, g.Name())
+			versions, err := ioutil.ReadDir(groupDir)
+			if err != nil {
+				return nil, errors.Wrapf(err, "error reading directory %q to find API versions", groupDir)
+			}
+
+			gvs[g.Name()] = make([]string, 0)
+			for _, v := range versions {
+				if v.IsDir() {
+					// Ignore directories that do not contain any files, so generators
+					// do not get empty directories as arguments.
+					verDir := filepath.Join(groupDir, v.Name())
+					files, err := ioutil.ReadDir(verDir)
+					if err != nil {
+						return nil, errors.Wrapf(err, "error reading directory %q to find API source files", verDir)
+					}
+					for _, f := range files {
+						if !f.IsDir() && filepath.Ext(f.Name()) == ".go" {
+							vsplit := strings.Split(v.Name(), string(filepath.Separator))
+							// Strictly check if maybeVersion is a Kubernetes API version.
+							if strict {
+								maybeVersion := vsplit[0]
+								if versionRegexp.MatchString(maybeVersion) {
+									gvs[g.Name()] = append(gvs[g.Name()], maybeVersion)
+								}
+							} else {
+								gvs[g.Name()] = append(gvs[g.Name()], filepath.ToSlash(v.Name()))
+							}
+							break
+						}
+					}
+				}
+			}
+		}
+	}
+
+	if len(gvs) == 0 {
+		return nil, fmt.Errorf("no groups or versions found in %s", apisDir)
+	}
+	return gvs, nil
+}
+
+// CreateFQAPIs return a slice of all fully qualified pkg + groups + versions
+// of pkg and gvs in the format "pkg/groupA/v1".
+func CreateFQAPIs(pkg string, gvs map[string][]string) (apis []string) {
+	for g, vs := range gvs {
+		for _, v := range vs {
+			apis = append(apis, path.Join(pkg, g, v))
+		}
+	}
+	return apis
+}

internal/util/yamlutil/manifest.go

@@ -20,11 +20,13 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/operator-framework/operator-sdk/internal/pkg/scaffold"
 	"github.com/operator-framework/operator-sdk/internal/util/fileutil"
+	"github.com/operator-framework/operator-sdk/internal/util/k8sutil"
 
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -110,19 +112,17 @@ func GenerateCombinedGlobalManifest(crdsDir string) (*os.File, error) {
 		}
 	}()
 
-	files, err := ioutil.ReadDir(crdsDir)
+	crds, err := k8sutil.GetCRDs(crdsDir)
 	if err != nil {
-		return nil, fmt.Errorf("could not read deploy directory: (%v)", err)
+		return nil, errors.Wrapf(err, "error getting CRD's from %s", crdsDir)
 	}
 	combined := []byte{}
-	for _, file := range files {
-		if strings.HasSuffix(file.Name(), "crd.yaml") {
-			fileBytes, err := ioutil.ReadFile(filepath.Join(crdsDir, file.Name()))
-			if err != nil {
-				return nil, fmt.Errorf("could not read file %s: (%v)", filepath.Join(crdsDir, file.Name()), err)
-			}
-			combined = CombineManifests(combined, fileBytes)
+	for _, crd := range crds {
+		b, err := yaml.Marshal(crd)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error marshalling CRD %s bytes", crd.GetName())
 		}
+		combined = CombineManifests(combined, b)
 	}
 
 	if err := file.Chmod(os.FileMode(fileutil.DefaultFileMode)); err != nil {