diff --git a/.gitignore b/.gitignore index e69fc3c19cd..a2ed3453145 100644 --- a/.gitignore +++ b/.gitignore @@ -97,6 +97,7 @@ tags test/ansible-operator/ansible-operator test/helm-operator/helm-operator images/scorecard-proxy/scorecard-proxy +images/scorecard-test/scorecard-test # Test artifacts pkg/ansible/runner/testdata/valid.yaml diff --git a/cmd/operator-sdk/alpha/scorecard/cmd.go b/cmd/operator-sdk/alpha/scorecard/cmd.go index df00065eaff..414b5d1f7cd 100644 --- a/cmd/operator-sdk/alpha/scorecard/cmd.go +++ b/cmd/operator-sdk/alpha/scorecard/cmd.go @@ -15,20 +15,29 @@ package scorecard import ( + "encoding/json" "fmt" - "log" + + "time" scorecard "github.com/operator-framework/operator-sdk/internal/scorecard/alpha" + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/labels" ) func NewCmd() *cobra.Command { var ( - config string - //bundle string // TODO - to be implemented - selector string - //list bool // TODO - to be implemented + config string + outputFormat string + bundle string + selector string + kubeconfig string + namespace string + serviceAccount string + list bool + skipCleanup bool + waitTime time.Duration ) scorecardCmd := &cobra.Command{ Use: "scorecard", @@ -38,31 +47,87 @@ func NewCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { var err error - o := scorecard.Options{} + o := scorecard.Scorecard{ + ServiceAccount: serviceAccount, + Namespace: namespace, + BundlePath: bundle, + SkipCleanup: skipCleanup, + WaitTime: waitTime, + } + + if bundle == "" { + return fmt.Errorf("bundle flag required") + } + + o.Client, err = scorecard.GetKubeClient(kubeconfig) + if err != nil { + return fmt.Errorf("could not get kubernetes client: %w", err) + } + o.Config, err = scorecard.LoadConfig(config) if err != nil { - return fmt.Errorf("could not find config file %s", err.Error()) + return fmt.Errorf("could not find config file %w", err) } o.Selector, err = labels.Parse(selector) if err != nil { - return fmt.Errorf("could not parse selector %s", err.Error()) + return fmt.Errorf("could not parse selector %w", err) } - // TODO - process list and output formatting here? - - if err := scorecard.RunTests(o); err != nil { - log.Fatal(err) + var scorecardOutput v1alpha2.ScorecardOutput + if list { + scorecardOutput, err = o.ListTests() + if err != nil { + return fmt.Errorf("error listing tests %w", err) + } + } else { + scorecardOutput, err = o.RunTests() + if err != nil { + return fmt.Errorf("error running tests %w", err) + } } - return nil + + return printOutput(outputFormat, scorecardOutput) }, } scorecardCmd.Flags().StringVarP(&config, "config", "c", "", "path to a new to be defined DSL yaml formatted file that configures what tests get executed") - // scorecardCmd.Flags().StringVar(&bundle, "bundle", "", "path to the operator bundle contents on disk") + scorecardCmd.Flags().StringVar(&kubeconfig, "kubeconfig", "", "kubeconfig path") + + scorecardCmd.Flags().StringVar(&bundle, "bundle", "", "path to the operator bundle contents on disk") scorecardCmd.Flags().StringVarP(&selector, "selector", "l", "", "label selector to determine which tests are run") - // scorecardCmd.Flags().BoolVarP(&list, "list", "L", false, "option to enable listing which tests are run") + scorecardCmd.Flags().StringVarP(&namespace, "namespace", "n", "default", "namespace to run the test images in") + scorecardCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", + "Output format for results. Valid values: text, json") + scorecardCmd.Flags().StringVarP(&serviceAccount, "service-account", "s", "default", "Service account to use for tests") + scorecardCmd.Flags().BoolVarP(&list, "list", "L", false, "Option to enable listing which tests are run") + scorecardCmd.Flags().BoolVarP(&skipCleanup, "skip-cleanup", "x", false, "Disable resource cleanup after tests are run") + scorecardCmd.Flags().DurationVarP(&waitTime, "wait-time", "w", time.Duration(30*time.Second), + "seconds to wait for tests to complete. Example: 35s") return scorecardCmd } + +func printOutput(outputFormat string, output v1alpha2.ScorecardOutput) error { + switch outputFormat { + case "text": + o, err := output.MarshalText() + if err != nil { + fmt.Println(err.Error()) + return err + } + fmt.Printf("%s\n", o) + case "json": + bytes, err := json.MarshalIndent(output, "", " ") + if err != nil { + fmt.Println(err.Error()) + return err + } + fmt.Printf("%s\n", string(bytes)) + default: + return fmt.Errorf("invalid output format selected") + } + return nil + +} diff --git a/hack/image/build-scorecard-test-image.sh b/hack/image/build-scorecard-test-image.sh index 0f76cde49c4..85d9f7a3731 100755 --- a/hack/image/build-scorecard-test-image.sh +++ b/hack/image/build-scorecard-test-image.sh @@ -4,14 +4,14 @@ set -eux source hack/lib/image_lib.sh -# TODO build test image -#WD="$(dirname "$(pwd)")" -#GOOS=linux CGO_ENABLED=0 \ -# go build \ -# -gcflags "all=-trimpath=${WD}" \ -# -asmflags "all=-trimpath=${WD}" \ -# -o images/scorecard-test/scorecard-test \ -# images/scorecard-test/cmd/test/main.go +# build scorecard test image +WD="$(dirname "$(pwd)")" +GOOS=linux CGO_ENABLED=0 \ + go build \ + -gcflags "all=-trimpath=${WD}" \ + -asmflags "all=-trimpath=${WD}" \ + -o images/scorecard-test/scorecard-test \ + images/scorecard-test/cmd/test/main.go # Build base image pushd images/scorecard-test diff --git a/images/scorecard-test/Dockerfile b/images/scorecard-test/Dockerfile index 508d841db7f..575d2b27a12 100644 --- a/images/scorecard-test/Dockerfile +++ b/images/scorecard-test/Dockerfile @@ -5,13 +5,12 @@ ENV TEST=/usr/local/bin/scorecard-test \ USER_UID=1001 \ USER_NAME=test -# TODO install test binary -# COPY scorecard-test ${TEST} +# install test binary +COPY scorecard-test ${TEST} COPY bin /usr/local/bin RUN /usr/local/bin/user_setup - ENTRYPOINT ["/usr/local/bin/entrypoint"] USER ${USER_UID} diff --git a/images/scorecard-test/cmd/test/main.go b/images/scorecard-test/cmd/test/main.go index e055bfbd8ea..02ce1942089 100644 --- a/images/scorecard-test/cmd/test/main.go +++ b/images/scorecard-test/cmd/test/main.go @@ -17,11 +17,14 @@ package main import ( "encoding/json" "fmt" + "io/ioutil" "log" "os" + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2" scapiv1alpha2 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2" + "github.com/operator-framework/operator-sdk/internal/scorecard/alpha" "github.com/operator-framework/operator-sdk/internal/scorecard/alpha/tests" ) @@ -35,7 +38,8 @@ import ( // test image. const ( - bundlePath = "/scorecard" + // bundleTar is the tar file containing the bundle contents + bundleTar = "/scorecard/bundle.tar" ) func main() { @@ -44,7 +48,19 @@ func main() { log.Fatal("test name argument is required") } - cfg, err := tests.GetBundle(bundlePath) + // Create tmp directory for the untar'd bundle + tmpDir, err := ioutil.TempDir("/tmp", "scorecard-bundle") + if err != nil { + log.Fatal(err) + } + defer os.Remove(tmpDir) + + err = alpha.UntarFile(bundleTar, tmpDir) + if err != nil { + log.Fatalf("error untarring bundle %s", err.Error()) + } + + cfg, err := tests.GetBundle(tmpDir) if err != nil { log.Fatal(err.Error()) } @@ -53,23 +69,19 @@ func main() { switch entrypoint[0] { case tests.OLMBundleValidationTest: - result = tests.BundleValidationTest(cfg) + result = tests.BundleValidationTest(*cfg) case tests.OLMCRDsHaveValidationTest: - result = tests.CRDsHaveValidationTest(cfg) + result = tests.CRDsHaveValidationTest(*cfg) case tests.OLMCRDsHaveResourcesTest: - result = tests.CRDsHaveResourcesTest(cfg) + result = tests.CRDsHaveResourcesTest(*cfg) case tests.OLMSpecDescriptorsTest: - result = tests.SpecDescriptorsTest(cfg) + result = tests.SpecDescriptorsTest(*cfg) case tests.OLMStatusDescriptorsTest: - result = tests.StatusDescriptorsTest(cfg) - case tests.BasicCheckStatusTest: - result = tests.CheckStatusTest(cfg) + result = tests.StatusDescriptorsTest(*cfg) case tests.BasicCheckSpecTest: - result = tests.CheckSpecTest(cfg) + result = tests.CheckSpecTest(*cfg) default: - log.Fatal("invalid test name argument passed") - // TODO print out full list of test names to give a hint - // to the end user on what the valid tests are + result = printValidTests() } prettyJSON, err := json.MarshalIndent(result, "", " ") @@ -79,3 +91,20 @@ func main() { fmt.Printf("%s\n", string(prettyJSON)) } + +// printValidTests will print out full list of test names to give a hint to the end user on what the valid tests are +func printValidTests() (result v1alpha2.ScorecardTestResult) { + result.State = scapiv1alpha2.FailState + result.Errors = make([]string, 0) + result.Suggestions = make([]string, 0) + + str := fmt.Sprintf("Valid tests for this image include: %s, %s, %s, %s, %s, %s", + tests.OLMBundleValidationTest, + tests.OLMCRDsHaveValidationTest, + tests.OLMCRDsHaveResourcesTest, + tests.OLMSpecDescriptorsTest, + tests.OLMStatusDescriptorsTest, + tests.BasicCheckSpecTest) + result.Errors = append(result.Errors, str) + return result +} diff --git a/internal/scorecard/alpha/bundle.go b/internal/scorecard/alpha/bundle.go new file mode 100644 index 00000000000..a397cdb684b --- /dev/null +++ b/internal/scorecard/alpha/bundle.go @@ -0,0 +1,51 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alpha + +import ( + "fmt" + "io/ioutil" + "os" + + "k8s.io/apimachinery/pkg/util/rand" +) + +// getBundleData tars up the contents of a bundle from a path, and returns that tar file in []byte +func getBundleData(bundlePath string) (bundleData []byte, err error) { + + // make sure the bundle exists on disk + _, err = os.Stat(bundlePath) + if os.IsNotExist(err) { + return bundleData, fmt.Errorf("bundle path is not valid %w", err) + } + + tempTarFileName := fmt.Sprintf("%s%ctempBundle-%s.tar", os.TempDir(), os.PathSeparator, rand.String(4)) + + paths := []string{bundlePath} + err = CreateTarFile(tempTarFileName, paths) + if err != nil { + return bundleData, fmt.Errorf("error creating tar of bundle %w", err) + } + + defer os.Remove(tempTarFileName) + + var buf []byte + buf, err = ioutil.ReadFile(tempTarFileName) + if err != nil { + return bundleData, fmt.Errorf("error reading tar of bundle %w", err) + } + + return buf, err +} diff --git a/internal/scorecard/alpha/config.go b/internal/scorecard/alpha/config.go index 28e686fe0a8..c72dfce253d 100644 --- a/internal/scorecard/alpha/config.go +++ b/internal/scorecard/alpha/config.go @@ -14,16 +14,44 @@ package alpha -type ScorecardTest struct { - Name string `yaml:"name"` // The container test name - Image string `yaml:"image"` // The container image name - Entrypoint string `yaml:"entrypoint,omitempty"` // An optional entrypoint passed to the test image - Labels map[string]string `yaml:"labels"` // User defined labels used to filter tests - Description string `yaml:"description"` // User readable test description +import ( + "io/ioutil" + + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/yaml" +) + +type Test struct { + Name string `yaml:"name"` // The container test name + Image string `yaml:"image"` // The container image name + // An list of commands and arguments passed to the test image + Entrypoint []string `yaml:"entrypoint,omitempty"` + Labels map[string]string `yaml:"labels"` // User defined labels used to filter tests + Description string `yaml:"description"` // User readable test description + TestPod *v1.Pod `yaml:"-"` // Pod that ran the test } // Config represents the set of test configurations which scorecard // would run based on user input type Config struct { - Tests []ScorecardTest `yaml:"tests"` + Tests []Test `yaml:"tests"` +} + +// LoadConfig will find and return the scorecard config, the config file +// can be passed in via command line flag or from a bundle location or +// bundle image +func LoadConfig(configFilePath string) (Config, error) { + c := Config{} + + // TODO handle getting config from bundle (ondisk or image) + yamlFile, err := ioutil.ReadFile(configFilePath) + if err != nil { + return c, err + } + + if err := yaml.Unmarshal(yamlFile, &c); err != nil { + return c, err + } + + return c, nil } diff --git a/internal/scorecard/alpha/config_test.go b/internal/scorecard/alpha/config_test.go index 5660ee0f1bc..80a4471736a 100644 --- a/internal/scorecard/alpha/config_test.go +++ b/internal/scorecard/alpha/config_test.go @@ -30,11 +30,12 @@ func TestInvalidConfigPath(t *testing.T) { for _, c := range cases { t.Run(c.configPathValue, func(t *testing.T) { _, err := LoadConfig(c.configPathValue) - if err != nil && c.wantError { - t.Logf("Wanted error and got error : %v", err) - return - } else if err != nil && !c.wantError { - t.Errorf("Wanted result but got error: %v", err) + if err == nil && c.wantError { + t.Fatalf("Wanted error but got no error") + } else if err != nil { + if !c.wantError { + t.Fatalf("Wanted result but got error: %v", err) + } return } diff --git a/internal/scorecard/alpha/formatting.go b/internal/scorecard/alpha/formatting.go new file mode 100644 index 00000000000..ea19719de8e --- /dev/null +++ b/internal/scorecard/alpha/formatting.go @@ -0,0 +1,77 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alpha + +import ( + "encoding/json" + "fmt" + + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2" + "k8s.io/client-go/kubernetes" +) + +// getTestResults fetches the test pod logs and converts it into +// ScorecardOutput format +func getTestResults(client kubernetes.Interface, tests []Test) (output v1alpha2.ScorecardOutput) { + output.Results = make([]v1alpha2.ScorecardTestResult, 0) + + for _, test := range tests { + p := test.TestPod + logBytes, err := getPodLog(client, p) + if err != nil { + r := v1alpha2.ScorecardTestResult{} + r.Name = test.Name + r.Description = test.Description + r.Errors = []string{fmt.Sprintf("Error getting pod log %s", err.Error())} + output.Results = append(output.Results, r) + } else { + // marshal pod log into ScorecardTestResult + var sc v1alpha2.ScorecardTestResult + err := json.Unmarshal(logBytes, &sc) + if err != nil { + r := v1alpha2.ScorecardTestResult{} + r.Name = test.Name + r.Description = test.Description + r.Errors = []string{fmt.Sprintf("Error unmarshalling test result %s", err.Error())} + output.Results = append(output.Results, r) + } else { + output.Results = append(output.Results, sc) + } + } + } + return output +} + +// ListTests lists the scorecard tests as configured that would be +// run based on user selection +func (o Scorecard) ListTests() (output v1alpha2.ScorecardOutput, err error) { + tests := selectTests(o.Selector, o.Config.Tests) + if len(tests) == 0 { + fmt.Println("no tests selected") + return output, err + } + + output.Results = make([]v1alpha2.ScorecardTestResult, 0) + + for _, test := range tests { + testResult := v1alpha2.ScorecardTestResult{} + testResult.Name = test.Name + testResult.Labels = test.Labels + testResult.Description = test.Description + output.Results = append(output.Results, testResult) + } + + return output, err +} diff --git a/internal/scorecard/alpha/kubeclient.go b/internal/scorecard/alpha/kubeclient.go new file mode 100644 index 00000000000..8164c970ccd --- /dev/null +++ b/internal/scorecard/alpha/kubeclient.go @@ -0,0 +1,49 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alpha + +import ( + "os" + + "github.com/operator-framework/operator-sdk/pkg/k8sutil" + "k8s.io/client-go/kubernetes" + cruntime "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +// GetKubeClient will get a kubernetes client from the following sources: +// - a path to the kubeconfig file passed on the command line (--kubeconfig) +// - an environment variable that specifies the path (export KUBECONFIG) +// - the user's $HOME/.kube/config file +// - in-cluster connection for when the sdk is run within a cluster instead of +// the command line +func GetKubeClient(kubeconfig string) (client kubernetes.Interface, err error) { + + if kubeconfig != "" { + os.Setenv(k8sutil.KubeConfigEnvVar, kubeconfig) + } + + config, err := cruntime.GetConfig() + if err != nil { + return client, err + } + + // create the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return client, err + } + + return clientset, err +} diff --git a/internal/scorecard/alpha/labels_test.go b/internal/scorecard/alpha/labels_test.go index 0b96157ee40..adea88fe957 100644 --- a/internal/scorecard/alpha/labels_test.go +++ b/internal/scorecard/alpha/labels_test.go @@ -46,11 +46,12 @@ func TestEmptySelector(t *testing.T) { } selector, err := labels.Parse(c.selectorValue) - if err != nil && c.wantError { - t.Logf("Wanted error and got error : %v", err) - return - } else if err != nil && !c.wantError { - t.Errorf("Wanted result but got error: %v", err) + if err == nil && c.wantError { + t.Fatalf("Wanted error but got no error") + } else if err != nil { + if !c.wantError { + t.Fatalf("Wanted result but got error: %v", err) + } return } @@ -67,40 +68,52 @@ func TestEmptySelector(t *testing.T) { const testConfig = `tests: - name: "customtest1" image: quay.io/someuser/customtest1:v0.0.1 + entrypoint: + - custom-test labels: suite: custom test: customtest1 description: an ISV custom test that does... - name: "customtest2" image: quay.io/someuser/customtest2:v0.0.1 + entrypoint: + - custom-test labels: suite: custom test: customtest2 description: an ISV custom test that does... - name: "basic-check-spec" image: quay.io/redhat/basictests:v0.0.1 - entrypoint: basic-check-spec + entrypoint: + - scorecard-test + - basic-check-spec labels: suite: basic test: basic-check-spec-test description: check the spec test - name: "basic-check-status" image: quay.io/redhat/basictests:v0.0.1 - entrypoint: basic-check-status + entrypoint: + - scorecard-test + - basic-check-status labels: suite: basic test: basic-check-status-test description: check the status test - name: "olm-bundle-validation" image: quay.io/redhat/olmtests:v0.0.1 - entrypoint: olm-bundle-validation + entrypoint: + - scorecard-test + - olm-bundle-validation labels: suite: olm test: olm-bundle-validation-test description: validate the bundle test - name: "olm-crds-have-validation" image: quay.io/redhat/olmtests:v0.0.1 - entrypoint: olm-crds-have-validation + entrypoint: + - scorecard-test + - olm-crds-have-validation labels: suite: olm test: olm-crds-have-validation-test @@ -109,5 +122,8 @@ const testConfig = `tests: image: quay.io/redhat/kuttltests:v0.0.1 labels: suite: kuttl + entrypoint: + - kuttl-test + - olm-status-descriptors description: Kuttl tests ` diff --git a/internal/scorecard/alpha/scorecard.go b/internal/scorecard/alpha/scorecard.go index d474b3ed1b3..bddfd309306 100644 --- a/internal/scorecard/alpha/scorecard.go +++ b/internal/scorecard/alpha/scorecard.go @@ -15,80 +15,95 @@ package alpha import ( - "errors" "fmt" - "io/ioutil" - "log" "strings" + "time" + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2" "github.com/operator-framework/operator-sdk/version" - "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" ) -type Options struct { - Config Config - Selector labels.Selector - List bool - OutputFormat string - Client kubernetes.Interface +type Scorecard struct { + Config Config + Selector labels.Selector + BundlePath string + WaitTime time.Duration + Kubeconfig string + Namespace string + bundleConfigMap *v1.ConfigMap + ServiceAccount string + Client kubernetes.Interface + SkipCleanup bool } // RunTests executes the scorecard tests as configured -func RunTests(o Options) error { +func (o Scorecard) RunTests() (testOutput v1alpha2.ScorecardOutput, err error) { tests := selectTests(o.Selector, o.Config.Tests) + if len(tests) == 0 { + fmt.Println("no tests selected") + return testOutput, err + } - for i := 0; i < len(tests); i++ { - if err := runTest(tests[i]); err != nil { - return fmt.Errorf("test %s failed %s", tests[i].Name, err.Error()) - } + bundleData, err := getBundleData(o.BundlePath) + if err != nil { + return testOutput, fmt.Errorf("error getting bundle data %w", err) } - return nil -} + // create a ConfigMap holding the bundle contents + o.bundleConfigMap, err = createConfigMap(o, bundleData) + if err != nil { + return testOutput, fmt.Errorf("error creating ConfigMap %w", err) + } -// LoadConfig will find and return the scorecard config, the config file -// can be passed in via command line flag or from a bundle location or -// bundle image -func LoadConfig(configFilePath string) (Config, error) { - c := Config{} + for i, test := range tests { + var err error + tests[i].TestPod, err = o.runTest(test) + if err != nil { + return testOutput, fmt.Errorf("test %s failed %w", test.Name, err) + } + } - // TODO handle getting config from bundle (ondisk or image) - yamlFile, err := ioutil.ReadFile(configFilePath) - if err != nil { - return c, err + if !o.SkipCleanup { + defer deletePods(o.Client, tests) + defer deleteConfigMap(o.Client, o.bundleConfigMap) } - if err := yaml.Unmarshal(yamlFile, &c); err != nil { - return c, err + err = o.waitForTestsToComplete(tests) + if err != nil { + return testOutput, err } - return c, nil + testOutput = getTestResults(o.Client, tests) + + return testOutput, err } // selectTests applies an optionally passed selector expression // against the configured set of tests, returning the selected tests -func selectTests(selector labels.Selector, tests []ScorecardTest) []ScorecardTest { +func selectTests(selector labels.Selector, tests []Test) []Test { - selected := make([]ScorecardTest, 0) - for i := 0; i < len(tests); i++ { - if selector.String() == "" || selector.Matches(labels.Set(tests[i].Labels)) { + selected := make([]Test, 0) + + for _, test := range tests { + if selector.String() == "" || selector.Matches(labels.Set(test.Labels)) { // TODO olm manifests check - selected = append(selected, tests[i]) + selected = append(selected, test) } } return selected } // runTest executes a single test -// TODO once tests exists, handle the test output -func runTest(test ScorecardTest) error { - if test.Name == "" { - return errors.New("todo - remove later, only for linter") - } - log.Printf("running test %s labels %v", test.Name, test.Labels) - return nil +func (o Scorecard) runTest(test Test) (result *v1.Pod, err error) { + + // Create a Pod to run the test + podDef := getPodDefinition(test, o) + result, err = o.Client.CoreV1().Pods(o.Namespace).Create(podDef) + return result, err } func ConfigDocLink() string { @@ -99,3 +114,30 @@ func ConfigDocLink() string { "https://github.com/operator-framework/operator-sdk/blob/%s/doc/test-framework/scorecard.md", version.Version) } + +// waitForTestsToComplete waits for a fixed amount of time while +// checking for test pods to complete +func (o Scorecard) waitForTestsToComplete(tests []Test) (err error) { + waitTimeInSeconds := int(o.WaitTime.Seconds()) + for elapsedSeconds := 0; elapsedSeconds < waitTimeInSeconds; elapsedSeconds++ { + allPodsCompleted := true + for _, test := range tests { + p := test.TestPod + var tmp *v1.Pod + tmp, err = o.Client.CoreV1().Pods(p.Namespace).Get(p.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error getting pod %s %w", p.Name, err) + } + if tmp.Status.Phase != v1.PodSucceeded { + allPodsCompleted = false + } + + } + if allPodsCompleted { + return nil + } + time.Sleep(1 * time.Second) + } + return fmt.Errorf("error - wait time of %d seconds has been exceeded", o.WaitTime) + +} diff --git a/internal/scorecard/alpha/tar.go b/internal/scorecard/alpha/tar.go new file mode 100644 index 00000000000..c4bee3240f3 --- /dev/null +++ b/internal/scorecard/alpha/tar.go @@ -0,0 +1,190 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alpha + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// CreateTarFile walks paths to create tar file tarName +func CreateTarFile(tarName string, paths []string) (err error) { + tarFile, err := os.Create(tarName) + if err != nil { + return err + } + defer func() { + err = tarFile.Close() + }() + + absTar, err := filepath.Abs(tarName) + if err != nil { + return err + } + + // enable compression if file ends in .gz + tw := tar.NewWriter(tarFile) + if strings.HasSuffix(tarName, ".gz") || strings.HasSuffix(tarName, ".gzip") { + gz := gzip.NewWriter(tarFile) + defer gz.Close() + tw = tar.NewWriter(gz) + } + defer tw.Close() + + // walk each specified path and add encountered file to tar + for _, path := range paths { + // validate path + path = filepath.Clean(path) + absPath, err := filepath.Abs(path) + if err != nil { + fmt.Println(err) + continue + } + if absPath == absTar { + continue + } + if absPath == filepath.Dir(absTar) { + continue + } + + walker := func(file string, finfo os.FileInfo, err error) error { + if err != nil { + return err + } + + // fill in header info using func FileInfoHeader + hdr, err := tar.FileInfoHeader(finfo, finfo.Name()) + if err != nil { + return err + } + + relFilePath := file + if filepath.IsAbs(path) { + relFilePath, err = filepath.Rel(path, file) + if err != nil { + return err + } + } + // ensure header has relative file path + hdr.Name = relFilePath + + hdr.Name = strings.TrimPrefix(relFilePath, path) + if err := tw.WriteHeader(hdr); err != nil { + return err + } + // if path is a dir, dont continue + if finfo.Mode().IsDir() { + return nil + } + + // add file to tar + srcFile, err := os.Open(file) + if err != nil { + return err + } + defer srcFile.Close() + _, err = io.Copy(tw, srcFile) + if err != nil { + return err + } + return nil + } + + // build tar + err = filepath.Walk(path, walker) + if err != nil { + return fmt.Errorf("failed to add %s to tar: %w", path, err) + } + } + return nil +} + +// untar a file into a location +func UntarFile(tarName, target string) (err error) { + tarFile, err := os.Open(tarName) + if err != nil { + return err + } + defer func() { + err = tarFile.Close() + }() + + absPath, err := filepath.Abs(target) + if err != nil { + return err + } + + tr := tar.NewReader(tarFile) + if strings.HasSuffix(tarName, ".gz") || strings.HasSuffix(tarName, ".gzip") { + gz, err := gzip.NewReader(tarFile) + if err != nil { + return err + } + defer gz.Close() + tr = tar.NewReader(gz) + } + + // untar each segment + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + // determine proper file path info + finfo := hdr.FileInfo() + fileName := hdr.Name + if filepath.IsAbs(fileName) { + fileName, err = filepath.Rel("/", fileName) + if err != nil { + return err + } + } + absFileName := filepath.Join(absPath, fileName) + + if finfo.Mode().IsDir() { + if err := os.MkdirAll(absFileName, 0755); err != nil { + return err + } + continue + } + + // create new file with original file mode + file, err := os.OpenFile(absFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, finfo.Mode().Perm()) + if err != nil { + return err + } + n, cpErr := io.Copy(file, tr) + if closeErr := file.Close(); closeErr != nil { // close file immediately + return err + } + if cpErr != nil { + return cpErr + } + if n != finfo.Size() { + return fmt.Errorf("unexpected bytes written: wrote %d, want %d", n, finfo.Size()) + } + } + return nil + +} diff --git a/internal/scorecard/alpha/testconfigmap.go b/internal/scorecard/alpha/testconfigmap.go new file mode 100644 index 00000000000..bcd0c1aefde --- /dev/null +++ b/internal/scorecard/alpha/testconfigmap.go @@ -0,0 +1,61 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alpha + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes" +) + +// createConfigMap creates a ConfigMap that will hold the bundle +// contents to be mounted into the test Pods +func createConfigMap(o Scorecard, bundleData []byte) (configMap *v1.ConfigMap, err error) { + cfg := getConfigMapDefinition(o.Namespace, bundleData) + configMap, err = o.Client.CoreV1().ConfigMaps(o.Namespace).Create(cfg) + return configMap, err +} + +// getConfigMapDefinition returns a ConfigMap definition that +// will hold the bundle contents and eventually will be mounted +// into each test Pod +func getConfigMapDefinition(namespace string, bundleData []byte) *v1.ConfigMap { + configMapName := fmt.Sprintf("scorecard-test-%s", rand.String(4)) + data := make(map[string][]byte) + data["bundle.tar"] = bundleData + return &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: namespace, + Labels: map[string]string{ + "app": "scorecard-test", + }, + }, + BinaryData: data, + } +} + +// deleteConfigMap deletes the test bundle ConfigMap and is called +// as part of the test run cleanup +func deleteConfigMap(client kubernetes.Interface, configMap *v1.ConfigMap) { + err := client.CoreV1().ConfigMaps(configMap.Namespace).Delete(configMap.Name, &metav1.DeleteOptions{}) + if err != nil { + log.Errorf("Error deleting configMap %s %s", configMap.Name, err.Error()) + } +} diff --git a/internal/scorecard/alpha/testdata/config.yaml b/internal/scorecard/alpha/testdata/config.yaml new file mode 100644 index 00000000000..9dfb270ad69 --- /dev/null +++ b/internal/scorecard/alpha/testdata/config.yaml @@ -0,0 +1,79 @@ +tests: +- name: "customtest1" + image: quay.io/jemccorm/customtest1:v0.0.1 + entrypoint: + - custom-test + labels: + suite: custom + test: customtest1 + description: an ISV custom test that does... +- name: "customtest2" + entrypoint: + - custom-test + image: quay.io/jemccorm/customtest2:v0.0.1 + labels: + suite: custom + test: customtest2 + description: an ISV custom test that does... +- name: "basic-check-spec" + image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - basic-check-spec + labels: + suite: basic + test: basic-check-spec-test + description: check the spec test +- name: "olm-bundle-validation" + image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-bundle-validation + labels: + suite: olm + test: olm-bundle-validation-test + description: validate the bundle test +- name: "olm-crds-have-validation" + image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-crds-have-validation + labels: + suite: olm + test: olm-crds-have-validation-test + description: CRDs have validation +- name: "olm-crds-have-resources" + image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-crds-have-resources + labels: + suite: olm + test: olm-crds-have-resources-test + description: CRDs have resources +- name: "olm-spec-descriptors" + image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-spec-descriptors + labels: + suite: olm + test: olm-spec-descriptors-test + description: OLM Spec Descriptors +- name: "olm-status-descriptors" + image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-status-descriptors + labels: + suite: olm + test: olm-status-descriptors-test + description: OLM Status Descriptors +- name: "kuttl-tests" + image: quay.io/operator-framework/scorecard-kuttl:dev + labels: + suite: kuttl + description: Kuttl tests + entrypoint: + - kuttl-test + - olm-status-descriptors diff --git a/internal/scorecard/alpha/testdata/pod.yaml b/internal/scorecard/alpha/testdata/pod.yaml new file mode 100644 index 00000000000..3b36f466a8b --- /dev/null +++ b/internal/scorecard/alpha/testdata/pod.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Pod +metadata: + name: scorecard-test + namespace: default +spec: + containers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: quay.io/operator-framework/scorecard-test:dev + imagePullPolicy: IfNotPresent + name: scorecard-test + command: ["/usr/local/bin/scorecard-test"] + args: ["basic-check-spec"] + resources: {} + volumeMounts: + - mountPath: /scorecard + name: scorecard-bundle + readOnly: true + dnsPolicy: ClusterFirst + restartPolicy: Never + securityContext: {} + serviceAccount: default + serviceAccountName: default + volumes: + - name: scorecard-bundle + configMap: + name: scorecard-bundle diff --git a/internal/scorecard/alpha/testpod.go b/internal/scorecard/alpha/testpod.go new file mode 100644 index 00000000000..963cf8fcb34 --- /dev/null +++ b/internal/scorecard/alpha/testpod.go @@ -0,0 +1,100 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alpha + +import ( + "bytes" + "fmt" + "io" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes" +) + +// getPodDefinition fills out a Pod definition based on +// information from the test +func getPodDefinition(test Test, o Scorecard) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("scorecard-test-%s", rand.String(4)), + Namespace: o.Namespace, + Labels: map[string]string{ + "app": "scorecard-test", + }, + }, + Spec: v1.PodSpec{ + ServiceAccountName: o.ServiceAccount, + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "scorecard-test", + Image: "quay.io/operator-framework/scorecard-test:dev", + ImagePullPolicy: v1.PullIfNotPresent, + Command: test.Entrypoint, + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/scorecard", + Name: "scorecard-bundle", + ReadOnly: true, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "scorecard-bundle", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: o.bundleConfigMap.Name, + }, + }, + }, + }, + }, + }, + } +} + +func getPodLog(client kubernetes.Interface, pod *v1.Pod) (logOutput []byte, err error) { + + req := client.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{}) + podLogs, err := req.Stream() + if err != nil { + return logOutput, err + } + defer podLogs.Close() + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + return logOutput, err + } + return buf.Bytes(), err +} + +func deletePods(client kubernetes.Interface, tests []Test) { + for _, test := range tests { + p := test.TestPod + err := client.CoreV1().Pods(p.Namespace).Delete(p.Name, &metav1.DeleteOptions{}) + if err != nil { + log.Errorf("Error deleting pod %s %s\n", p.Name, err.Error()) + } + + } +} diff --git a/internal/scorecard/alpha/tests/basic_tests.go b/internal/scorecard/alpha/tests/basic.go similarity index 65% rename from internal/scorecard/alpha/tests/basic_tests.go rename to internal/scorecard/alpha/tests/basic.go index d7061c26842..a3569ae06d2 100644 --- a/internal/scorecard/alpha/tests/basic_tests.go +++ b/internal/scorecard/alpha/tests/basic.go @@ -15,27 +15,16 @@ package tests import ( + "github.com/operator-framework/operator-registry/pkg/registry" scapiv1alpha2 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2" ) const ( - BasicCheckStatusTest = "basic-check-status" - BasicCheckSpecTest = "basic-check-spec" + BasicCheckSpecTest = "basic-check-spec" ) -// CheckStatusTest verifies that CRs have a status block -func CheckStatusTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { - r := scapiv1alpha2.ScorecardTestResult{} - r.Name = BasicCheckStatusTest - r.Description = "Custom Resource has a Status Block" - r.State = scapiv1alpha2.PassState - r.Errors = make([]string, 0) - r.Suggestions = make([]string, 0) - return r -} - // CheckSpecTest verifies that CRs have a spec block -func CheckSpecTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { +func CheckSpecTest(bundle registry.Bundle) scapiv1alpha2.ScorecardTestResult { r := scapiv1alpha2.ScorecardTestResult{} r.Name = BasicCheckSpecTest r.Description = "Custom Resource has a Spec Block" diff --git a/internal/scorecard/alpha/tests/bundle_test.go b/internal/scorecard/alpha/tests/bundle_test.go index 1dc65dfa681..cb0f12622cc 100644 --- a/internal/scorecard/alpha/tests/bundle_test.go +++ b/internal/scorecard/alpha/tests/bundle_test.go @@ -15,11 +15,11 @@ package tests import ( - "log" - "path/filepath" "testing" + "github.com/operator-framework/operator-registry/pkg/registry" scapiv1alpha2 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) func TestBundlePath(t *testing.T) { @@ -34,11 +34,7 @@ func TestBundlePath(t *testing.T) { for _, c := range cases { t.Run(c.bundlePath, func(t *testing.T) { - abs, err := filepath.Abs(c.bundlePath) - if err != nil { - log.Println(err) - } - _, err = GetBundle(abs) + _, err := GetBundle(c.bundlePath) if err != nil && c.wantError { t.Logf("Wanted error and got error : %v", err) return @@ -63,12 +59,7 @@ func TestBundleCRs(t *testing.T) { for _, c := range cases { t.Run(c.bundlePath, func(t *testing.T) { - abs, err := filepath.Abs(c.bundlePath) - if err != nil { - t.Errorf("Invalid filepath") - } - var cfg TestBundle - cfg, err = GetBundle(abs) + bundle, err := GetBundle(c.bundlePath) if err != nil && c.wantError { t.Logf("Wanted error and got error : %v", err) return @@ -76,8 +67,14 @@ func TestBundleCRs(t *testing.T) { t.Errorf("Wanted result but got error: %v", err) return } - if len(cfg.Bundles) != c.crCount { - t.Errorf("Wanted %d CRs but got: %d", c.crCount, len(cfg.Bundles)) + var crList []unstructured.Unstructured + crList, err = GetCRs(*bundle) + if err != nil { + t.Error(err) + return + } + if len(crList) != c.crCount { + t.Errorf("Wanted %d CRs but got: %d", c.crCount, len(crList)) return } @@ -91,9 +88,8 @@ func TestBasicAndOLM(t *testing.T) { cases := []struct { bundlePath string state scapiv1alpha2.State - function func(TestBundle) scapiv1alpha2.ScorecardTestResult + function func(registry.Bundle) scapiv1alpha2.ScorecardTestResult }{ - {"../testdata", scapiv1alpha2.PassState, CheckStatusTest}, {"../testdata", scapiv1alpha2.PassState, CheckSpecTest}, {"../testdata", scapiv1alpha2.PassState, BundleValidationTest}, {"../testdata", scapiv1alpha2.PassState, CRDsHaveValidationTest}, @@ -106,17 +102,12 @@ func TestBasicAndOLM(t *testing.T) { for _, c := range cases { t.Run(c.bundlePath, func(t *testing.T) { - abs, err := filepath.Abs(c.bundlePath) - if err != nil { - t.Errorf("Invalid filepath") - } - var cfg TestBundle - cfg, err = GetBundle(abs) + bundle, err := GetBundle(c.bundlePath) if err != nil { t.Errorf("Error getting bundle %s", err.Error()) } - result := c.function(cfg) + result := c.function(*bundle) if result.State != scapiv1alpha2.PassState { t.Errorf("%s result State %v expected", result.Name, scapiv1alpha2.PassState) return diff --git a/internal/scorecard/alpha/tests/crhelper.go b/internal/scorecard/alpha/tests/crhelper.go new file mode 100644 index 00000000000..1a890d7a023 --- /dev/null +++ b/internal/scorecard/alpha/tests/crhelper.go @@ -0,0 +1,50 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "encoding/json" + "fmt" + + "github.com/operator-framework/operator-registry/pkg/registry" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// GetCRs parses a Bundle's CSV for CRs +func GetCRs(bundle registry.Bundle) (crList []unstructured.Unstructured, err error) { + + // get CRs from CSV's alm-examples annotation, assume single bundle + + csv, err := bundle.ClusterServiceVersion() + if err != nil { + return crList, fmt.Errorf("error in csv retrieval %s", err.Error()) + } + + if csv.GetAnnotations() == nil { + return crList, nil + } + + almExamples := csv.ObjectMeta.Annotations["alm-examples"] + + if almExamples == "" { + return crList, nil + } + + err = json.Unmarshal([]byte(almExamples), &crList) + if err != nil { + return nil, fmt.Errorf("failed to parse alm-examples annotation: %v", err) + } + return crList, nil +} diff --git a/internal/scorecard/alpha/tests/olm_tests.go b/internal/scorecard/alpha/tests/olm.go similarity index 84% rename from internal/scorecard/alpha/tests/olm_tests.go rename to internal/scorecard/alpha/tests/olm.go index 64d492ad28b..2196cf34ea3 100644 --- a/internal/scorecard/alpha/tests/olm_tests.go +++ b/internal/scorecard/alpha/tests/olm.go @@ -15,6 +15,7 @@ package tests import ( + "github.com/operator-framework/operator-registry/pkg/registry" scapiv1alpha2 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2" ) @@ -27,7 +28,7 @@ const ( ) // BundleValidationTest validates an on-disk bundle -func BundleValidationTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { +func BundleValidationTest(bundle registry.Bundle) scapiv1alpha2.ScorecardTestResult { r := scapiv1alpha2.ScorecardTestResult{} r.Name = OLMBundleValidationTest r.Description = "Validates bundle contents" @@ -39,7 +40,7 @@ func BundleValidationTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { } // CRDsHaveValidationTest verifies all CRDs have a validation section -func CRDsHaveValidationTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { +func CRDsHaveValidationTest(bundle registry.Bundle) scapiv1alpha2.ScorecardTestResult { r := scapiv1alpha2.ScorecardTestResult{} r.Name = OLMCRDsHaveValidationTest r.Description = "All CRDs have an OpenAPI validation subsection" @@ -50,7 +51,7 @@ func CRDsHaveValidationTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { } // CRDsHaveResourcesTest verifies CRDs have resources listed in its owned CRDs section -func CRDsHaveResourcesTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { +func CRDsHaveResourcesTest(bundle registry.Bundle) scapiv1alpha2.ScorecardTestResult { r := scapiv1alpha2.ScorecardTestResult{} r.Name = OLMCRDsHaveResourcesTest r.Description = "All Owned CRDs contain a resources subsection" @@ -62,7 +63,7 @@ func CRDsHaveResourcesTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { } // SpecDescriptorsTest verifies all spec fields have descriptors -func SpecDescriptorsTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { +func SpecDescriptorsTest(bundle registry.Bundle) scapiv1alpha2.ScorecardTestResult { r := scapiv1alpha2.ScorecardTestResult{} r.Name = OLMSpecDescriptorsTest r.Description = "All spec fields have matching descriptors in the CSV" @@ -73,7 +74,7 @@ func SpecDescriptorsTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { } // StatusDescriptorsTest verifies all CRDs have status descriptors -func StatusDescriptorsTest(conf TestBundle) scapiv1alpha2.ScorecardTestResult { +func StatusDescriptorsTest(bundle registry.Bundle) scapiv1alpha2.ScorecardTestResult { r := scapiv1alpha2.ScorecardTestResult{} r.Name = OLMStatusDescriptorsTest r.Description = "All status fields have matching descriptors in the CSV" diff --git a/internal/scorecard/alpha/tests/test_bundle.go b/internal/scorecard/alpha/tests/test_bundle.go index a79b8fd8516..099f1403983 100644 --- a/internal/scorecard/alpha/tests/test_bundle.go +++ b/internal/scorecard/alpha/tests/test_bundle.go @@ -16,25 +16,21 @@ package tests import ( "bytes" - "encoding/json" "fmt" + "os" "github.com/operator-framework/api/pkg/manifests" - "github.com/operator-framework/api/pkg/validation/errors" "github.com/operator-framework/operator-registry/pkg/registry" "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) -// TestBundle holds the bundle contents to be tested -type TestBundle struct { - BundleErrors []errors.ManifestResult - Bundles []*registry.Bundle - CRs []unstructured.Unstructured -} +// GetBundle parses a Bundle from a given on-disk path returning a bundle +func GetBundle(bundlePath string) (bundle *registry.Bundle, err error) { -// GetBundle parses a Bundle from a given on-disk path returning a TestBundle -func GetBundle(bundlePath string) (cfg TestBundle, err error) { + // validate the path + if _, err := os.Stat(bundlePath); os.IsNotExist(err) { + return nil, err + } validationLogOutput := new(bytes.Buffer) origOutput := logrus.StandardLogger().Out @@ -43,50 +39,21 @@ func GetBundle(bundlePath string) (cfg TestBundle, err error) { // TODO evaluate another API call that would support the new // bundle format - _, cfg.Bundles, cfg.BundleErrors = manifests.GetManifestsDir(bundlePath) - - // get CRs from CSV's alm-examples annotation, assume single bundle - cfg.CRs = make([]unstructured.Unstructured, 0) - - if len(cfg.Bundles) == 0 { - return cfg, fmt.Errorf("no bundle found") - } - - csv, err := cfg.Bundles[0].ClusterServiceVersion() - if err != nil { - return cfg, fmt.Errorf("error in csv retrieval %s", err.Error()) - } + var bundles []*registry.Bundle + //var bundleErrors []errors.ManifestResult + _, bundles, _ = manifests.GetManifestsDir(bundlePath) - if csv.GetAnnotations() == nil { - return cfg, nil + if len(bundles) == 0 { + return nil, fmt.Errorf("bundle was not found") } - - almExamples := csv.ObjectMeta.Annotations["alm-examples"] - - if almExamples == "" { - return cfg, nil + if bundles[0] == nil { + return nil, fmt.Errorf("bundle is invalid nil value") } - - if len(cfg.Bundles) > 0 { - var crInterfaces []map[string]interface{} - err = json.Unmarshal([]byte(almExamples), &crInterfaces) - if err != nil { - return cfg, err - } - for i := 0; i < len(crInterfaces); i++ { - buff := new(bytes.Buffer) - enc := json.NewEncoder(buff) - err := enc.Encode(crInterfaces[i]) - if err != nil { - return cfg, err - } - obj := &unstructured.Unstructured{} - if err := obj.UnmarshalJSON(buff.Bytes()); err != nil { - return cfg, err - } - cfg.CRs = append(cfg.CRs, *obj) - } + bundle = bundles[0] + _, err = bundle.ClusterServiceVersion() + if err != nil { + return nil, fmt.Errorf("error in csv retrieval %s", err.Error()) } - return cfg, err + return bundle, nil }