UPSTREAM: <carry>: use OTE for spec filtering in e2e tests. this necessitates generating labels that match specs, and appending them to the spec name for that purpose only.

comment

Author: smg247

.openshift-tests-extension/openshift_payload_hyperkube.json

@@ -80076,4 +80076,4 @@
     "source": "openshift:payload:hyperkube",
     "lifecycle": "blocking"
   }
-]
+]

openshift-hack/cmd/k8s-tests-ext/environment_selectors.go

@@ -1,6 +1,10 @@
 package main
 
-import et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+import (
+	"fmt"
+
+	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+)
 
 // addEnvironmentSelectors adds the environmentSelector field to appropriate specs to facilitate including or excluding
 // them based on attributes of the cluster they are running on
@@ -16,7 +20,7 @@ func addEnvironmentSelectors(specs et.ExtensionTestSpecs) {
 	specs.SelectAny([]et.SelectFunction{ // Since these must use "NameContainsAll" they cannot be included in filterByPlatform
 		et.NameContainsAll("[sig-network] LoadBalancers [Feature:LoadBalancer]", "UDP"),
 		et.NameContainsAll("[sig-network] LoadBalancers [Feature:LoadBalancer]", "session affinity"),
-	}).Exclude(et.PlatformEquals("aws"))
+	}).Exclude(et.PlatformEquals("aws")).AddLabel("[Skipped:aws]")
 }
 
 // filterByPlatform is a helper function to do, simple, "NameContains" filtering on tests by platform
@@ -118,7 +122,9 @@ func filterByPlatform(specs et.ExtensionTestSpecs) {
 			selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 		}
 
-		specs.SelectAny(selectFunctions).Exclude(et.PlatformEquals(platform))
+		specs.SelectAny(selectFunctions).
+			Exclude(et.PlatformEquals(platform)).
+			AddLabel(fmt.Sprintf("[Skipped:%s]", platform))
 	}
 }
 
@@ -160,7 +166,9 @@ func filterByExternalConnectivity(specs et.ExtensionTestSpecs) {
 			selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 		}
 
-		specs.SelectAny(selectFunctions).Exclude(et.ExternalConnectivityEquals(externalConnectivity))
+		specs.SelectAny(selectFunctions).
+			Exclude(et.ExternalConnectivityEquals(externalConnectivity)).
+			AddLabel(fmt.Sprintf("[Skipped:%s]", externalConnectivity))
 	}
 }
 
@@ -189,7 +197,9 @@ func filterByTopology(specs et.ExtensionTestSpecs) {
 			selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 		}
 
-		specs.SelectAny(selectFunctions).Exclude(et.TopologyEquals(topology))
+		specs.SelectAny(selectFunctions).
+			Exclude(et.TopologyEquals(topology)).
+			AddLabel(fmt.Sprintf("[Skipped:%s]", topology))
 	}
 }
 
@@ -208,7 +218,9 @@ func filterByNoOptionalCapabilities(specs et.ExtensionTestSpecs) {
 	for _, exclusion := range exclusions {
 		selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 	}
-	specs.SelectAny(selectFunctions).Exclude(et.NoOptionalCapabilitiesExist())
+	specs.SelectAny(selectFunctions).
+		Exclude(et.NoOptionalCapabilitiesExist()).
+		AddLabel("[Skipped:NoOptionalCapabilities]")
 }
 
 // filterByNetwork is a helper function to do, simple, "NameContains" filtering on tests by network
@@ -226,6 +238,8 @@ func filterByNetwork(specs et.ExtensionTestSpecs) {
 			selectFunctions = append(selectFunctions, et.NameContains(exclusion))
 		}
 
-		specs.SelectAny(selectFunctions).Exclude(et.NetworkEquals(network))
+		specs.SelectAny(selectFunctions).
+			Exclude(et.NetworkEquals(network)).
+			AddLabel(fmt.Sprintf("[Skipped:%s]", network))
 	}
 }

openshift-hack/cmd/k8s-tests-ext/k8s-tests.go

@@ -4,7 +4,10 @@ import (
 	"flag"
 	"os"
 	"reflect"
+	"strconv"
 
+	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+	"github.com/sirupsen/logrus"
 	"k8s.io/kubernetes/openshift-hack/e2e/annotate/generated"
 	"k8s.io/kubernetes/test/e2e/framework"
 
@@ -14,7 +17,6 @@ import (
 	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
 	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
-	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
 	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
 	v "github.com/openshift-eng/openshift-tests-extension/pkg/version"
 
@@ -97,11 +99,22 @@ func main() {
 	//		  the environmental skip code from the enhancement once its implemented.
 	//		- Make sure to account for test renames that occur because of removal of these
 	//		  annotations
-	specs.Walk(func(spec *et.ExtensionTestSpec) {
-		if annotations, ok := generated.Annotations[spec.Name]; ok {
-			spec.Name += annotations
+	var omitAnnotations bool
+	omitAnnotationsVal := os.Getenv("OMIT_ANNOTATIONS")
+	if omitAnnotationsVal != "" {
+		omitAnnotations, err = strconv.ParseBool(omitAnnotationsVal)
+		if err != nil {
+			logrus.WithError(err).Fatal("Failed to parse OMIT_ANNOTATIONS")
 		}
-	})
+	}
+	if !omitAnnotations {
+		logrus.Infof("adding annotations as OMIT_ANNOTATIONS was either false, or not provided")
+		specs.Walk(func(spec *et.ExtensionTestSpec) {
+			if annotations, ok := generated.Annotations[spec.Name]; ok {
+				spec.Name += annotations
+			}
+		})
+	}
 
 	specs = filterOutDisabledSpecs(specs)
 	addLabelsToSpecs(specs)

openshift-hack/cmd/k8s-tests-ext/labels.go

@@ -7,7 +7,7 @@ import (
 func addLabelsToSpecs(specs et.ExtensionTestSpecs) {
 	var namesByLabel = map[string][]string{
 		// tests too slow to be part of conformance
-		"Slow": {
+		"[Slow]": {
 			"[sig-scalability]",                            // disable from the default set for now
 			"should create and stop a working application", // Inordinately slow tests
 
@@ -16,13 +16,13 @@ func addLabelsToSpecs(specs et.ExtensionTestSpecs) {
 			"validates that there exists conflict between pods with same hostPort and protocol but one using 0.0.0.0 hostIP", // 5m, really?
 		},
 		// tests that are known flaky
-		"Flaky": {
+		"[Flaky]": {
 			"Job should run a job to completion when tasks sometimes fail and are not locally restarted", // seems flaky, also may require too many resources
 			// TODO(node): test works when run alone, but not in the suite in CI
 			"[Feature:HPA] Horizontal pod autoscaling (scale resource: CPU) [sig-autoscaling] ReplicationController light Should scale from 1 pod to 2 pods",
 		},
 		// tests that must be run without competition
-		"Serial": {
+		"[Serial]": {
 			"[Disruptive]",
 			"[Feature:Performance]", // requires isolation
 

openshift-hack/e2e/kube_e2e_test.go

@@ -1,6 +1,7 @@
 package e2e
 
 //go:generate go run -mod vendor ./annotate/cmd -- ./annotate/generated/zz_generated.annotations.go
+//go:generate go run -mod vendor ./label/cmd -- ./label/generated/zz_generated.labels.go
 
 // This file duplicates most of test/e2e/e2e_test.go but limits the included
 // tests (via include.go) to tests that are relevant to openshift.
@@ -16,7 +17,6 @@ import (
 	"time"
 
 	"gopkg.in/yaml.v2"
-
 	// Never, ever remove the line with "/ginkgo". Without it,
 	// the ginkgo test runner will not detect that this
 	// directory contains a Ginkgo test suite.
@@ -36,8 +36,8 @@ import (
 	testfixtures "k8s.io/kubernetes/test/fixtures"
 	"k8s.io/kubernetes/test/utils/image"
 
-	// Ensure test annotation
-	"k8s.io/kubernetes/openshift-hack/e2e/annotate/generated"
+	// Ensure test label generation
+	"k8s.io/kubernetes/openshift-hack/e2e/label/generated"
 )
 
 func TestMain(m *testing.M) {
@@ -109,17 +109,17 @@ func TestMain(m *testing.M) {
 }
 
 func TestE2E(t *testing.T) {
-	// TODO(soltysh): this is raw copy from end of openshift-hack/e2e/annotate/generated/zz_generated.annotations.go
-	// https://issues.redhat.com/browse/OCPBUGS-25641
 	ginkgo.GetSuite().SetAnnotateFn(func(name string, node types.TestSpec) {
-		if newLabels, ok := generated.Annotations[name]; ok {
-			node.AppendText(newLabels)
+		if newLabels, ok := generated.Labels[name]; ok {
+			node.AppendText(fmt.Sprintf(" %s", strings.Replace(newLabels, ",", " ", -1)))
 		} else {
-			panic(fmt.Sprintf("unable to find test %s", name))
+			// If the name isn't found in the generated labels file, it is because the test has been disabled via OTE
+			node.AppendText(" [Disabled:missing]")
 		}
 		if strings.Contains(name, "Kubectl client Kubectl prune with applyset should apply and prune objects") {
 			fmt.Printf("Trying to annotate %q\n", name)
 		}
+
 	})
 
 	e2e.RunE2ETests(t)

openshift-hack/e2e/label/cmd/main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"k8s.io/kubernetes/openshift-hack/e2e/label"
+)
+
+func main() {
+	goPath := os.Getenv("GOPATH")
+	externalBinaryPath := fmt.Sprintf("%s/bin/k8s-tests-ext", goPath)
+	label.Run(externalBinaryPath)
+}