Skip to content

Commit 9e967b8

Browse files
smg247smg247
committed
UPSTREAM: <carry>: use OTE for spec filtering in e2e tests. this necessitates generating labels that match specs, and appending them to the spec name for that purpose only.
comment
1 parent ab01feb commit 9e967b8

File tree

10 files changed

+12948
-26
lines changed

10 files changed

+12948
-26
lines changed

Diff for: ‎.openshift-tests-extension/openshift_payload_hyperkube.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -80076,4 +80076,4 @@
8007680076
"source": "openshift:payload:hyperkube",
8007780077
"lifecycle": "blocking"
8007880078
}
80079-
]
80079+
]

Diff for: ‎openshift-hack/cmd/k8s-tests-ext/environment_selectors.go

+22-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package main
22

3-
import et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
3+
import (
4+
"fmt"
5+
6+
et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
7+
)
48

59
// addEnvironmentSelectors adds the environmentSelector field to appropriate specs to facilitate including or excluding
610
// them based on attributes of the cluster they are running on
@@ -16,11 +20,11 @@ func addEnvironmentSelectors(specs et.ExtensionTestSpecs) {
1620
specs.SelectAny([]et.SelectFunction{ // Since these must use "NameContainsAll" they cannot be included in filterByPlatform
1721
et.NameContainsAll("[sig-network] LoadBalancers [Feature:LoadBalancer]", "UDP"),
1822
et.NameContainsAll("[sig-network] LoadBalancers [Feature:LoadBalancer]", "session affinity"),
19-
}).Exclude(et.PlatformEquals("aws"))
23+
}).Exclude(et.PlatformEquals("aws")).AddLabel("[Skipped:aws]")
2024

2125
specs.SelectAny([]et.SelectFunction{ // Since these must use "NameContainsAll" they cannot be included in filterByNetwork
2226
et.NameContainsAll("NetworkPolicy", "named port"),
23-
}).Exclude(et.NetworkEquals("OVNKubernetes"))
27+
}).Exclude(et.NetworkEquals("OVNKubernetes")).AddLabel("[Skipped:Network/OVNKubernetes]")
2428
}
2529

2630
// filterByPlatform is a helper function to do, simple, "NameContains" filtering on tests by platform
@@ -122,7 +126,9 @@ func filterByPlatform(specs et.ExtensionTestSpecs) {
122126
selectFunctions = append(selectFunctions, et.NameContains(exclusion))
123127
}
124128

125-
specs.SelectAny(selectFunctions).Exclude(et.PlatformEquals(platform))
129+
specs.SelectAny(selectFunctions).
130+
Exclude(et.PlatformEquals(platform)).
131+
AddLabel(fmt.Sprintf("[Skipped:%s]", platform))
126132
}
127133
}
128134

@@ -164,7 +170,9 @@ func filterByExternalConnectivity(specs et.ExtensionTestSpecs) {
164170
selectFunctions = append(selectFunctions, et.NameContains(exclusion))
165171
}
166172

167-
specs.SelectAny(selectFunctions).Exclude(et.ExternalConnectivityEquals(externalConnectivity))
173+
specs.SelectAny(selectFunctions).
174+
Exclude(et.ExternalConnectivityEquals(externalConnectivity)).
175+
AddLabel(fmt.Sprintf("[Skipped:%s]", externalConnectivity))
168176
}
169177
}
170178

@@ -193,7 +201,9 @@ func filterByTopology(specs et.ExtensionTestSpecs) {
193201
selectFunctions = append(selectFunctions, et.NameContains(exclusion))
194202
}
195203

196-
specs.SelectAny(selectFunctions).Exclude(et.TopologyEquals(topology))
204+
specs.SelectAny(selectFunctions).
205+
Exclude(et.TopologyEquals(topology)).
206+
AddLabel(fmt.Sprintf("[Skipped:%s]", topology))
197207
}
198208
}
199209

@@ -212,7 +222,9 @@ func filterByNoOptionalCapabilities(specs et.ExtensionTestSpecs) {
212222
for _, exclusion := range exclusions {
213223
selectFunctions = append(selectFunctions, et.NameContains(exclusion))
214224
}
215-
specs.SelectAny(selectFunctions).Exclude(et.NoOptionalCapabilitiesExist())
225+
specs.SelectAny(selectFunctions).
226+
Exclude(et.NoOptionalCapabilitiesExist()).
227+
AddLabel("[Skipped:NoOptionalCapabilities]")
216228
}
217229

218230
// filterByNetwork is a helper function to do, simple, "NameContains" filtering on tests by network
@@ -225,6 +237,8 @@ func filterByNetwork(specs et.ExtensionTestSpecs) {
225237
selectFunctions = append(selectFunctions, et.NameContains(exclusion))
226238
}
227239

228-
specs.SelectAny(selectFunctions).Exclude(et.NetworkEquals(network))
240+
specs.SelectAny(selectFunctions).
241+
Exclude(et.NetworkEquals(network)).
242+
AddLabel(fmt.Sprintf("[Skipped:%s]", network))
229243
}
230244
}

Diff for: ‎openshift-hack/cmd/k8s-tests-ext/k8s-tests.go

+18-5
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ import (
44
"flag"
55
"os"
66
"reflect"
7+
"strconv"
78

9+
et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
10+
"github.com/sirupsen/logrus"
811
"k8s.io/kubernetes/openshift-hack/e2e/annotate/generated"
912
"k8s.io/kubernetes/test/e2e/framework"
1013

@@ -14,7 +17,6 @@ import (
1417
"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
1518
"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
1619
e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
17-
et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
1820
g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
1921
v "github.com/openshift-eng/openshift-tests-extension/pkg/version"
2022

@@ -97,11 +99,22 @@ func main() {
9799
// the environmental skip code from the enhancement once its implemented.
98100
// - Make sure to account for test renames that occur because of removal of these
99101
// annotations
100-
specs.Walk(func(spec *et.ExtensionTestSpec) {
101-
if annotations, ok := generated.Annotations[spec.Name]; ok {
102-
spec.Name += annotations
102+
var omitAnnotations bool
103+
omitAnnotationsVal := os.Getenv("OMIT_ANNOTATIONS")
104+
if omitAnnotationsVal != "" {
105+
omitAnnotations, err = strconv.ParseBool(omitAnnotationsVal)
106+
if err != nil {
107+
logrus.WithError(err).Fatal("Failed to parse OMIT_ANNOTATIONS")
103108
}
104-
})
109+
}
110+
if !omitAnnotations {
111+
logrus.Infof("adding annotations as OMIT_ANNOTATIONS was either false, or not provided")
112+
specs.Walk(func(spec *et.ExtensionTestSpec) {
113+
if annotations, ok := generated.Annotations[spec.Name]; ok {
114+
spec.Name += annotations
115+
}
116+
})
117+
}
105118

106119
specs = filterOutDisabledSpecs(specs)
107120
addLabelsToSpecs(specs)

Diff for: ‎openshift-hack/cmd/k8s-tests-ext/labels.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
func addLabelsToSpecs(specs et.ExtensionTestSpecs) {
88
var namesByLabel = map[string][]string{
99
// tests too slow to be part of conformance
10-
"Slow": {
10+
"[Slow]": {
1111
"[sig-scalability]", // disable from the default set for now
1212
"should create and stop a working application", // Inordinately slow tests
1313

@@ -16,13 +16,13 @@ func addLabelsToSpecs(specs et.ExtensionTestSpecs) {
1616
"validates that there exists conflict between pods with same hostPort and protocol but one using 0.0.0.0 hostIP", // 5m, really?
1717
},
1818
// tests that are known flaky
19-
"Flaky": {
19+
"[Flaky]": {
2020
"Job should run a job to completion when tasks sometimes fail and are not locally restarted", // seems flaky, also may require too many resources
2121
// TODO(node): test works when run alone, but not in the suite in CI
2222
"[Feature:HPA] Horizontal pod autoscaling (scale resource: CPU) [sig-autoscaling] ReplicationController light Should scale from 1 pod to 2 pods",
2323
},
2424
// tests that must be run without competition
25-
"Serial": {
25+
"[Serial]": {
2626
"[Disruptive]",
2727
"[Feature:Performance]", // requires isolation
2828

Diff for: ‎openshift-hack/e2e/kube_e2e_test.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package e2e
22

33
//go:generate go run -mod vendor ./annotate/cmd -- ./annotate/generated/zz_generated.annotations.go
4+
//go:generate go run -mod vendor ./label/cmd -- ./label/generated/zz_generated.labels.go
45

56
// This file duplicates most of test/e2e/e2e_test.go but limits the included
67
// tests (via include.go) to tests that are relevant to openshift.
@@ -16,7 +17,6 @@ import (
1617
"time"
1718

1819
"gopkg.in/yaml.v2"
19-
2020
// Never, ever remove the line with "/ginkgo". Without it,
2121
// the ginkgo test runner will not detect that this
2222
// directory contains a Ginkgo test suite.
@@ -36,8 +36,8 @@ import (
3636
testfixtures "k8s.io/kubernetes/test/fixtures"
3737
"k8s.io/kubernetes/test/utils/image"
3838

39-
// Ensure test annotation
40-
"k8s.io/kubernetes/openshift-hack/e2e/annotate/generated"
39+
// Ensure test label generation
40+
"k8s.io/kubernetes/openshift-hack/e2e/label/generated"
4141
)
4242

4343
func TestMain(m *testing.M) {
@@ -109,17 +109,17 @@ func TestMain(m *testing.M) {
109109
}
110110

111111
func TestE2E(t *testing.T) {
112-
// TODO(soltysh): this is raw copy from end of openshift-hack/e2e/annotate/generated/zz_generated.annotations.go
113-
// https://issues.redhat.com/browse/OCPBUGS-25641
114112
ginkgo.GetSuite().SetAnnotateFn(func(name string, node types.TestSpec) {
115-
if newLabels, ok := generated.Annotations[name]; ok {
116-
node.AppendText(newLabels)
113+
if newLabels, ok := generated.Labels[name]; ok {
114+
node.AppendText(fmt.Sprintf(" %s", strings.Replace(newLabels, ",", " ", -1)))
117115
} else {
118-
panic(fmt.Sprintf("unable to find test %s", name))
116+
// If the name isn't found in the generated labels file, it is because the test has been disabled via OTE
117+
node.AppendText(" [Disabled:missing]")
119118
}
120119
if strings.Contains(name, "Kubectl client Kubectl prune with applyset should apply and prune objects") {
121120
fmt.Printf("Trying to annotate %q\n", name)
122121
}
122+
123123
})
124124

125125
e2e.RunE2ETests(t)

Diff for: ‎openshift-hack/e2e/label/cmd/main.go

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
7+
"k8s.io/kubernetes/openshift-hack/e2e/label"
8+
)
9+
10+
func main() {
11+
goPath := os.Getenv("GOPATH")
12+
externalBinaryPath := fmt.Sprintf("%s/bin/k8s-tests-ext", goPath)
13+
label.Run(externalBinaryPath)
14+
}

0 commit comments

Comments
 (0)