Skip to content

Commit a692692

Browse files
kfswainkfswain
committed
Updating test infra to work for multiple tests
1 parent fb03d81 commit a692692

File tree

3 files changed

+67
-17
lines changed

3 files changed

+67
-17
lines changed

Diff for: ‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ test: manifests generate fmt vet envtest ## Run tests.
123123

124124
.PHONY: test-integration
125125
test-integration: manifests generate fmt vet envtest ## Run tests.
126-
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./test/integration/epp/...
126+
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./test/integration/epp/... -race -coverprofile cover.out
127127

128128
.PHONY: test-e2e
129129
test-e2e: ## Run end-to-end tests against an existing Kubernetes cluster with at least 3 available GPUs.

Diff for: ‎pkg/epp/server/controller_manager.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
ctrl "sigs.k8s.io/controller-runtime"
2929
"sigs.k8s.io/controller-runtime/pkg/cache"
3030
"sigs.k8s.io/controller-runtime/pkg/client"
31+
"sigs.k8s.io/controller-runtime/pkg/manager"
3132
"sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2"
3233
)
3334

@@ -40,7 +41,7 @@ func init() {
4041

4142
// NewDefaultManager creates a new controller manager with default configuration.
4243
func NewDefaultManager(namespace, name string, restConfig *rest.Config) (ctrl.Manager, error) {
43-
manager, err := ctrl.NewManager(restConfig, ctrl.Options{
44+
defaultOpts := ctrl.Options{
4445
Scheme: scheme,
4546
Cache: cache.Options{
4647
ByObject: map[client.Object]cache.ByObject{
@@ -65,7 +66,13 @@ func NewDefaultManager(namespace, name string, restConfig *rest.Config) (ctrl.Ma
6566
},
6667
},
6768
},
68-
})
69+
}
70+
return NewManagerWithOptions(restConfig, defaultOpts)
71+
}
72+
73+
// NewManagerWithOptions creates a new controller manager with injectable options.
74+
func NewManagerWithOptions(restConfig *rest.Config, opts manager.Options) (ctrl.Manager, error) {
75+
manager, err := ctrl.NewManager(restConfig, opts)
6976
if err != nil {
7077
return nil, fmt.Errorf("failed to create controller manager: %v", err)
7178
}

Diff for: ‎test/integration/epp/hermetic_test.go

+57-14
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ import (
4343
"google.golang.org/grpc/credentials/insecure"
4444
"google.golang.org/protobuf/testing/protocmp"
4545
"google.golang.org/protobuf/types/known/structpb"
46+
corev1 "k8s.io/api/core/v1"
47+
"k8s.io/apimachinery/pkg/fields"
4648
"k8s.io/apimachinery/pkg/runtime"
4749
"k8s.io/apimachinery/pkg/types"
4850
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -51,7 +53,10 @@ import (
5153
"k8s.io/component-base/metrics/legacyregistry"
5254
metricsutils "k8s.io/component-base/metrics/testutil"
5355
ctrl "sigs.k8s.io/controller-runtime"
56+
"sigs.k8s.io/controller-runtime/pkg/cache"
57+
"sigs.k8s.io/controller-runtime/pkg/client"
5458
k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
59+
"sigs.k8s.io/controller-runtime/pkg/config"
5560
"sigs.k8s.io/controller-runtime/pkg/envtest"
5661
"sigs.k8s.io/controller-runtime/pkg/manager"
5762
"sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2"
@@ -78,6 +83,13 @@ var (
7883
logger = logutil.NewTestLogger().V(logutil.VERBOSE)
7984
)
8085

86+
func TestMain(m *testing.M) {
87+
cleanup := BeforeSuite()
88+
code := m.Run()
89+
cleanup()
90+
os.Exit(code)
91+
}
92+
8193
func TestKubeInferenceModelRequest(t *testing.T) {
8294
tests := []struct {
8395
name string
@@ -337,11 +349,6 @@ func TestKubeInferenceModelRequest(t *testing.T) {
337349
wantErr: false,
338350
},
339351
}
340-
341-
// Set up global k8sclient and extproc server runner with test environment config
342-
cleanup := BeforeSuit(t)
343-
defer cleanup()
344-
345352
for _, test := range tests {
346353
t.Run(test.name, func(t *testing.T) {
347354
client, cleanup := setUpHermeticServer(t, test.pods, false)
@@ -1266,10 +1273,6 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
12661273
},
12671274
}
12681275

1269-
// Set up global k8sclient and extproc server runner with test environment config
1270-
cleanup := BeforeSuit(t)
1271-
defer cleanup()
1272-
12731276
for _, test := range tests {
12741277
t.Run(test.name, func(t *testing.T) {
12751278
client, cleanup := setUpHermeticServer(t, test.pods, true)
@@ -1380,7 +1383,7 @@ func fakePod(index int) backendmetrics.Pod {
13801383
}
13811384

13821385
// Sets up a test environment and returns the runner struct
1383-
func BeforeSuit(t *testing.T) func() {
1386+
func BeforeSuite() func() {
13841387
// Set up mock k8s API Client
13851388
testEnv = &envtest.Environment{
13861389
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")},
@@ -1404,7 +1407,7 @@ func BeforeSuit(t *testing.T) func() {
14041407
// Init runtime.
14051408
ctrl.SetLogger(logger)
14061409

1407-
mgr, err := server.NewDefaultManager("default", "vllm-llama2-7b-pool", cfg)
1410+
mgr, err := server.NewManagerWithOptions(cfg, managerTestOptions("default", "vllm-llama2-7b-pool"))
14081411
if err != nil {
14091412
logutil.Fatal(logger, err, "Failed to create controller manager")
14101413
}
@@ -1425,7 +1428,7 @@ func BeforeSuit(t *testing.T) func() {
14251428
logutil.Fatal(logger, err, "Failed to setup server runner")
14261429
}
14271430

1428-
// Start the controller manager in go routine, not blocking
1431+
// Start the controller manager in a go routine, not blocking
14291432
go func() {
14301433
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
14311434
logutil.Fatal(logger, err, "Failed to start manager")
@@ -1466,14 +1469,16 @@ func BeforeSuit(t *testing.T) func() {
14661469
}
14671470
}
14681471

1469-
assert.EventuallyWithT(t, func(t *assert.CollectT) {
1472+
assert.Eventually(nil, func() bool {
14701473
modelExist := serverRunner.Datastore.ModelGet("my-model")
14711474
synced := serverRunner.Datastore.PoolHasSynced() && modelExist != nil
1472-
assert.True(t, synced, "Timeout waiting for the pool and models to sync")
1475+
return synced
14731476
}, 10*time.Second, 10*time.Millisecond)
14741477

14751478
return func() {
14761479
_ = testEnv.Stop()
1480+
_ = k8sClient.DeleteAllOf(context.Background(), &v1alpha2.InferencePool{})
1481+
_ = k8sClient.DeleteAllOf(context.Background(), &v1alpha2.InferenceModel{})
14771482
}
14781483
}
14791484

@@ -1601,3 +1606,41 @@ func registerMetricsHandler(mgr manager.Manager, port int) error {
16011606
}
16021607
return nil
16031608
}
1609+
1610+
// inject options that allow multiple test runs to run
1611+
// https://github.com/kubernetes-sigs/controller-runtime/issues/2937
1612+
func managerTestOptions(namespace, name string) ctrl.Options {
1613+
return ctrl.Options{
1614+
Scheme: scheme,
1615+
Cache: cache.Options{
1616+
ByObject: map[client.Object]cache.ByObject{
1617+
&corev1.Pod{}: {
1618+
Namespaces: map[string]cache.Config{
1619+
namespace: {},
1620+
},
1621+
},
1622+
&v1alpha2.InferencePool{}: {
1623+
Namespaces: map[string]cache.Config{
1624+
namespace: {
1625+
FieldSelector: fields.SelectorFromSet(fields.Set{
1626+
"metadata.name": name,
1627+
}),
1628+
},
1629+
},
1630+
},
1631+
&v1alpha2.InferenceModel{}: {
1632+
Namespaces: map[string]cache.Config{
1633+
namespace: {},
1634+
},
1635+
},
1636+
},
1637+
},
1638+
Controller: config.Controller{
1639+
SkipNameValidation: boolPointer(true),
1640+
},
1641+
}
1642+
}
1643+
1644+
func boolPointer(b bool) *bool {
1645+
return &b
1646+
}

0 commit comments

Comments
 (0)