Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit dfb97dd

Browse files
sutaakarSrihari1192
authored andcommittedSep 27, 2023
Create OLM upgrade e2e scenario using codeflare SDK
e On branch olm-upgrade-e2e-184
1 parent 09192e3 commit dfb97dd

13 files changed

+215
-59
lines changed
 

‎.github/workflows/olm_tests.yaml

+2-6
Original file line numberDiff line numberDiff line change
@@ -138,9 +138,7 @@ jobs:
138138
export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
139139
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
140140
set -euo pipefail
141-
go test -timeout 30m -v ./test/e2e -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
142-
env:
143-
RUN_OLM_TESTS: true
141+
go test -timeout 30m -v ./test/upgrade -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
144142
145143
- name: Update Operator to the built version
146144
run: |
@@ -176,9 +174,7 @@ jobs:
176174
export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
177175
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
178176
set -euo pipefail
179-
go test -timeout 30m -v ./test/e2e -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
180-
env:
181-
RUN_OLM_TESTS: true
177+
go test -timeout 30m -v ./test/upgrade -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
182178
183179
- name: Run e2e tests against built operator
184180
run: |

‎test/e2e/mnist_pytorch_mcad_job_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,6 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
145145
test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium).
146146
Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
147147

148-
// Retrieving the job logs once it has completed or timed out
149-
defer WriteJobLogs(test, job.Namespace, job.Name)
150-
151148
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
152149
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
153150
Or(

‎test/e2e/mnist_raycluster_sdk_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -192,9 +192,6 @@ func TestMNISTRayClusterSDK(t *testing.T) {
192192
test.Expect(err).NotTo(HaveOccurred())
193193
test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)
194194

195-
// Retrieving the job logs once it has completed or timed out
196-
defer WriteJobLogs(test, job.Namespace, job.Name)
197-
198195
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
199196
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
200197
Or(

‎test/e2e/mnist_rayjob.py

100644100755
File mode changed.

‎test/e2e/setup.sh

+3-5
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,9 @@ set -euo pipefail
1818
: "${KUBERAY_VERSION}"
1919

2020
echo Deploying KubeRay "${KUBERAY_VERSION}"
21-
kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=90s"
21+
kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s"
2222

23-
kubectl create ns codeflare-system --dry-run=client -o yaml | kubectl apply -f -
24-
25-
cat <<EOF | kubectl apply -n codeflare-system -f -
23+
cat <<EOF | kubectl apply -f -
2624
apiVersion: rbac.authorization.k8s.io/v1
2725
kind: ClusterRole
2826
metadata:
@@ -44,7 +42,7 @@ rules:
4442
- delete
4543
EOF
4644

47-
cat <<EOF | kubectl apply -n codeflare-system -f -
45+
cat <<EOF | kubectl apply -f -
4846
kind: ClusterRoleBinding
4947
apiVersion: rbac.authorization.k8s.io/v1
5048
metadata:
File renamed without changes.

‎test/support/batch.go

-21
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ import (
2020
"github.com/onsi/gomega"
2121

2222
batchv1 "k8s.io/api/batch/v1"
23-
corev1 "k8s.io/api/core/v1"
2423
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25-
"k8s.io/apimachinery/pkg/labels"
2624
)
2725

2826
func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job {
@@ -37,22 +35,3 @@ func GetJob(t Test, namespace, name string) *batchv1.Job {
3735
t.T().Helper()
3836
return Job(t, namespace, name)(t)
3937
}
40-
41-
func WriteJobLogs(t Test, namespace, name string) {
42-
t.T().Helper()
43-
44-
job := GetJob(t, namespace, name)
45-
46-
pods := GetPods(t, job.Namespace, metav1.ListOptions{
47-
LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)},
48-
)
49-
50-
if len(pods) == 0 {
51-
t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name)
52-
} else {
53-
for i, pod := range pods {
54-
t.T().Logf("Retrieving Pod %s/%s logs", pod.Namespace, pod.Name)
55-
WriteToOutputDir(t, pod.Name, Log, GetPodLogs(t, &pods[i], corev1.PodLogOptions{}))
56-
}
57-
}
58-
}

‎test/support/core.go

+32
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,35 @@ func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte {
5757

5858
return bytes
5959
}
60+
61+
func storeAllPodLogs(t Test, namespace *corev1.Namespace) {
62+
t.T().Helper()
63+
64+
pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), metav1.ListOptions{})
65+
t.Expect(err).NotTo(gomega.HaveOccurred())
66+
67+
for _, pod := range pods.Items {
68+
for _, container := range pod.Spec.Containers {
69+
t.T().Logf("Retrieving Pod Container %s/%s/%s logs", pod.Namespace, pod.Name, container.Name)
70+
storeContainerLog(t, namespace, pod.Name, container.Name)
71+
}
72+
}
73+
}
74+
75+
func storeContainerLog(t Test, namespace *corev1.Namespace, podName, containerName string) {
76+
t.T().Helper()
77+
78+
options := corev1.PodLogOptions{Container: containerName}
79+
stream, err := t.Client().Core().CoreV1().Pods(namespace.Name).GetLogs(podName, &options).Stream(t.Ctx())
80+
t.Expect(err).NotTo(gomega.HaveOccurred())
81+
82+
defer func() {
83+
t.Expect(stream.Close()).To(gomega.Succeed())
84+
}()
85+
86+
bytes, err := io.ReadAll(stream)
87+
t.Expect(err).NotTo(gomega.HaveOccurred())
88+
89+
containerLogFileName := "pod-" + podName + "-" + containerName
90+
WriteToOutputDir(t, containerLogFileName, Log, bytes)
91+
}

‎test/support/events.go

+148
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
Copyright 2023.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package support
18+
19+
import (
20+
"bytes"
21+
"fmt"
22+
23+
"github.com/onsi/gomega"
24+
25+
corev1 "k8s.io/api/core/v1"
26+
eventsv1 "k8s.io/api/events/v1"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
)
29+
30+
// Based on https://github.com/apache/incubator-kie-kogito-operator/blob/28b2d3dc945e48659b199cca33723568b848f72e/test/pkg/framework/logging.go
31+
32+
const (
33+
eventLastSeenKey = "LAST_SEEN"
34+
eventFirstSeenKey = "FIRST_SEEN"
35+
eventNameKey = "NAME"
36+
eventSubObjectKey = "SUBOBJECT"
37+
eventTypeKey = "TYPE"
38+
eventReasonKey = "REASON"
39+
eventMessageKey = "MESSAGE"
40+
41+
eventLogFileName = "events"
42+
)
43+
44+
var eventKeys = []string{
45+
eventLastSeenKey,
46+
eventFirstSeenKey,
47+
eventNameKey,
48+
eventSubObjectKey,
49+
eventTypeKey,
50+
eventReasonKey,
51+
eventMessageKey,
52+
}
53+
54+
func storeEvents(t Test, namespace *corev1.Namespace) {
55+
t.T().Helper()
56+
57+
events, err := t.Client().Core().EventsV1().Events(namespace.Name).List(t.Ctx(), metav1.ListOptions{})
58+
t.Expect(err).NotTo(gomega.HaveOccurred())
59+
60+
bytes, err := renderEventContent(eventKeys, mapEventsToKeys(events))
61+
t.Expect(err).NotTo(gomega.HaveOccurred())
62+
63+
WriteToOutputDir(t, eventLogFileName, Log, bytes)
64+
}
65+
66+
func mapEventsToKeys(eventList *eventsv1.EventList) []map[string]string {
67+
eventMaps := []map[string]string{}
68+
69+
for _, event := range eventList.Items {
70+
eventMap := make(map[string]string)
71+
eventMap[eventLastSeenKey] = getDefaultEventValueIfNull(event.DeprecatedLastTimestamp.Format("2006-01-02 15:04:05"))
72+
eventMap[eventFirstSeenKey] = getDefaultEventValueIfNull(event.DeprecatedFirstTimestamp.Format("2006-01-02 15:04:05"))
73+
eventMap[eventNameKey] = getDefaultEventValueIfNull(event.GetName())
74+
eventMap[eventSubObjectKey] = getDefaultEventValueIfNull(event.Regarding.FieldPath)
75+
eventMap[eventTypeKey] = getDefaultEventValueIfNull(event.Type)
76+
eventMap[eventReasonKey] = getDefaultEventValueIfNull(event.Reason)
77+
eventMap[eventMessageKey] = getDefaultEventValueIfNull(event.Note)
78+
79+
eventMaps = append(eventMaps, eventMap)
80+
}
81+
return eventMaps
82+
}
83+
84+
func getDefaultEventValueIfNull(value string) string {
85+
if len(value) <= 0 {
86+
return "-"
87+
}
88+
return value
89+
}
90+
91+
func renderEventContent(keys []string, dataMaps []map[string]string) ([]byte, error) {
92+
var content bytes.Buffer
93+
// Get size of strings to be written, to be able to format correctly
94+
maxStringSizeMap := make(map[string]int)
95+
for _, key := range keys {
96+
maxSize := len(key)
97+
for _, dataMap := range dataMaps {
98+
if len(dataMap[key]) > maxSize {
99+
maxSize = len(dataMap[key])
100+
}
101+
}
102+
maxStringSizeMap[key] = maxSize
103+
}
104+
105+
// Write headers
106+
for _, header := range keys {
107+
if _, err := content.WriteString(header); err != nil {
108+
return nil, fmt.Errorf("error in writing the header: %v", err)
109+
}
110+
if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[header] - len(header) + 1)); err != nil {
111+
return nil, fmt.Errorf("error in writing headers: %v", err)
112+
}
113+
if _, err := content.WriteString(" | "); err != nil {
114+
return nil, fmt.Errorf("error in writing headers : %v", err)
115+
}
116+
}
117+
if _, err := content.WriteString("\n"); err != nil {
118+
return nil, fmt.Errorf("error in writing headers '|': %v", err)
119+
120+
}
121+
122+
// Write events
123+
for _, dataMap := range dataMaps {
124+
for _, key := range keys {
125+
if _, err := content.WriteString(dataMap[key]); err != nil {
126+
return nil, fmt.Errorf("error in writing events: %v", err)
127+
}
128+
if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[key] - len(dataMap[key]) + 1)); err != nil {
129+
return nil, fmt.Errorf("error in writing events: %v", err)
130+
}
131+
if _, err := content.WriteString(" | "); err != nil {
132+
return nil, fmt.Errorf("error in writing events: %v", err)
133+
}
134+
}
135+
if _, err := content.WriteString("\n"); err != nil {
136+
return nil, fmt.Errorf("error in writing events: %v", err)
137+
}
138+
}
139+
return content.Bytes(), nil
140+
}
141+
142+
func getWhitespaceStr(size int) string {
143+
whiteSpaceStr := ""
144+
for i := 0; i < size; i++ {
145+
whiteSpaceStr += " "
146+
}
147+
return whiteSpaceStr
148+
}

‎test/support/namespace.go

+17
Original file line numberDiff line numberDiff line change
@@ -76,11 +76,28 @@ func CreateTestNamespaceWithName(t Test, namespaceName string, options ...Option
7676
return namespace
7777
}
7878

79+
func GetNamespaceWithName(t Test, namespaceName string) *corev1.Namespace {
80+
81+
namespace, err := t.Client().Core().CoreV1().Namespaces().Get(t.Ctx(), namespaceName, metav1.GetOptions{})
82+
83+
if err != nil {
84+
t.T().Errorf("Failed to retrieve namespace with name %s: %v", namespaceName, err)
85+
}
86+
return namespace
87+
}
88+
7989
func DeleteTestNamespace(t Test, namespaceName string) {
8090
t.T().Helper()
8191
propagationPolicy := metav1.DeletePropagationBackground
92+
StoreNamespaceLogs(t, namespaceName)
8293
err := t.Client().Core().CoreV1().Namespaces().Delete(t.Ctx(), namespaceName, metav1.DeleteOptions{
8394
PropagationPolicy: &propagationPolicy,
8495
})
8596
t.Expect(err).NotTo(gomega.HaveOccurred())
8697
}
98+
99+
func StoreNamespaceLogs(t Test, namespaceName string) {
100+
namespace := GetNamespaceWithName(t, namespaceName)
101+
storeAllPodLogs(t, namespace)
102+
storeEvents(t, namespace)
103+
}

‎test/support/ray_api.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) {
2929
t.T().Helper()
3030
logs, err := rayClient.GetJobLogs(jobID)
3131
t.Expect(err).NotTo(gomega.HaveOccurred())
32-
WriteToOutputDir(t, jobID, Log, []byte(logs))
32+
WriteToOutputDir(t, "ray-job-log-"+jobID, Log, []byte(logs))
3333
}
3434

3535
func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse {

‎test/support/test.go

+2
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@ func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Names
131131
t.T().Helper()
132132
namespace := createTestNamespace(t, options...)
133133
t.T().Cleanup(func() {
134+
storeAllPodLogs(t, namespace)
135+
storeEvents(t, namespace)
134136
deleteTestNamespace(t, namespace)
135137
})
136138
return namespace

‎test/e2e/olm_upgrade_test.go renamed to ‎test/upgrade/olm_upgrade_test.go

+10-20
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,9 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package e2e
17+
package upgrade
1818

1919
import (
20-
"os"
2120
"testing"
2221

2322
. "github.com/onsi/gomega"
@@ -29,31 +28,31 @@ import (
2928
rbacv1 "k8s.io/api/rbac/v1"
3029
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3130

31+
. "github.com/project-codeflare/codeflare-operator/test/e2e"
3232
. "github.com/project-codeflare/codeflare-operator/test/support"
3333
)
3434

3535
var (
36-
namespace *corev1.Namespace
37-
namespaceName string = "test-ns-olmupgrade"
38-
serviceAccountName string = "sdk-user"
36+
namespaceName = "test-ns-olmupgrade"
37+
serviceAccountName = "sdk-user"
3938
)
4039

4140
// Creates a Ray cluster
4241
func TestMNISTRayClusterUp(t *testing.T) {
4342

4443
test := With(t)
4544
test.T().Parallel()
46-
if os.Getenv("RUN_OLM_TESTS") != "true" {
47-
test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set")
48-
}
45+
4946
// Create a namespace
50-
namespace = CreateTestNamespaceWithName(test, namespaceName)
47+
namespace := CreateTestNamespaceWithName(test, namespaceName)
5148
test.T().Logf("Created namespace %s successfully", namespace.Name)
5249

5350
// Delete namespace only if test failed
5451
defer func() {
5552
if t.Failed() {
5653
DeleteTestNamespace(test, namespaceName)
54+
} else {
55+
StoreNamespaceLogs(test, namespaceName)
5756
}
5857
}()
5958

@@ -69,7 +68,7 @@ func TestMNISTRayClusterUp(t *testing.T) {
6968
},
7069
BinaryData: map[string][]byte{
7170
// SDK script
72-
"raycluster_sdk.py": ReadFile(test, "raycluster_sdk.py"),
71+
"start_ray_cluster.py": ReadFile(test, "start_ray_cluster.py"),
7372
},
7473
Immutable: Ptr(true),
7574
}
@@ -167,7 +166,7 @@ func TestMNISTRayClusterUp(t *testing.T) {
167166
// FIXME: switch to base Python image once the dependency on OpenShift CLI is removed
168167
// See https://github.com/project-codeflare/codeflare-sdk/pull/146
169168
Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e",
170-
Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python raycluster_sdk.py" + " " + namespace.Name},
169+
Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python start_ray_cluster.py" + " " + namespace.Name},
171170
VolumeMounts: []corev1.VolumeMount{
172171
{
173172
Name: "test",
@@ -198,9 +197,6 @@ func TestMNISTRayClusterUp(t *testing.T) {
198197
test.Expect(err).NotTo(HaveOccurred())
199198
test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)
200199

201-
// Retrieving the job logs once it has completed or timed out
202-
defer WriteJobLogs(test, job.Namespace, job.Name)
203-
204200
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
205201
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
206202
Or(
@@ -219,9 +215,6 @@ func TestMnistJobSubmit(t *testing.T) {
219215

220216
test := With(t)
221217
test.T().Parallel()
222-
if os.Getenv("RUN_OLM_TESTS") != "true" {
223-
test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set")
224-
}
225218

226219
//delete the namespace after test complete
227220
defer DeleteTestNamespace(test, namespaceName)
@@ -305,9 +298,6 @@ func TestMnistJobSubmit(t *testing.T) {
305298
test.Expect(err).NotTo(HaveOccurred())
306299
test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)
307300

308-
// Retrieving the job logs once it has completed or timed out
309-
defer WriteJobLogs(test, job.Namespace, job.Name)
310-
311301
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
312302
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
313303
Or(

0 commit comments

Comments
 (0)
Please sign in to comment.