Skip to content

Commit 65b8889

Browse files
sutaakaropenshift-merge-robot
authored andcommitted
e2e test support: Store test pod logs and events
1 parent 1fa4979 commit 65b8889

File tree

8 files changed

+186
-33
lines changed

8 files changed

+186
-33
lines changed

test/e2e/mnist_pytorch_mcad_job_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,6 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
145145
test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium).
146146
Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
147147

148-
// Retrieving the job logs once it has completed or timed out
149-
defer WriteJobLogs(test, job.Namespace, job.Name)
150-
151148
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
152149
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
153150
Or(

test/e2e/mnist_raycluster_sdk_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -192,9 +192,6 @@ func TestMNISTRayClusterSDK(t *testing.T) {
192192
test.Expect(err).NotTo(HaveOccurred())
193193
test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)
194194

195-
// Retrieving the job logs once it has completed or timed out
196-
defer WriteJobLogs(test, job.Namespace, job.Name)
197-
198195
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
199196
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
200197
Or(

test/e2e/setup.sh

+3-5
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,9 @@ set -euo pipefail
1818
: "${KUBERAY_VERSION}"
1919

2020
echo Deploying KubeRay "${KUBERAY_VERSION}"
21-
kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=90s"
21+
kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s"
2222

23-
kubectl create ns codeflare-system --dry-run=client -o yaml | kubectl apply -f -
24-
25-
cat <<EOF | kubectl apply -n codeflare-system -f -
23+
cat <<EOF | kubectl apply -f -
2624
apiVersion: rbac.authorization.k8s.io/v1
2725
kind: ClusterRole
2826
metadata:
@@ -44,7 +42,7 @@ rules:
4442
- delete
4543
EOF
4644

47-
cat <<EOF | kubectl apply -n codeflare-system -f -
45+
cat <<EOF | kubectl apply -f -
4846
kind: ClusterRoleBinding
4947
apiVersion: rbac.authorization.k8s.io/v1
5048
metadata:

test/support/batch.go

-21
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ import (
2020
"github.com/onsi/gomega"
2121

2222
batchv1 "k8s.io/api/batch/v1"
23-
corev1 "k8s.io/api/core/v1"
2423
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25-
"k8s.io/apimachinery/pkg/labels"
2624
)
2725

2826
func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job {
@@ -37,22 +35,3 @@ func GetJob(t Test, namespace, name string) *batchv1.Job {
3735
t.T().Helper()
3836
return Job(t, namespace, name)(t)
3937
}
40-
41-
func WriteJobLogs(t Test, namespace, name string) {
42-
t.T().Helper()
43-
44-
job := GetJob(t, namespace, name)
45-
46-
pods := GetPods(t, job.Namespace, metav1.ListOptions{
47-
LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)},
48-
)
49-
50-
if len(pods) == 0 {
51-
t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name)
52-
} else {
53-
for i, pod := range pods {
54-
t.T().Logf("Retrieving Pod %s/%s logs", pod.Namespace, pod.Name)
55-
WriteToOutputDir(t, pod.Name, Log, GetPodLogs(t, &pods[i], corev1.PodLogOptions{}))
56-
}
57-
}
58-
}

test/support/core.go

+32
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,35 @@ func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte {
5757

5858
return bytes
5959
}
60+
61+
func storeAllPodLogs(t Test, namespace *corev1.Namespace) {
62+
t.T().Helper()
63+
64+
pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), metav1.ListOptions{})
65+
t.Expect(err).NotTo(gomega.HaveOccurred())
66+
67+
for _, pod := range pods.Items {
68+
for _, container := range pod.Spec.Containers {
69+
t.T().Logf("Retrieving Pod Container %s/%s/%s logs", pod.Namespace, pod.Name, container.Name)
70+
storeContainerLog(t, namespace, pod.Name, container.Name)
71+
}
72+
}
73+
}
74+
75+
func storeContainerLog(t Test, namespace *corev1.Namespace, podName, containerName string) {
76+
t.T().Helper()
77+
78+
options := corev1.PodLogOptions{Container: containerName}
79+
stream, err := t.Client().Core().CoreV1().Pods(namespace.Name).GetLogs(podName, &options).Stream(t.Ctx())
80+
t.Expect(err).NotTo(gomega.HaveOccurred())
81+
82+
defer func() {
83+
t.Expect(stream.Close()).To(gomega.Succeed())
84+
}()
85+
86+
bytes, err := io.ReadAll(stream)
87+
t.Expect(err).NotTo(gomega.HaveOccurred())
88+
89+
containerLogFileName := "pod-" + podName + "-" + containerName
90+
WriteToOutputDir(t, containerLogFileName, Log, bytes)
91+
}

test/support/events.go

+148
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
Copyright 2023.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package support
18+
19+
import (
20+
"bytes"
21+
"fmt"
22+
23+
"github.com/onsi/gomega"
24+
25+
corev1 "k8s.io/api/core/v1"
26+
eventsv1 "k8s.io/api/events/v1"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
)
29+
30+
// Based on https://github.com/apache/incubator-kie-kogito-operator/blob/28b2d3dc945e48659b199cca33723568b848f72e/test/pkg/framework/logging.go
31+
32+
const (
33+
eventLastSeenKey = "LAST_SEEN"
34+
eventFirstSeenKey = "FIRST_SEEN"
35+
eventNameKey = "NAME"
36+
eventSubObjectKey = "SUBOBJECT"
37+
eventTypeKey = "TYPE"
38+
eventReasonKey = "REASON"
39+
eventMessageKey = "MESSAGE"
40+
41+
eventLogFileName = "events"
42+
)
43+
44+
var eventKeys = []string{
45+
eventLastSeenKey,
46+
eventFirstSeenKey,
47+
eventNameKey,
48+
eventSubObjectKey,
49+
eventTypeKey,
50+
eventReasonKey,
51+
eventMessageKey,
52+
}
53+
54+
func storeEvents(t Test, namespace *corev1.Namespace) {
55+
t.T().Helper()
56+
57+
events, err := t.Client().Core().EventsV1().Events(namespace.Name).List(t.Ctx(), metav1.ListOptions{})
58+
t.Expect(err).NotTo(gomega.HaveOccurred())
59+
60+
bytes, err := renderEventContent(eventKeys, mapEventsToKeys(events))
61+
t.Expect(err).NotTo(gomega.HaveOccurred())
62+
63+
WriteToOutputDir(t, eventLogFileName, Log, bytes)
64+
}
65+
66+
func mapEventsToKeys(eventList *eventsv1.EventList) []map[string]string {
67+
eventMaps := []map[string]string{}
68+
69+
for _, event := range eventList.Items {
70+
eventMap := make(map[string]string)
71+
eventMap[eventLastSeenKey] = getDefaultEventValueIfNull(event.DeprecatedLastTimestamp.Format("2006-01-02 15:04:05"))
72+
eventMap[eventFirstSeenKey] = getDefaultEventValueIfNull(event.DeprecatedFirstTimestamp.Format("2006-01-02 15:04:05"))
73+
eventMap[eventNameKey] = getDefaultEventValueIfNull(event.GetName())
74+
eventMap[eventSubObjectKey] = getDefaultEventValueIfNull(event.Regarding.FieldPath)
75+
eventMap[eventTypeKey] = getDefaultEventValueIfNull(event.Type)
76+
eventMap[eventReasonKey] = getDefaultEventValueIfNull(event.Reason)
77+
eventMap[eventMessageKey] = getDefaultEventValueIfNull(event.Note)
78+
79+
eventMaps = append(eventMaps, eventMap)
80+
}
81+
return eventMaps
82+
}
83+
84+
func getDefaultEventValueIfNull(value string) string {
85+
if len(value) <= 0 {
86+
return "-"
87+
}
88+
return value
89+
}
90+
91+
func renderEventContent(keys []string, dataMaps []map[string]string) ([]byte, error) {
92+
var content bytes.Buffer
93+
// Get size of strings to be written, to be able to format correctly
94+
maxStringSizeMap := make(map[string]int)
95+
for _, key := range keys {
96+
maxSize := len(key)
97+
for _, dataMap := range dataMaps {
98+
if len(dataMap[key]) > maxSize {
99+
maxSize = len(dataMap[key])
100+
}
101+
}
102+
maxStringSizeMap[key] = maxSize
103+
}
104+
105+
// Write headers
106+
for _, header := range keys {
107+
if _, err := content.WriteString(header); err != nil {
108+
return nil, fmt.Errorf("error in writing the header: %v", err)
109+
}
110+
if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[header] - len(header) + 1)); err != nil {
111+
return nil, fmt.Errorf("error in writing headers: %v", err)
112+
}
113+
if _, err := content.WriteString(" | "); err != nil {
114+
return nil, fmt.Errorf("error in writing headers : %v", err)
115+
}
116+
}
117+
if _, err := content.WriteString("\n"); err != nil {
118+
return nil, fmt.Errorf("error in writing headers '|': %v", err)
119+
120+
}
121+
122+
// Write events
123+
for _, dataMap := range dataMaps {
124+
for _, key := range keys {
125+
if _, err := content.WriteString(dataMap[key]); err != nil {
126+
return nil, fmt.Errorf("error in writing events: %v", err)
127+
}
128+
if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[key] - len(dataMap[key]) + 1)); err != nil {
129+
return nil, fmt.Errorf("error in writing events: %v", err)
130+
}
131+
if _, err := content.WriteString(" | "); err != nil {
132+
return nil, fmt.Errorf("error in writing events: %v", err)
133+
}
134+
}
135+
if _, err := content.WriteString("\n"); err != nil {
136+
return nil, fmt.Errorf("error in writing events: %v", err)
137+
}
138+
}
139+
return content.Bytes(), nil
140+
}
141+
142+
func getWhitespaceStr(size int) string {
143+
whiteSpaceStr := ""
144+
for i := 0; i < size; i++ {
145+
whiteSpaceStr += " "
146+
}
147+
return whiteSpaceStr
148+
}

test/support/ray_api.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) {
2929
t.T().Helper()
3030
logs, err := rayClient.GetJobLogs(jobID)
3131
t.Expect(err).NotTo(gomega.HaveOccurred())
32-
WriteToOutputDir(t, jobID, Log, []byte(logs))
32+
WriteToOutputDir(t, "ray-job-log-"+jobID, Log, []byte(logs))
3333
}
3434

3535
func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse {

test/support/test.go

+2
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@ func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Names
131131
t.T().Helper()
132132
namespace := createTestNamespace(t, options...)
133133
t.T().Cleanup(func() {
134+
storeAllPodLogs(t, namespace)
135+
storeEvents(t, namespace)
134136
deleteTestNamespace(t, namespace)
135137
})
136138
return namespace

0 commit comments

Comments
 (0)