Skip to content

Commit 7831f04

Browse files
mgencurmgencur
committed
CNTRLPLANE-249 Build CI to cover Cilium CNI test for HCP on kubevirt platform
This commit brings a workflow for running conformance tests on hosted cluster created by kubevirt. The manamenent cluster is bare metal. * Introduce step hypershift-kubevirt-health-check-nodecount for cases when HostedCluster doesn't get Complete until Cilium CNI (or other network stack) is installed on Nodes. In this case, we only check that number of available nodes matches the one from HostedCluster resource. * Exclude same tests as for hypershift-aws-conformance-cilium workflow as they fail for Cilium in general (not specific for kubevirt) * Exclude "StatefulSet Basic" and "StatefulSet Non-retain". Copied from hypershift-kubevirt-baremetalds-conformance workflow.These tests are flaky on Kubevirt. * Exclude "[Feature:bond]" until openshift/origin#29630 is merged.
1 parent 1c5e1ec commit 7831f04

11 files changed

+255
-0
lines changed

‎ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.19__periodics.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,16 @@ tests:
162162
ODF_OPERATOR_SUB_SOURCE: redhat-operators-v4-17
163163
REDHAT_OPERATORS_INDEX_TAG: v4.17
164164
workflow: hypershift-kubevirt-baremetalds-conformance
165+
- as: e2e-kubevirt-metal-conformance-cilium
166+
minimum_interval: 168h
167+
steps:
168+
cluster_profile: equinix-ocp-hcp
169+
env:
170+
LVM_OPERATOR_SUB_CHANNEL: stable-4.19
171+
ODF_OPERATOR_SUB_CHANNEL: stable-4.18
172+
ODF_OPERATOR_SUB_SOURCE: redhat-operators-v4-18
173+
REDHAT_OPERATORS_INDEX_TAG: v4.18
174+
workflow: hypershift-kubevirt-baremetalds-conformance-cilium
165175
- as: e2e-kubevirt-metal-ovn-disconnected
166176
cron: 0 8 * * *
167177
steps:

‎ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-4.19-periodics.yaml

+75
Original file line numberDiff line numberDiff line change
@@ -1101,6 +1101,81 @@ periodics:
11011101
- name: result-aggregator
11021102
secret:
11031103
secretName: result-aggregator
1104+
- agent: kubernetes
1105+
cluster: build11
1106+
decorate: true
1107+
decoration_config:
1108+
skip_cloning: true
1109+
extra_refs:
1110+
- base_ref: release-4.19
1111+
org: openshift
1112+
repo: hypershift
1113+
labels:
1114+
ci-operator.openshift.io/cloud: equinix-ocp-metal
1115+
ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-hcp
1116+
ci-operator.openshift.io/variant: periodics
1117+
ci.openshift.io/generator: prowgen
1118+
job-release: "4.19"
1119+
pj-rehearse.openshift.io/can-be-rehearsed: "true"
1120+
minimum_interval: 168h
1121+
name: periodic-ci-openshift-hypershift-release-4.19-periodics-e2e-kubevirt-metal-conformance-cilium
1122+
spec:
1123+
containers:
1124+
- args:
1125+
- --gcs-upload-secret=/secrets/gcs/service-account.json
1126+
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
1127+
- --lease-server-credentials-file=/etc/boskos/credentials
1128+
- --report-credentials-file=/etc/report/credentials
1129+
- --secret-dir=/secrets/ci-pull-credentials
1130+
- --target=e2e-kubevirt-metal-conformance-cilium
1131+
- --variant=periodics
1132+
command:
1133+
- ci-operator
1134+
image: ci-operator:latest
1135+
imagePullPolicy: Always
1136+
name: ""
1137+
resources:
1138+
requests:
1139+
cpu: 10m
1140+
volumeMounts:
1141+
- mountPath: /etc/boskos
1142+
name: boskos
1143+
readOnly: true
1144+
- mountPath: /secrets/ci-pull-credentials
1145+
name: ci-pull-credentials
1146+
readOnly: true
1147+
- mountPath: /secrets/gcs
1148+
name: gcs-credentials
1149+
readOnly: true
1150+
- mountPath: /secrets/manifest-tool
1151+
name: manifest-tool-local-pusher
1152+
readOnly: true
1153+
- mountPath: /etc/pull-secret
1154+
name: pull-secret
1155+
readOnly: true
1156+
- mountPath: /etc/report
1157+
name: result-aggregator
1158+
readOnly: true
1159+
serviceAccountName: ci-operator
1160+
volumes:
1161+
- name: boskos
1162+
secret:
1163+
items:
1164+
- key: credentials
1165+
path: credentials
1166+
secretName: boskos-credentials
1167+
- name: ci-pull-credentials
1168+
secret:
1169+
secretName: ci-pull-credentials
1170+
- name: manifest-tool-local-pusher
1171+
secret:
1172+
secretName: manifest-tool-local-pusher
1173+
- name: pull-secret
1174+
secret:
1175+
secretName: registry-pull-credentials
1176+
- name: result-aggregator
1177+
secret:
1178+
secretName: result-aggregator
11041179
- agent: kubernetes
11051180
cluster: build11
11061181
cron: 0 4 * * *

‎ci-operator/step-registry/hypershift/kubevirt/baremetalds/conformance-cilium/OWNERS

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../OWNERS

‎ci-operator/step-registry/hypershift/kubevirt/baremetalds/conformance-cilium/hypershift-kubevirt-baremetalds-conformance-cilium-workflow.metadata.json

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"path": "hypershift/kubevirt/baremetalds/conformance-cilium/hypershift-kubevirt-baremetalds-conformance-cilium-workflow.yaml",
3+
"owners": {
4+
"approvers": [
5+
"davidvossel",
6+
"nirarg",
7+
"nunnatsa",
8+
"qinqon",
9+
"orenc1",
10+
"LiangquanLi930"
11+
],
12+
"reviewers": [
13+
"davidvossel",
14+
"nirarg",
15+
"nunnatsa",
16+
"qinqon",
17+
"orenc1",
18+
"LiangquanLi930"
19+
]
20+
}
21+
}

‎ci-operator/step-registry/hypershift/kubevirt/baremetalds/conformance-cilium/hypershift-kubevirt-baremetalds-conformance-cilium-workflow.yaml

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
workflow:
2+
as: hypershift-kubevirt-baremetalds-conformance-cilium
3+
documentation: |-
4+
Executes tests against a new ephemeral HyperShift cluster with Cilium CNI.
5+
Administrative access to the control plane is provided via the `KUBECONFIG`
6+
environment variable.
7+
8+
Learn more about HyperShift here: https://github.com/openshift/hypershift
9+
10+
Track HyperShift's development here: https://issues.redhat.com/projects/HOSTEDCP
11+
steps:
12+
post:
13+
- chain: hypershift-dump
14+
- chain: gather-core-dump
15+
- chain: hypershift-kubevirt-destroy
16+
- chain: baremetalds-ipi-post
17+
test:
18+
- chain: hypershift-conformance
19+
pre:
20+
- chain: baremetalds-ipi-pre
21+
- ref: enable-qe-catalogsource
22+
- chain: hypershift-kubevirt-baremetalds-lvm
23+
- chain: hypershift-kubevirt-baremetalds-metallb
24+
- chain: hypershift-kubevirt-baremetalds-odf
25+
- ref: hypershift-kubevirt-install
26+
- ref: hypershift-install
27+
- ref: hypershift-kubevirt-create
28+
- ref: hypershift-kubevirt-baremetalds-proxy
29+
- ref: hypershift-kubevirt-health-check-nodecount
30+
- ref: cucushift-hypershift-extended-cilium
31+
- ref: cucushift-hypershift-extended-cilium-health-check
32+
env:
33+
HYPERSHIFT_NETWORK_TYPE: "Other" # Required for Cilium.
34+
METALLB_OPERATOR_SUB_SOURCE: qe-app-registry
35+
LOCAL_STORAGE_OPERATOR_SUB_SOURCE: qe-app-registry
36+
LVM_OPERATOR_SUB_CHANNEL: stable-4.19
37+
LVM_OPERATOR_SUB_SOURCE: qe-app-registry
38+
LVM_OPERATOR_SUB_INSTALL_NAMESPACE: openshift-lvm-storage
39+
ODF_OPERATOR_SUB_SOURCE: redhat-operators-v4-18
40+
ETCD_STORAGE_CLASS: lvms-vg1
41+
TEST_SKIPS: Netpol NetworkPolicy between server and client should allow egress
42+
access to server in CIDR block \[Feature:NetworkPolicy\] \[Suite:openshift/conformance/parallel\]
43+
\[Suite:k8s\]\| Netpol NetworkPolicy between server and client should ensure
44+
an IP overlapping both IPBlock.CIDR and IPBlock.Except is allowed \[Feature:NetworkPolicy\]
45+
\[Suite:openshift/conformance/parallel\] \[Suite:k8s\]\| Services should serve
46+
endpoints on same port and different protocols \[Conformance\] \[Suite:openshift/conformance/parallel/minimal\]
47+
\[Suite:k8s\]\| Netpol NetworkPolicy between server and client should enforce
48+
except clause while egress access to server in CIDR block \[Feature:NetworkPolicy\]
49+
\[Suite:openshift/conformance/parallel\] \[Suite:k8s\]\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
50+
should work with UDP \[Suite:openshift/conformance/parallel\]\| Unidling with
51+
Deployments \[apigroup:route.openshift.io\] should work with TCP (when fully
52+
idled) \[Suite:openshift/conformance/parallel\]\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
53+
should work with TCP (when fully idled) \[Suite:openshift/conformance/parallel\]\|
54+
Unidling with Deployments \[apigroup:route.openshift.io\] should work with
55+
UDP \[Suite:openshift/conformance/parallel\]\| DNS should answer queries using
56+
the local DNS endpoint \[Suite:openshift/conformance/parallel\] \| \[Feature:bond\] \| StatefulSet Basic \| StatefulSet Non-retain
57+
PACKET_OS: rocky_9
58+
DEVSCRIPTS_CONFIG: |
59+
IP_STACK=v4
60+
NETWORK_TYPE=OVNKubernetes
61+
NUM_WORKERS=0
62+
NUM_MASTERS=3
63+
MASTER_VCPU=16
64+
MASTER_MEMORY=81920
65+
VM_EXTRADISKS=true
66+
VM_EXTRADISKS_LIST="vda vdb"
67+
VM_EXTRADISKS_SIZE=250G

‎ci-operator/step-registry/hypershift/kubevirt/create/hypershift-kubevirt-create-commands.sh

+3
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,9 @@ if [[ -f "${SHARED_DIR}/GPU_DEVICE_NAME" ]]; then
172172
EXTRA_ARGS="${EXTRA_ARGS} --host-device-name $(cat "${SHARED_DIR}/GPU_DEVICE_NAME"),count:2"
173173
fi
174174

175+
if [[ -n "${HYPERSHIFT_NETWORK_TYPE}" ]]; then
176+
EXTRA_ARGS="${EXTRA_ARGS} --network-type=${HYPERSHIFT_NETWORK_TYPE}"
177+
fi
175178

176179
echo "$(date) Creating HyperShift guest cluster ${CLUSTER_NAME}"
177180
# Workaround for: https://issues.redhat.com/browse/OCPBUGS-42867

‎ci-operator/step-registry/hypershift/kubevirt/create/hypershift-kubevirt-create-ref.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ ref:
1414
- name: HYPERSHIFT_NODE_CPU_CORES
1515
default: "4"
1616
documentation: "The number of CPU cores that will be assigned for the kubevirt hosted cluster's nodes."
17+
- name: HYPERSHIFT_NETWORK_TYPE
18+
default: ""
19+
documentation: "Specifies the cluster SDN provider."
1720
- name: RUN_HOSTEDCLUSTER_CREATION
1821
default: "true"
1922
documentation: |-

‎ci-operator/step-registry/hypershift/kubevirt/health-check/nodecount/OWNERS

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
approvers:
2+
- davidvossel
3+
- nirarg
4+
- nunnatsa
5+
- qinqon
6+
- orenc1
7+
- LiangquanLi930
8+
options: {}
9+
reviewers:
10+
- davidvossel
11+
- nirarg
12+
- nunnatsa
13+
- qinqon
14+
- orenc1
15+
- LiangquanLi930

‎ci-operator/step-registry/hypershift/kubevirt/health-check/nodecount/hypershift-kubevirt-health-check-nodecount-commands.sh

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
3+
set -o nounset
4+
set -o errexit
5+
set -o pipefail
6+
7+
MCE=${MCE_VERSION:-""}
8+
CLUSTER_NAME="$(echo -n $PROW_JOB_ID|sha256sum|cut -c-20)"
9+
if [[ -n ${MCE} ]] ; then
10+
CLUSTER_NAMESPACE_PREFIX=local-cluster
11+
else
12+
CLUSTER_NAMESPACE_PREFIX=clusters
13+
fi
14+
CLUSTER_NAMESPACE=${CLUSTER_NAMESPACE_PREFIX}-${CLUSTER_NAME}
15+
16+
echo "Waiting for nested cluster's node count to reach the desired replicas count in the NodePool"
17+
until \
18+
[[ $(oc get nodepool ${CLUSTER_NAME} -n ${CLUSTER_NAMESPACE_PREFIX} -o jsonpath='{.spec.replicas}') \
19+
== $(oc --kubeconfig=${SHARED_DIR}/nested_kubeconfig get nodes --no-headers | wc -l) ]]; do
20+
echo "$(date --rfc-3339=seconds) Nested cluster's node count is not equal to the desired replicas in the NodePool. Retrying in 30 seconds."
21+
oc get vmi -n ${CLUSTER_NAMESPACE}
22+
sleep 30s
23+
done

‎ci-operator/step-registry/hypershift/kubevirt/health-check/nodecount/hypershift-kubevirt-health-check-nodecount-ref.metadata.json

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"path": "hypershift/kubevirt/health-check/nodecount/hypershift-kubevirt-health-check-nodecount-ref.yaml",
3+
"owners": {
4+
"approvers": [
5+
"davidvossel",
6+
"nirarg",
7+
"nunnatsa",
8+
"qinqon",
9+
"orenc1",
10+
"LiangquanLi930"
11+
],
12+
"reviewers": [
13+
"davidvossel",
14+
"nirarg",
15+
"nunnatsa",
16+
"qinqon",
17+
"orenc1",
18+
"LiangquanLi930"
19+
]
20+
}
21+
}

‎ci-operator/step-registry/hypershift/kubevirt/health-check/nodecount/hypershift-kubevirt-health-check-nodecount-ref.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
ref:
2+
as: hypershift-kubevirt-health-check-nodecount
3+
from: cli
4+
grace_period: 10m
5+
timeout: 45m0s
6+
env:
7+
- name: MCE_VERSION
8+
default: ""
9+
documentation: "version of the mce if installed. (\"\", 2.2, 2.3)"
10+
commands: hypershift-kubevirt-health-check-nodecount-commands.sh
11+
resources:
12+
requests:
13+
cpu: 100m
14+
memory: 100Mi
15+
documentation: |-
16+
HyperShift HostedCluster health - check node count against HostedCluster resource.

0 commit comments

Comments
 (0)