From ecac8516402345e593f15d602c63e66f9ce656fb Mon Sep 17 00:00:00 2001 From: Maciej Szulik Date: Tue, 4 Oct 2022 10:04:00 +0200 Subject: [PATCH 1/2] Update API alerts after recent bump to k8s 1.25 --- bindata/assets/alerts/api-usage.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bindata/assets/alerts/api-usage.yaml b/bindata/assets/alerts/api-usage.yaml index 998c8a67bd..715201ba39 100644 --- a/bindata/assets/alerts/api-usage.yaml +++ b/bindata/assets/alerts/api-usage.yaml @@ -16,7 +16,7 @@ spec: a successful upgrade to the next cluster version. Refer to `oc get apirequestcounts {{ $labels.resource }}.{{ $labels.version }}.{{ $labels.group }} -o yaml` to identify the workload. expr: | - group(apiserver_requested_deprecated_apis{removed_release="1.25"}) by (group,version,resource) and (sum by(group,version,resource) (rate(apiserver_request_total{system_client!="kube-controller-manager",system_client!="cluster-policy-controller"}[4h]))) > 0 + group(apiserver_requested_deprecated_apis{removed_release="1.26"}) by (group,version,resource) and (sum by(group,version,resource) (rate(apiserver_request_total{system_client!="kube-controller-manager",system_client!="cluster-policy-controller"}[4h]))) > 0 for: 1h labels: namespace: openshift-kube-apiserver @@ -30,7 +30,7 @@ spec: a successful upgrade to the next EUS cluster version. Refer to `oc get apirequestcounts {{ $labels.resource }}.{{ $labels.version }}.{{ $labels.group }} -o yaml` to identify the workload. expr: | - group(apiserver_requested_deprecated_apis{removed_release=~"1\\.2[5]"}) by (group,version,resource) and (sum by(group,version,resource) (rate(apiserver_request_total{system_client!="kube-controller-manager",system_client!="cluster-policy-controller"}[4h]))) > 0 + group(apiserver_requested_deprecated_apis{removed_release=~"1\\.2[67]"}) by (group,version,resource) and (sum by(group,version,resource) (rate(apiserver_request_total{system_client!="kube-controller-manager",system_client!="cluster-policy-controller"}[4h]))) > 0 for: 1h labels: From 3d985c7f15d5a56ee1c7e706185af2e0ffbf4829 Mon Sep 17 00:00:00 2001 From: Maciej Szulik Date: Wed, 5 Oct 2022 12:19:58 +0200 Subject: [PATCH 2/2] Improve CertRotation test to check reason condition --- test/e2e/certrotation_test.go | 75 +++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/test/e2e/certrotation_test.go b/test/e2e/certrotation_test.go index f3227159dc..b8f1cc6431 100644 --- a/test/e2e/certrotation_test.go +++ b/test/e2e/certrotation_test.go @@ -2,11 +2,20 @@ package e2e import ( "context" + "fmt" + "strings" "testing" "time" + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + configv1 "github.com/openshift/api/config/v1" operatorv1 "github.com/openshift/api/operator/v1" configclient "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" "github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient" @@ -14,11 +23,6 @@ import ( configv1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers" "github.com/openshift/library-go/pkg/operator/genericoperatorclient" "github.com/openshift/library-go/pkg/operator/v1helpers" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" ) func TestCertRotationTimeUpgradeable(t *testing.T) { @@ -29,12 +33,13 @@ func TestCertRotationTimeUpgradeable(t *testing.T) { configClient, err := configclient.NewForConfig(kubeConfig) require.NoError(t, err) - ctx := context.TODO() + ctx := context.Background() _, operatorStatus, _, err := operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) require.NoError(t, err) require.True(t, v1helpers.IsOperatorConditionTrue(operatorStatus.Conditions, "CertRotationTimeUpgradeable")) kubeClient := kubernetes.NewForConfigOrDie(kubeConfig) + t.Logf("Creating unsupported-cert-rotation-config...") _, err = kubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).Create(context.TODO(), &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "unsupported-cert-rotation-config"}, Data: map[string]string{"base": "2y"}, @@ -44,28 +49,56 @@ func TestCertRotationTimeUpgradeable(t *testing.T) { kubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).Delete(context.TODO(), "unsupported-cert-rotation-config", metav1.DeleteOptions{}) }() - // TODO better detection maybe someday - time.Sleep(5 * time.Second) + err = wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) { + _, operatorStatus, _, err := operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) + if err != nil { + return false, err + } + clusteroperator, err := configClient.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) + if err != nil { + return false, err + } - _, operatorStatus, _, err = operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) - require.NoError(t, err) - require.True(t, v1helpers.IsOperatorConditionFalse(operatorStatus.Conditions, "CertRotationTimeUpgradeable")) - clusteroperator, err := configClient.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) + certRotationCondition := v1helpers.FindOperatorCondition(operatorStatus.Conditions, "CertRotationTimeUpgradeable") + upgradeableCondition := configv1helpers.FindStatusCondition(clusteroperator.Status.Conditions, "Upgradeable") + if certRotationCondition == nil || upgradeableCondition == nil { + return false, fmt.Errorf("Couldn't find CertRotationTimeUpgradeable or Upgradeable condition") + } + if certRotationCondition.Status == operatorv1.ConditionFalse && + upgradeableCondition.Status == configv1.ConditionFalse && strings.Contains(upgradeableCondition.Reason, "CertRotationTime") { + return true, nil + } + t.Logf("\nCertRotationTimeUpgradeable: %#v\nUpgradeable: %#v", certRotationCondition, upgradeableCondition) + return false, nil + }) require.NoError(t, err) - require.True(t, configv1helpers.IsStatusConditionFalse(clusteroperator.Status.Conditions, "Upgradeable")) + t.Logf("Removing unsupported-cert-rotation-config...") err = kubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).Delete(context.TODO(), "unsupported-cert-rotation-config", metav1.DeleteOptions{}) require.NoError(t, err) - // TODO better detection maybe someday - time.Sleep(5 * time.Second) - _, operatorStatus, _, err = operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) - require.NoError(t, err) - require.True(t, v1helpers.IsOperatorConditionTrue(operatorStatus.Conditions, "CertRotationTimeUpgradeable")) - clusteroperator, err = configClient.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) + err = wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) { + _, operatorStatus, _, err := operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) + if err != nil { + return false, err + } + clusteroperator, err := configClient.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) + if err != nil { + return false, err + } + certRotationCondition := v1helpers.FindOperatorCondition(operatorStatus.Conditions, "CertRotationTimeUpgradeable") + upgradeableCondition := configv1helpers.FindStatusCondition(clusteroperator.Status.Conditions, "Upgradeable") + if certRotationCondition == nil || upgradeableCondition == nil { + return false, fmt.Errorf("Couldn't find CertRotationTimeUpgradeable or Upgradeable condition") + } + if certRotationCondition.Status == operatorv1.ConditionTrue && + (upgradeableCondition.Status == configv1.ConditionTrue || !strings.Contains(upgradeableCondition.Reason, "CertRotationTime")) { + return true, nil + } + t.Logf("\nCertRotationTimeUpgradeable: %#v\nUpgradeable: %#v", certRotationCondition, upgradeableCondition) + return false, nil + }) require.NoError(t, err) - require.True(t, configv1helpers.IsStatusConditionTrue(clusteroperator.Status.Conditions, "Upgradeable")) - } func TestCertRotationStompOnBadType(t *testing.T) {