Skip to content

Commit 9ce4f74

Browse files
sanchezlsanchezl
committedAug 26, 2021
add KubeletVersionSkewController
1 parent 8516fbf commit 9ce4f74

File tree

3 files changed

+443
-0
lines changed

3 files changed

+443
-0
lines changed
 

Diff for: ‎pkg/operator/kubeletversionskewcontroller/kubelet_version_skew_controller.go

+255
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
package kubeletversionskewcontroller
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"regexp"
7+
"sort"
8+
"strings"
9+
10+
"github.com/blang/semver"
11+
operatorv1 "github.com/openshift/api/operator/v1"
12+
"github.com/openshift/library-go/pkg/controller/factory"
13+
"github.com/openshift/library-go/pkg/operator/events"
14+
"github.com/openshift/library-go/pkg/operator/management"
15+
"github.com/openshift/library-go/pkg/operator/status"
16+
"github.com/openshift/library-go/pkg/operator/v1helpers"
17+
corev1 "k8s.io/api/core/v1"
18+
"k8s.io/apimachinery/pkg/labels"
19+
"k8s.io/apimachinery/pkg/util/runtime"
20+
corev1listers "k8s.io/client-go/listers/core/v1"
21+
)
22+
23+
const (
24+
KubeletMinorVersionUpgradeableConditionType = "KubeletMinorVersionUpgradeable"
25+
26+
KubeletVersionUnknownReason = "KubeletVersionUnknown"
27+
KubeletMinorVersionSyncedReason = "KubeletMinorVersionsSynced"
28+
KubeletMinorVersionSupportedNextUpgradeReason = "KubeletMinorVersionSupportedNextUpgrade"
29+
KubeletMinorVersionUnsupportedNextUpgradeReason = "KubeletMinorVersionUnsupportedNextUpgrade"
30+
KubeletMinorVersionUnsupportedReason = "KubeletMinorVersionUnsupported"
31+
KubeletMinorVersionAheadReason = "KubeletMinorVersionAhead"
32+
)
33+
34+
// KubeletVersionSkewController sets Upgradeable=False if the kubelet
35+
// version on a node prevents upgrading to a supported OpenShift version.
36+
//
37+
// For odd OpenShift minor versions, kubelet versions 0 or 1 minor
38+
// versions behind the API server version are supported.
39+
//
40+
// For even OpenShift minor versions, kubelet versions 0, 1, or 2
41+
// minor versions behind the API server version are supported.
42+
type KubeletVersionSkewController interface {
43+
factory.Controller
44+
}
45+
46+
func NewKubeletVersionSkewController(
47+
operatorClient v1helpers.OperatorClient,
48+
kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
49+
recorder events.Recorder,
50+
) *kubeletVersionSkewController {
51+
openShiftVersion := semver.MustParse(status.VersionForOperatorFromEnv())
52+
nextOpenShiftVersion := semver.Version{Major: openShiftVersion.Major, Minor: openShiftVersion.Minor + 1}
53+
c := &kubeletVersionSkewController{
54+
operatorClient: operatorClient,
55+
nodeLister: kubeInformersForNamespaces.InformersFor("").Core().V1().Nodes().Lister(),
56+
apiServerVersion: semver.MustParse(status.VersionForOperandFromEnv()),
57+
minSupportedSkew: minSupportedKubeletSkewForOpenShiftVersion(openShiftVersion),
58+
minSupportedSkewNextVersion: minSupportedKubeletSkewForOpenShiftVersion(nextOpenShiftVersion),
59+
}
60+
c.Controller = factory.New().
61+
WithSync(c.sync).
62+
WithInformers(kubeInformersForNamespaces.InformersFor("").Core().V1().Nodes().Informer()).
63+
ToController("KubeletVersionSkewController", recorder.WithComponentSuffix("kubelet-version-skew-controller"))
64+
return c
65+
}
66+
67+
func minSupportedKubeletSkewForOpenShiftVersion(v semver.Version) int {
68+
switch v.Minor % 2 {
69+
case 0: // even OpenShift versions
70+
return -2
71+
case 1: // odd OpenShift versions
72+
return -1
73+
default:
74+
panic("should not happen")
75+
}
76+
}
77+
78+
type kubeletVersionSkewController struct {
79+
factory.Controller
80+
operatorClient v1helpers.OperatorClient
81+
nodeLister corev1listers.NodeLister
82+
apiServerVersion semver.Version
83+
minSupportedSkew int
84+
minSupportedSkewNextVersion int
85+
}
86+
87+
func (c *kubeletVersionSkewController) sync(_ context.Context, _ factory.SyncContext) error {
88+
operatorSpec, _, _, err := c.operatorClient.GetOperatorState()
89+
if err != nil {
90+
return err
91+
}
92+
if !management.IsOperatorManaged(operatorSpec.ManagementState) {
93+
return nil
94+
}
95+
96+
nodes, err := c.nodeLister.List(labels.Everything())
97+
if err != nil {
98+
return err
99+
}
100+
sort.Sort(byName(nodes))
101+
102+
var errors nodeKubeletInfos
103+
var skewedUnsupported nodeKubeletInfos
104+
var skewedLimit nodeKubeletInfos
105+
var skewedButOK nodeKubeletInfos
106+
var synced nodeKubeletInfos
107+
var unsupported nodeKubeletInfos
108+
109+
// for each node, check kubelet version
110+
for _, node := range nodes {
111+
kubeletVersion, err := nodeKubeletVersion(node)
112+
if err != nil {
113+
runtime.HandleError(fmt.Errorf("unable to determine kubelet version on node %s: %w", node.Name, err))
114+
errors = append(errors, nodeKubeletInfo{node: node.Name, err: err})
115+
continue
116+
}
117+
skew := int(kubeletVersion.Minor - c.apiServerVersion.Minor)
118+
// Assume that an OpenShift minor version upgrade also bumps to the next kube minor version. Revisit
119+
// this in the future if an OpenShift minor version upgrade ever skips or repeats a kube minor version.
120+
skewNextVersion := skew - 1
121+
switch {
122+
case skew == 0:
123+
// synced
124+
synced = append(synced, nodeKubeletInfo{node: node.Name, version: &kubeletVersion})
125+
case skew < c.minSupportedSkew:
126+
// already in an unsupported state
127+
skewedUnsupported = append(skewedUnsupported, nodeKubeletInfo{node: node.Name, version: &kubeletVersion})
128+
case skewNextVersion < c.minSupportedSkewNextVersion:
129+
// upgrading to next minor version of API server would result in an unsupported config
130+
skewedLimit = append(skewedLimit, nodeKubeletInfo{node: node.Name, version: &kubeletVersion})
131+
case skew < 0:
132+
// behind, but upgrading to next minor version of API server is supported
133+
skewedButOK = append(skewedButOK, nodeKubeletInfo{node: node.Name, version: &kubeletVersion})
134+
default:
135+
// kubelet version newer than api server version. possibly in the middle of a rollback.
136+
unsupported = append(unsupported, nodeKubeletInfo{node: node.Name, version: &kubeletVersion})
137+
}
138+
}
139+
140+
condition := operatorv1.OperatorCondition{Type: KubeletMinorVersionUpgradeableConditionType}
141+
// use the most "severe" reason to set the condition status
142+
switch {
143+
case len(skewedUnsupported) > 0:
144+
condition.Reason = KubeletMinorVersionUnsupportedReason
145+
condition.Status = operatorv1.ConditionFalse
146+
switch len(skewedUnsupported) {
147+
case 1:
148+
condition.Message = fmt.Sprintf("Unsupported kubelet minor version (%v) on node %s is too far behind the target API server version (%v).", skewedUnsupported.version(), skewedUnsupported.nodes(), c.apiServerVersion)
149+
case 2, 3:
150+
condition.Message = fmt.Sprintf("Unsupported kubelet minor versions on nodes %s are too far behind the target API server version (%v).", skewedUnsupported.nodes(), c.apiServerVersion)
151+
default:
152+
condition.Message = fmt.Sprintf("Unsupported kubelet minor versions on %d nodes are too far behind the target API server version (%v).", len(skewedUnsupported), c.apiServerVersion)
153+
}
154+
case len(unsupported) > 0:
155+
condition.Reason = KubeletMinorVersionAheadReason
156+
condition.Status = operatorv1.ConditionUnknown
157+
switch len(unsupported) {
158+
case 1:
159+
condition.Message = fmt.Sprintf("Unsupported kubelet minor version (%v) on node %s is ahead of the target API server version (%v).", unsupported.version(), unsupported.nodes(), c.apiServerVersion)
160+
case 2, 3:
161+
condition.Message = fmt.Sprintf("Unsupported kubelet minor versions on nodes %s are ahead of the target API server version (%v).", unsupported.nodes(), c.apiServerVersion)
162+
default:
163+
condition.Message = fmt.Sprintf("Unsupported kubelet minor versions on %d nodes are ahead of the target API server version (%v).", len(unsupported), c.apiServerVersion)
164+
}
165+
case len(errors) > 0:
166+
condition.Reason = KubeletVersionUnknownReason
167+
condition.Status = operatorv1.ConditionUnknown
168+
switch len(errors) {
169+
case 1:
170+
condition.Message = fmt.Sprintf("Unable to determine the kubelet version on node %s: %v", errors.nodes(), errors.error())
171+
case 2, 3:
172+
condition.Message = fmt.Sprintf("Unable to determine the kubelet version on nodes %s.", errors.nodes())
173+
default:
174+
condition.Message = fmt.Sprintf("Unable to determine the kubelet version on %d nodes.", len(errors))
175+
}
176+
case len(skewedLimit) > 0:
177+
condition.Reason = KubeletMinorVersionUnsupportedNextUpgradeReason
178+
condition.Status = operatorv1.ConditionFalse
179+
switch len(skewedLimit) {
180+
case 1:
181+
condition.Message = fmt.Sprintf("Kubelet minor version (%v) on node %s will not be supported in the next OpenShift minor version upgrade.", skewedLimit.version(), skewedLimit.nodes())
182+
case 2, 3:
183+
condition.Message = fmt.Sprintf("Kubelet minor versions on nodes %s will not be supported in the next OpenShift minor version upgrade.", skewedLimit.nodes())
184+
default:
185+
condition.Message = fmt.Sprintf("Kubelet minor versions on %d nodes will not be supported in the next OpenShift minor version upgrade.", len(skewedLimit))
186+
}
187+
case len(skewedButOK) > 0:
188+
condition.Reason = KubeletMinorVersionSupportedNextUpgradeReason
189+
condition.Status = operatorv1.ConditionTrue
190+
switch len(skewedButOK) {
191+
case 1:
192+
condition.Message = fmt.Sprintf("Kubelet minor version (%v) on node %s is behind the expected API server version; nevertheless, it will continue to be supported in the next OpenShift minor version upgrade.", skewedButOK.version(), skewedButOK.nodes())
193+
case 2, 3:
194+
condition.Message = fmt.Sprintf("Kubelet minor versions on nodes %s are behind the expected API server version; nevertheless, they will continue to be supported in the next OpenShift minor version upgrade.", skewedButOK.nodes())
195+
default:
196+
condition.Message = fmt.Sprintf("Kubelet minor versions on %d nodes are behind the expected API server version; nevertheless, they will continue to be supported in the next OpenShift minor version upgrade.", len(skewedButOK))
197+
}
198+
default:
199+
condition.Reason = KubeletMinorVersionSyncedReason
200+
condition.Status = operatorv1.ConditionTrue
201+
condition.Message = "Kubelet and API server minor versions are synced."
202+
}
203+
204+
_, _, err = v1helpers.UpdateStatus(c.operatorClient, v1helpers.UpdateConditionFn(condition))
205+
return err
206+
}
207+
208+
type nodeKubeletInfo struct {
209+
node string
210+
version *semver.Version
211+
err error
212+
}
213+
214+
type nodeKubeletInfos []nodeKubeletInfo
215+
216+
func (n nodeKubeletInfos) nodes() string {
217+
var s []string
218+
for _, i := range n {
219+
s = append(s, i.node)
220+
}
221+
switch len(s) {
222+
case 0, 1:
223+
case 2:
224+
return strings.Join(s, " and ")
225+
default:
226+
s[len(s)-1] = "and " + s[len(s)-1]
227+
}
228+
return strings.Join(s, ", ")
229+
}
230+
231+
func (n nodeKubeletInfos) error() error {
232+
if len(n) > 0 {
233+
return n[0].err
234+
}
235+
return nil
236+
}
237+
238+
func (n nodeKubeletInfos) version() *semver.Version {
239+
if len(n) > 0 {
240+
return n[0].version
241+
}
242+
return nil
243+
}
244+
245+
func nodeKubeletVersion(node *corev1.Node) (semver.Version, error) {
246+
return semver.Parse(strings.TrimPrefix(node.Status.NodeInfo.KubeletVersion, "v"))
247+
}
248+
249+
var byNodeRegexp = regexp.MustCompile(`node [^ ]*`)
250+
251+
type byName []*corev1.Node
252+
253+
func (n byName) Len() int { return len(n) }
254+
func (n byName) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
255+
func (n byName) Less(i, j int) bool { return strings.Compare(n[i].Name, n[j].Name) < 0 }

Diff for: ‎pkg/operator/kubeletversionskewcontroller/kubelet_version_skew_controller_test.go

+180
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
package kubeletversionskewcontroller
2+
3+
import (
4+
"fmt"
5+
"testing"
6+
7+
"github.com/blang/semver"
8+
operatorv1 "github.com/openshift/api/operator/v1"
9+
"github.com/openshift/library-go/pkg/operator/v1helpers"
10+
corev1 "k8s.io/api/core/v1"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/apimachinery/pkg/util/diff"
13+
corev1listers "k8s.io/client-go/listers/core/v1"
14+
"k8s.io/client-go/tools/cache"
15+
)
16+
17+
func Test_kubeletVersionSkewController_Sync(t *testing.T) {
18+
19+
evenOpenShiftVersion := "4.8.0"
20+
oddOpenShiftVersion := "4.9.0"
21+
apiServerVersion := "1.21.1"
22+
skewedKubeletVersions := func(s ...int) []string {
23+
var v []string
24+
for i, skew := range s {
25+
v = append(v, fmt.Sprintf("1.%d.%d", 21+skew, i))
26+
}
27+
return v
28+
}
29+
30+
testCases := []struct {
31+
name string
32+
ocpVersion string
33+
kubeletVersions []string
34+
expectedStatus operatorv1.ConditionStatus
35+
expectedReason string
36+
expectedMsgLines string
37+
}{
38+
{
39+
name: "Synced/Even",
40+
ocpVersion: evenOpenShiftVersion,
41+
kubeletVersions: skewedKubeletVersions(0, 0, 0),
42+
expectedStatus: operatorv1.ConditionTrue,
43+
expectedReason: KubeletMinorVersionSyncedReason,
44+
expectedMsgLines: "Kubelet and API server minor versions are synced.",
45+
},
46+
{
47+
name: "Synced/Odd",
48+
ocpVersion: oddOpenShiftVersion,
49+
kubeletVersions: skewedKubeletVersions(0, 0, 0),
50+
expectedStatus: operatorv1.ConditionTrue,
51+
expectedReason: KubeletMinorVersionSyncedReason,
52+
expectedMsgLines: "Kubelet and API server minor versions are synced.",
53+
},
54+
{
55+
name: "ErrorParsingKubeletVersion",
56+
ocpVersion: oddOpenShiftVersion,
57+
kubeletVersions: []string{"Invalid", "1.21.2", "1.20.3"},
58+
expectedStatus: operatorv1.ConditionUnknown,
59+
expectedReason: KubeletVersionUnknownReason,
60+
expectedMsgLines: "Unable to determine the kubelet version on node test000: No Major.Minor.Patch elements found",
61+
},
62+
{
63+
name: "UnsupportedNextUpgrade/Even",
64+
ocpVersion: evenOpenShiftVersion,
65+
kubeletVersions: skewedKubeletVersions(0, -1, 0),
66+
expectedStatus: operatorv1.ConditionFalse,
67+
expectedReason: KubeletMinorVersionUnsupportedNextUpgradeReason,
68+
expectedMsgLines: "Kubelet minor version (1.20.1) on node test001 will not be supported in the next OpenShift minor version upgrade.",
69+
},
70+
{
71+
name: "UnsupportedNextUpgrade/Odd",
72+
ocpVersion: oddOpenShiftVersion,
73+
kubeletVersions: skewedKubeletVersions(0, -2, 0),
74+
expectedStatus: operatorv1.ConditionFalse,
75+
expectedReason: KubeletMinorVersionUnsupportedReason,
76+
expectedMsgLines: "Unsupported kubelet minor version (1.19.1) on node test001 is too far behind the target API server version (1.21.1).",
77+
},
78+
{
79+
name: "TwoNodesNotSynced",
80+
ocpVersion: evenOpenShiftVersion,
81+
kubeletVersions: skewedKubeletVersions(0, -1, -1),
82+
expectedStatus: operatorv1.ConditionFalse,
83+
expectedReason: KubeletMinorVersionUnsupportedNextUpgradeReason,
84+
expectedMsgLines: "Kubelet minor versions on nodes test001 and test002 will not be supported in the next OpenShift minor version upgrade.",
85+
},
86+
{
87+
name: "ThreeNodesNotSynced",
88+
ocpVersion: evenOpenShiftVersion,
89+
kubeletVersions: skewedKubeletVersions(0, -1, -1, -1),
90+
expectedStatus: operatorv1.ConditionFalse,
91+
expectedReason: KubeletMinorVersionUnsupportedNextUpgradeReason,
92+
expectedMsgLines: "Kubelet minor versions on nodes test001, test002, and test003 will not be supported in the next OpenShift minor version upgrade.",
93+
},
94+
{
95+
name: "ManyNodesNotSynced",
96+
ocpVersion: evenOpenShiftVersion,
97+
kubeletVersions: skewedKubeletVersions(0, -1, -1, -1, -1, -1, 0, 0),
98+
expectedStatus: operatorv1.ConditionFalse,
99+
expectedReason: KubeletMinorVersionUnsupportedNextUpgradeReason,
100+
expectedMsgLines: "Kubelet minor versions on 5 nodes will not be supported in the next OpenShift minor version upgrade.",
101+
},
102+
{
103+
name: "SkewedUnsupported/Even",
104+
ocpVersion: evenOpenShiftVersion,
105+
kubeletVersions: skewedKubeletVersions(0, -3, 0),
106+
expectedStatus: operatorv1.ConditionFalse,
107+
expectedReason: KubeletMinorVersionUnsupportedReason,
108+
expectedMsgLines: "Unsupported kubelet minor version (1.18.1) on node test001 is too far behind the target API server version (1.21.1).",
109+
},
110+
{
111+
name: "SkewedUnsupported/Odd",
112+
ocpVersion: oddOpenShiftVersion,
113+
kubeletVersions: skewedKubeletVersions(0, -2, 0),
114+
expectedStatus: operatorv1.ConditionFalse,
115+
expectedReason: KubeletMinorVersionUnsupportedReason,
116+
expectedMsgLines: "Unsupported kubelet minor version (1.19.1) on node test001 is too far behind the target API server version (1.21.1).",
117+
},
118+
{
119+
name: "SkewedButOK/Odd",
120+
ocpVersion: oddOpenShiftVersion,
121+
kubeletVersions: skewedKubeletVersions(-1, 0, 0),
122+
expectedStatus: operatorv1.ConditionTrue,
123+
expectedReason: KubeletMinorVersionSupportedNextUpgradeReason,
124+
expectedMsgLines: "Kubelet minor version (1.20.0) on node test000 is behind the expected API server version; nevertheless, it will continue to be supported in the next OpenShift minor version upgrade.",
125+
},
126+
{
127+
name: "Unsupported",
128+
ocpVersion: oddOpenShiftVersion,
129+
kubeletVersions: skewedKubeletVersions(0, -1, 1),
130+
expectedStatus: operatorv1.ConditionUnknown,
131+
expectedReason: KubeletMinorVersionAheadReason,
132+
expectedMsgLines: "Unsupported kubelet minor version (1.22.2) on node test002 is ahead of the target API server version (1.21.1).",
133+
},
134+
}
135+
for _, tc := range testCases {
136+
t.Run(tc.name, func(t *testing.T) {
137+
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{})
138+
for i, kv := range tc.kubeletVersions {
139+
indexer.Add(&corev1.Node{
140+
ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("test%03d", i)},
141+
Status: corev1.NodeStatus{NodeInfo: corev1.NodeSystemInfo{KubeletVersion: kv}},
142+
})
143+
}
144+
status := &operatorv1.StaticPodOperatorStatus{}
145+
ocpVersion := semver.MustParse(tc.ocpVersion)
146+
nextOpenShiftVersion := semver.Version{Major: ocpVersion.Major, Minor: ocpVersion.Minor + 1}
147+
c := &kubeletVersionSkewController{
148+
operatorClient: v1helpers.NewFakeStaticPodOperatorClient(
149+
&operatorv1.StaticPodOperatorSpec{OperatorSpec: operatorv1.OperatorSpec{ManagementState: operatorv1.Managed}},
150+
status, nil, nil,
151+
),
152+
nodeLister: corev1listers.NewNodeLister(indexer),
153+
apiServerVersion: semver.MustParse(apiServerVersion),
154+
minSupportedSkew: minSupportedKubeletSkewForOpenShiftVersion(ocpVersion),
155+
minSupportedSkewNextVersion: minSupportedKubeletSkewForOpenShiftVersion(nextOpenShiftVersion),
156+
}
157+
err := c.sync(nil, nil)
158+
if err != nil {
159+
t.Fatalf("sync() unexpected err: %v", err)
160+
}
161+
if len(status.Conditions) != 1 || status.Conditions[0].Type != KubeletMinorVersionUpgradeableConditionType {
162+
t.Errorf("Expected %s condition type.", KubeletMinorVersionUpgradeableConditionType)
163+
}
164+
condition := status.Conditions[0]
165+
if tc.expectedStatus != condition.Status {
166+
t.Errorf("Condition status: expected %s, actual %s", tc.expectedStatus, condition.Status)
167+
}
168+
if tc.expectedReason != condition.Reason {
169+
t.Errorf("Condition reason: expected %s, actual %s", tc.expectedReason, condition.Reason)
170+
}
171+
if tc.expectedMsgLines != condition.Message {
172+
t.Errorf("Expected condition message to match %q.", tc.expectedMsgLines)
173+
t.Log(diff.StringDiff(tc.expectedMsgLines, condition.Message))
174+
}
175+
if t.Failed() {
176+
t.Logf(condition.Message)
177+
}
178+
})
179+
}
180+
}

Diff for: ‎pkg/operator/starter.go

+8
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation/configobservercontroller"
2121
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/connectivitycheckcontroller"
2222
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/featureupgradablecontroller"
23+
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/kubeletversionskewcontroller"
2324
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/nodekubeconfigcontroller"
2425
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient"
2526
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/resourcesynccontroller"
@@ -335,6 +336,12 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
335336
controllerContext.EventRecorder,
336337
)
337338

339+
kubeletVersionSkewController := kubeletversionskewcontroller.NewKubeletVersionSkewController(
340+
operatorClient,
341+
kubeInformersForNamespaces,
342+
controllerContext.EventRecorder,
343+
)
344+
338345
// register termination metrics
339346
terminationobserver.RegisterMetrics()
340347

@@ -364,6 +371,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
364371
go auditPolicyController.Run(ctx, 1)
365372
go staleConditionsController.Run(ctx, 1)
366373
go connectivityCheckController.Run(ctx, 1)
374+
go kubeletVersionSkewController.Run(ctx, 1)
367375

368376
<-ctx.Done()
369377
return nil

0 commit comments

Comments
 (0)
Please sign in to comment.