@@ -97,6 +97,7 @@ var _ = SIGDescribe(feature.CriProxy, framework.WithSerial(), func() {
97
97
})
98
98
99
99
ginkgo .It ("Image pull retry backs off on error." , func (ctx context.Context ) {
100
+ // inject PullImage failed to trigger backoff
100
101
expectedErr := fmt .Errorf ("PullImage failed" )
101
102
err := addCRIProxyInjector (func (apiName string ) error {
102
103
if apiName == criproxy .PullImage {
@@ -120,15 +121,22 @@ var _ = SIGDescribe(feature.CriProxy, framework.WithSerial(), func() {
120
121
isExpectedErrMsg := strings .Contains (eventMsg , expectedErr .Error ())
121
122
gomega .Expect (isExpectedErrMsg ).To (gomega .BeTrueBecause ("we injected an exception into the PullImage interface of the cri proxy" ))
122
123
123
- // remove error so after backoff we will succeed
124
+ // Wait for ~40s worth of backoffs to occur so we can confirm the backoff growth.
125
+ podErr = e2epod .WaitForPodContainerStarted (ctx , f .ClientSet , f .Namespace .Name , pod .Name , 0 , 40 * time .Second )
126
+ gomega .Expect (podErr ).To (gomega .HaveOccurred (), "Expected container not to start from repeadedly backing off image pulls" )
127
+
128
+ // Remove error so after next backoff we will succeed.
124
129
_ = resetCRIProxyInjector ()
125
130
126
131
podErr = e2epod .WaitForPodRunningInNamespace (ctx , f .ClientSet , pod )
127
132
framework .ExpectNoError (podErr )
128
133
134
+ // Parse observed backoffs (TODO: don't use Events, but logs) and compare to expectations
129
135
durations , err := getImageBackOffDurations (ctx , f , pod .Name )
130
136
framework .ExpectNoError (err )
137
+ gomega .Expect (durations ).Error ().ShouldNot (gomega .BeNil (), "Should have observed backoffs in Pod event log" )
131
138
gomega .Expect (durations [0 ]).Should (gomega .BeNumerically ("~" , time .Duration (10 * time .Second ), time .Duration (2 * time .Second )))
139
+ // TODO: and check the next set of durations are 2x, etc
132
140
133
141
})
134
142
})
@@ -191,25 +199,26 @@ func getImageBackOffDurations(ctx context.Context, f *framework.Framework, podNa
191
199
var backoffs []time.Duration
192
200
193
201
type BackOffRecord struct {
194
- podName string
195
202
initialEventTime time.Time
196
203
backoffEventTimes []time.Time
197
204
duration time.Duration
198
205
}
199
206
200
207
records := make (map [int ]* BackOffRecord )
208
+ records [0 ] = & BackOffRecord {}
201
209
var backoffCount int
202
210
var pullTime time.Time
203
211
var r * BackOffRecord
212
+ // I'm doing this here for events but really it needs to be off kubelet logs or some kind of synchronous watch
213
+ // Because the events normalize to the latest
214
+ // But this is the idea
204
215
for _ , event := range events .Items {
205
216
if event .InvolvedObject .Name == podName {
206
-
207
217
switch event .Reason {
208
218
case kubeletevents .PullingImage :
209
219
if ! pullTime .IsZero () {
210
220
if event .FirstTimestamp .Time .After (pullTime ) {
211
221
r = records [backoffCount ]
212
- r .podName = podName
213
222
r .duration = r .initialEventTime .Sub (r .backoffEventTimes [len (r .backoffEventTimes )- 1 ])
214
223
backoffs = append (backoffs , r .duration )
215
224
backoffCount ++
0 commit comments