Skip to content

Commit 0bc4171

Browse files
psschweipsschwei
authored
Adds progressDeadline annotation on per revision basis (#12751)
* adds progressDeadline annotation on per revision basis Signed-off-by: Paul S. Schweigert <[email protected]> This PR allows setting the progress-deadline on each revision by passing an annotation, autoscaling.knative.dev/progressDeadline. It aims to improve the experience where revisions don't fail until the progressDeadline is reached, which by default is 10 minutes. Not everyone necessarily wants to wait that long, but previously the only resource was for the cluster admin to lower the deadline universally (i.e. change the default). With this change, we allow users to "fail fast", if they so desire. * remove unnecessary conversion Signed-off-by: Paul S. Schweigert <[email protected]> * address review pt1 Signed-off-by: Paul S. Schweigert <[email protected]> * unit test Signed-off-by: Paul S. Schweigert <[email protected]> * fixing annotation in tests Signed-off-by: Paul S. Schweigert <[email protected]> * update key to serving.knative.dev Signed-off-by: Paul S. Schweigert <[email protected]> * move annotation to serving register Signed-off-by: Paul S. Schweigert <[email protected]> * validate revisions, not services or routes Signed-off-by: Paul S. Schweigert <[email protected]>
1 parent 082ed55 commit 0bc4171

File tree

11 files changed

+241
-8
lines changed

11 files changed

+241
-8
lines changed

Diff for: ‎pkg/apis/autoscaling/v1alpha1/pa_lifecycle.go

+7
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"knative.dev/pkg/apis"
2828
"knative.dev/pkg/kmap"
2929
"knative.dev/serving/pkg/apis/autoscaling"
30+
"knative.dev/serving/pkg/apis/serving"
3031
"knative.dev/serving/pkg/autoscaler/config/autoscalerconfig"
3132
)
3233

@@ -160,6 +161,12 @@ func (pa *PodAutoscaler) PanicThresholdPercentage() (percentage float64, ok bool
160161
return pa.annotationFloat64(autoscaling.PanicThresholdPercentageAnnotation)
161162
}
162163

164+
// ProgressDeadline returns the progress deadline annotation value, or false if not present.
165+
func (pa *PodAutoscaler) ProgressDeadline() (time.Duration, bool) {
166+
// the value is validated in the webhook
167+
return pa.annotationDuration(serving.ProgressDeadlineAnnotation)
168+
}
169+
163170
// InitialScale returns the initial scale on the revision if present, or false if not present.
164171
func (pa *PodAutoscaler) InitialScale() (int32, bool) {
165172
// The value is validated in the webhook.

Diff for: ‎pkg/apis/autoscaling/v1alpha1/pa_lifecycle_test.go

+48
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
apistest "knative.dev/pkg/apis/testing"
3131
"knative.dev/pkg/ptr"
3232
"knative.dev/serving/pkg/apis/autoscaling"
33+
"knative.dev/serving/pkg/apis/serving"
3334
"knative.dev/serving/pkg/autoscaler/config/autoscalerconfig"
3435
)
3536

@@ -781,6 +782,53 @@ func TestScaleDownDelayAnnotation(t *testing.T) {
781782
}
782783
}
783784

785+
func TestProgressDelayAnnotation(t *testing.T) {
786+
cases := []struct {
787+
name string
788+
pa *PodAutoscaler
789+
wantDelay time.Duration
790+
wantOK bool
791+
}{{
792+
name: "not present",
793+
pa: pa(map[string]string{}),
794+
wantDelay: 0,
795+
wantOK: false,
796+
}, {
797+
name: "present",
798+
pa: pa(map[string]string{
799+
serving.ProgressDeadlineAnnotationKey: "120s",
800+
}),
801+
wantDelay: 120 * time.Second,
802+
wantOK: true,
803+
}, {
804+
name: "complex",
805+
pa: pa(map[string]string{
806+
serving.ProgressDeadlineAnnotationKey: "2m33s",
807+
}),
808+
wantDelay: 153 * time.Second,
809+
wantOK: true,
810+
}, {
811+
name: "invalid",
812+
pa: pa(map[string]string{
813+
serving.ProgressDeadlineAnnotationKey: "365d",
814+
}),
815+
wantDelay: 0,
816+
wantOK: false,
817+
}}
818+
819+
for _, tc := range cases {
820+
t.Run(tc.name, func(t *testing.T) {
821+
gotDelay, gotOK := tc.pa.ProgressDeadline()
822+
if gotDelay != tc.wantDelay {
823+
t.Errorf("ProgressDeadline = %v, want: %v", gotDelay, tc.wantDelay)
824+
}
825+
if gotOK != tc.wantOK {
826+
t.Errorf("OK = %v, want: %v", gotOK, tc.wantOK)
827+
}
828+
})
829+
}
830+
}
831+
784832
func TestWindowAnnotation(t *testing.T) {
785833
cases := []struct {
786834
name string

Diff for: ‎pkg/apis/serving/register.go

+6
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,9 @@ const (
122122
// that will result to the Route/KService getting a cluster local
123123
// domain suffix.
124124
VisibilityClusterLocal = "cluster-local"
125+
126+
// ProgressDeadlineAnnotationKey is the label key for the per revision progress deadline to set for the deployment
127+
ProgressDeadlineAnnotationKey = GroupName + "/progress-deadline"
125128
)
126129

127130
var (
@@ -159,4 +162,7 @@ var (
159162
QueueSidecarResourcePercentageAnnotationKey,
160163
"queue.sidecar." + GroupName + "/resourcePercentage",
161164
}
165+
ProgressDeadlineAnnotation = kmap.KeyPriority{
166+
ProgressDeadlineAnnotationKey,
167+
}
162168
)

Diff for: ‎pkg/apis/serving/v1/revision_validation.go

+29
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"fmt"
2222
"strconv"
2323
"strings"
24+
"time"
2425

2526
"k8s.io/apimachinery/pkg/api/validation"
2627
"knative.dev/pkg/apis"
@@ -68,6 +69,7 @@ func (rts *RevisionTemplateSpec) Validate(ctx context.Context) *apis.FieldError
6869
// it follows the requirements on the name.
6970
errs = errs.Also(validateRevisionName(ctx, rts.Name, rts.GenerateName))
7071
errs = errs.Also(validateQueueSidecarAnnotation(rts.Annotations).ViaField("metadata.annotations"))
72+
errs = errs.Also(validateProgressDeadlineAnnotation(rts.Annotations).ViaField("metadata.annotations"))
7173
return errs
7274
}
7375

@@ -195,3 +197,30 @@ func validateQueueSidecarAnnotation(m map[string]string) *apis.FieldError {
195197
}
196198
return nil
197199
}
200+
201+
// ValidateProgressDeadlineAnnotation validates the revision progress deadline annotation.
202+
func validateProgressDeadlineAnnotation(annos map[string]string) *apis.FieldError {
203+
if k, v, _ := serving.ProgressDeadlineAnnotation.Get(annos); v != "" {
204+
// Parse as duration.
205+
d, err := time.ParseDuration(v)
206+
if err != nil {
207+
return apis.ErrInvalidValue(v, k)
208+
}
209+
// Validate that it has second precision.
210+
if d.Round(time.Second) != d {
211+
return &apis.FieldError{
212+
// Even if tempting %v won't work here, since it might output the value spelled differently.
213+
Message: fmt.Sprintf("progress-deadline=%s is not at second precision", v),
214+
Paths: []string{k},
215+
}
216+
}
217+
// And positive.
218+
if d < 0 {
219+
return &apis.FieldError{
220+
Message: fmt.Sprintf("progress-deadline=%s must be positive", v),
221+
Paths: []string{k},
222+
}
223+
}
224+
}
225+
return nil
226+
}

Diff for: ‎pkg/apis/serving/v1/revision_validation_test.go

+81
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,87 @@ func TestRevisionTemplateSpecValidation(t *testing.T) {
966966
},
967967
},
968968
want: nil,
969+
}, {
970+
name: "Valid progress-deadline",
971+
ctx: autoscalerConfigCtx(true, 1),
972+
rts: &RevisionTemplateSpec{
973+
ObjectMeta: metav1.ObjectMeta{
974+
Annotations: map[string]string{
975+
serving.ProgressDeadlineAnnotationKey: "1m3s",
976+
},
977+
},
978+
Spec: RevisionSpec{
979+
PodSpec: corev1.PodSpec{
980+
Containers: []corev1.Container{{
981+
Image: "helloworld",
982+
}},
983+
},
984+
},
985+
},
986+
want: nil,
987+
}, {
988+
name: "progress-deadline too precise",
989+
ctx: autoscalerConfigCtx(true, 1),
990+
rts: &RevisionTemplateSpec{
991+
ObjectMeta: metav1.ObjectMeta{
992+
Annotations: map[string]string{
993+
serving.ProgressDeadlineAnnotationKey: "1m3s34ms",
994+
},
995+
},
996+
Spec: RevisionSpec{
997+
PodSpec: corev1.PodSpec{
998+
Containers: []corev1.Container{{
999+
Image: "helloworld",
1000+
}},
1001+
},
1002+
},
1003+
},
1004+
want: (&apis.FieldError{
1005+
Message: "progress-deadline=1m3s34ms is not at second precision",
1006+
Paths: []string{serving.ProgressDeadlineAnnotationKey},
1007+
}).ViaField("metadata.annotations"),
1008+
}, {
1009+
name: "invalid progress-deadline duration",
1010+
ctx: autoscalerConfigCtx(true, 1),
1011+
rts: &RevisionTemplateSpec{
1012+
ObjectMeta: metav1.ObjectMeta{
1013+
Annotations: map[string]string{
1014+
serving.ProgressDeadlineAnnotationKey: "not-a-duration",
1015+
},
1016+
},
1017+
Spec: RevisionSpec{
1018+
PodSpec: corev1.PodSpec{
1019+
Containers: []corev1.Container{{
1020+
Image: "helloworld",
1021+
}},
1022+
},
1023+
},
1024+
},
1025+
want: (&apis.FieldError{
1026+
Message: "invalid value: not-a-duration",
1027+
Paths: []string{serving.ProgressDeadlineAnnotationKey},
1028+
}).ViaField("metadata.annotations"),
1029+
}, {
1030+
name: "negative progress-deadline",
1031+
ctx: autoscalerConfigCtx(true, 1),
1032+
rts: &RevisionTemplateSpec{
1033+
ObjectMeta: metav1.ObjectMeta{
1034+
Annotations: map[string]string{
1035+
serving.ProgressDeadlineAnnotationKey: "-1m3s",
1036+
},
1037+
},
1038+
Spec: RevisionSpec{
1039+
PodSpec: corev1.PodSpec{
1040+
Containers: []corev1.Container{{
1041+
Image: "helloworld",
1042+
}},
1043+
},
1044+
},
1045+
},
1046+
want: (&apis.FieldError{
1047+
Message: "progress-deadline=-1m3s must be positive",
1048+
Paths: []string{serving.ProgressDeadlineAnnotationKey},
1049+
}).ViaField("metadata.annotations"),
9691050
}}
9701051

9711052
for _, test := range tests {

Diff for: ‎pkg/apis/serving/v1/route_validation.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,7 @@ import (
2929
func (r *Route) Validate(ctx context.Context) *apis.FieldError {
3030
errs := serving.ValidateObjectMetadata(ctx, r.GetObjectMeta(), false).Also(
3131
r.validateLabels().ViaField("labels"))
32-
errs = errs.Also(serving.ValidateRolloutDurationAnnotation(
33-
r.GetAnnotations()).ViaField("annotations"))
32+
errs = errs.Also(serving.ValidateRolloutDurationAnnotation(r.GetAnnotations()).ViaField("annotations"))
3433
errs = errs.ViaField("metadata")
3534
errs = errs.Also(r.Spec.Validate(apis.WithinSpec(ctx)).ViaField("spec"))
3635

Diff for: ‎pkg/apis/serving/v1/service_validation.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,7 @@ func (s *Service) Validate(ctx context.Context) (errs *apis.FieldError) {
3131
// spec validation.
3232
if !apis.IsInStatusUpdate(ctx) {
3333
errs = errs.Also(serving.ValidateObjectMetadata(ctx, s.GetObjectMeta(), false))
34-
errs = errs.Also(serving.ValidateRolloutDurationAnnotation(
35-
s.GetAnnotations()).ViaField("annotations"))
34+
errs = errs.Also(serving.ValidateRolloutDurationAnnotation(s.GetAnnotations()).ViaField("annotations"))
3635
errs = errs.ViaField("metadata")
3736

3837
ctx = apis.WithinParent(ctx, s.ObjectMeta)

Diff for: ‎pkg/reconciler/autoscaling/kpa/scaler.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,12 @@ func (ks *scaler) handleScaleToZero(ctx context.Context, pa *autoscalingv1alpha1
185185
return 1, true
186186
}
187187
cfgD := cfgs.Deployment
188-
activationTimeout := cfgD.ProgressDeadline + activationTimeoutBuffer
188+
var activationTimeout time.Duration
189+
if progressDeadline, ok := pa.ProgressDeadline(); ok {
190+
activationTimeout = progressDeadline + activationTimeoutBuffer
191+
} else {
192+
activationTimeout = cfgD.ProgressDeadline + activationTimeoutBuffer
193+
}
189194

190195
now := time.Now()
191196
logger := logging.FromContext(ctx)

Diff for: ‎pkg/reconciler/autoscaling/kpa/scaler_test.go

+12
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,18 @@ func TestScaler(t *testing.T) {
323323
paMutation: func(k *autoscalingv1alpha1.PodAutoscaler) {
324324
paMarkActivating(k, time.Now().Add(-(activationTimeout + time.Second)))
325325
},
326+
}, {
327+
label: "scale to zero while activating after revision deadline exceeded",
328+
startReplicas: 1,
329+
scaleTo: 0,
330+
wantReplicas: 0,
331+
wantScaling: true,
332+
paMutation: func(k *autoscalingv1alpha1.PodAutoscaler) {
333+
progressDeadline := "5s"
334+
k.Annotations[serving.ProgressDeadlineAnnotationKey] = progressDeadline
335+
customActivationTimeout, _ := time.ParseDuration(progressDeadline)
336+
paMarkActivating(k, time.Now().Add(-(customActivationTimeout + +activationTimeoutBuffer + time.Second)))
337+
},
326338
}, {
327339
label: "scale down to minScale before grace period",
328340
startReplicas: 10,

Diff for: ‎pkg/reconciler/revision/resources/deploy.go

+12-2
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,13 @@ package resources
1919
import (
2020
"fmt"
2121
"strconv"
22+
"time"
2223

2324
network "knative.dev/networking/pkg"
2425
"knative.dev/pkg/kmeta"
2526
"knative.dev/pkg/ptr"
2627
"knative.dev/serving/pkg/apis/autoscaling"
28+
"knative.dev/serving/pkg/apis/serving"
2729
v1 "knative.dev/serving/pkg/apis/serving/v1"
2830
"knative.dev/serving/pkg/networking"
2931
"knative.dev/serving/pkg/queue"
@@ -262,13 +264,21 @@ func MakeDeployment(rev *v1.Revision, cfg *config.Config) (*appsv1.Deployment, e
262264
}
263265

264266
replicaCount := cfg.Autoscaler.InitialScale
265-
ann, found := rev.Annotations[autoscaling.InitialScaleAnnotationKey]
267+
_, ann, found := autoscaling.InitialScaleAnnotation.Get(rev.Annotations)
266268
if found {
267269
// Ignore errors and no error checking because already validated in webhook.
268270
rc, _ := strconv.ParseInt(ann, 10, 32)
269271
replicaCount = int32(rc)
270272
}
271273

274+
progressDeadline := int32(cfg.Deployment.ProgressDeadline.Seconds())
275+
_, pdAnn, pdFound := serving.ProgressDeadlineAnnotation.Get(rev.Annotations)
276+
if pdFound {
277+
// Ignore errors and no error checking because already validated in webhook.
278+
pd, _ := time.ParseDuration(pdAnn)
279+
progressDeadline = int32(pd.Seconds())
280+
}
281+
272282
labels := makeLabels(rev)
273283
anns := makeAnnotations(rev)
274284

@@ -285,7 +295,7 @@ func MakeDeployment(rev *v1.Revision, cfg *config.Config) (*appsv1.Deployment, e
285295
Spec: appsv1.DeploymentSpec{
286296
Replicas: ptr.Int32(replicaCount),
287297
Selector: makeSelector(rev),
288-
ProgressDeadlineSeconds: ptr.Int32(int32(cfg.Deployment.ProgressDeadline.Seconds())),
298+
ProgressDeadlineSeconds: ptr.Int32(progressDeadline),
289299
Strategy: appsv1.DeploymentStrategy{
290300
Type: appsv1.RollingUpdateDeploymentStrategyType,
291301
RollingUpdate: &appsv1.RollingUpdateDeployment{

Diff for: ‎pkg/reconciler/revision/resources/deploy_test.go

+38-1
Original file line numberDiff line numberDiff line change
@@ -1236,7 +1236,7 @@ func TestMakeDeployment(t *testing.T) {
12361236
map[string]string{sidecarIstioInjectAnnotation: "false"})
12371237
}),
12381238
}, {
1239-
name: "with ProgressDeadline override",
1239+
name: "with progress-deadline override",
12401240
dc: deployment.Config{
12411241
ProgressDeadline: 42 * time.Second,
12421242
},
@@ -1252,6 +1252,43 @@ func TestMakeDeployment(t *testing.T) {
12521252
want: appsv1deployment(func(deploy *appsv1.Deployment) {
12531253
deploy.Spec.ProgressDeadlineSeconds = ptr.Int32(42)
12541254
}),
1255+
}, {
1256+
name: "with progress-deadline annotation",
1257+
rev: revision("bar", "foo",
1258+
WithRevisionAnn("serving.knative.dev/progress-deadline", "42s"),
1259+
withContainers([]corev1.Container{{
1260+
Name: servingContainerName,
1261+
Image: "ubuntu",
1262+
ReadinessProbe: withTCPReadinessProbe(12345),
1263+
}}),
1264+
WithContainerStatuses([]v1.ContainerStatus{{
1265+
ImageDigest: "busybox@sha256:deadbeef",
1266+
}}), withoutLabels),
1267+
want: appsv1deployment(func(deploy *appsv1.Deployment) {
1268+
deploy.Spec.ProgressDeadlineSeconds = ptr.Int32(42)
1269+
deploy.Annotations = map[string]string{serving.ProgressDeadlineAnnotationKey: "42s"}
1270+
deploy.Spec.Template.Annotations = map[string]string{serving.ProgressDeadlineAnnotationKey: "42s"}
1271+
}),
1272+
}, {
1273+
name: "with ProgressDeadline annotation and configmap override",
1274+
dc: deployment.Config{
1275+
ProgressDeadline: 503 * time.Second,
1276+
},
1277+
rev: revision("bar", "foo",
1278+
WithRevisionAnn("serving.knative.dev/progress-deadline", "42s"),
1279+
withContainers([]corev1.Container{{
1280+
Name: servingContainerName,
1281+
Image: "ubuntu",
1282+
ReadinessProbe: withTCPReadinessProbe(12345),
1283+
}}),
1284+
WithContainerStatuses([]v1.ContainerStatus{{
1285+
ImageDigest: "busybox@sha256:deadbeef",
1286+
}}), withoutLabels),
1287+
want: appsv1deployment(func(deploy *appsv1.Deployment) {
1288+
deploy.Spec.ProgressDeadlineSeconds = ptr.Int32(42)
1289+
deploy.Annotations = map[string]string{serving.ProgressDeadlineAnnotationKey: "42s"}
1290+
deploy.Spec.Template.Annotations = map[string]string{serving.ProgressDeadlineAnnotationKey: "42s"}
1291+
}),
12551292
}, {
12561293
name: "cluster initial scale",
12571294
acMutator: func(ac *autoscalerconfig.Config) {

0 commit comments

Comments
 (0)