Skip to content

Commit bde5983

Browse files
smarterclaytonsmarterclayton
committed
Implement a mid hook for recreate deployments
The 90% case for hooks is running a migration - if that requires the service to be down, that won't work for either pre or post deployments. Add a "mid" deployment hook that runs on recreate strategy while all containers are stopped.
1 parent 33a3092 commit bde5983

File tree

16 files changed

+224
-12
lines changed

16 files changed

+224
-12
lines changed

‎api/swagger-spec/oapi-v1.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18267,6 +18267,10 @@
1826718267
"$ref": "v1.LifecycleHook",
1826818268
"description": "a hook executed before the strategy starts the deployment"
1826918269
},
18270+
"mid": {
18271+
"$ref": "v1.LifecycleHook",
18272+
"description": "a hook executed after the strategy scales down the deployment and before it scales up"
18273+
},
1827018274
"post": {
1827118275
"$ref": "v1.LifecycleHook",
1827218276
"description": "a hook executed after the strategy finishes the deployment"

‎examples/sample-app/application-template-stibuild.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,21 @@
331331
"containerName": "ruby-helloworld-database"
332332
}
333333
},
334+
"mid": {
335+
"failurePolicy": "Abort",
336+
"execNewPod": {
337+
"command": [
338+
"/bin/true"
339+
],
340+
"env": [
341+
{
342+
"name": "CUSTOM_VAR2",
343+
"value": "custom_value2"
344+
}
345+
],
346+
"containerName": "ruby-helloworld-database"
347+
}
348+
},
334349
"post": {
335350
"failurePolicy": "Ignore",
336351
"execNewPod": {

‎pkg/api/deep_copy_generated.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1737,6 +1737,14 @@ func deepCopy_api_RecreateDeploymentStrategyParams(in deployapi.RecreateDeployme
17371737
} else {
17381738
out.Pre = nil
17391739
}
1740+
if in.Mid != nil {
1741+
out.Mid = new(deployapi.LifecycleHook)
1742+
if err := deepCopy_api_LifecycleHook(*in.Mid, out.Mid, c); err != nil {
1743+
return err
1744+
}
1745+
} else {
1746+
out.Mid = nil
1747+
}
17401748
if in.Post != nil {
17411749
out.Post = new(deployapi.LifecycleHook)
17421750
if err := deepCopy_api_LifecycleHook(*in.Post, out.Post, c); err != nil {

‎pkg/api/v1/conversion_generated.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3140,6 +3140,14 @@ func autoconvert_api_RecreateDeploymentStrategyParams_To_v1_RecreateDeploymentSt
31403140
} else {
31413141
out.Pre = nil
31423142
}
3143+
if in.Mid != nil {
3144+
out.Mid = new(deployapiv1.LifecycleHook)
3145+
if err := convert_api_LifecycleHook_To_v1_LifecycleHook(in.Mid, out.Mid, s); err != nil {
3146+
return err
3147+
}
3148+
} else {
3149+
out.Mid = nil
3150+
}
31433151
if in.Post != nil {
31443152
out.Post = new(deployapiv1.LifecycleHook)
31453153
if err := convert_api_LifecycleHook_To_v1_LifecycleHook(in.Post, out.Post, s); err != nil {
@@ -3668,6 +3676,14 @@ func autoconvert_v1_RecreateDeploymentStrategyParams_To_api_RecreateDeploymentSt
36683676
} else {
36693677
out.Pre = nil
36703678
}
3679+
if in.Mid != nil {
3680+
out.Mid = new(deployapi.LifecycleHook)
3681+
if err := convert_v1_LifecycleHook_To_api_LifecycleHook(in.Mid, out.Mid, s); err != nil {
3682+
return err
3683+
}
3684+
} else {
3685+
out.Mid = nil
3686+
}
36713687
if in.Post != nil {
36723688
out.Post = new(deployapi.LifecycleHook)
36733689
if err := convert_v1_LifecycleHook_To_api_LifecycleHook(in.Post, out.Post, s); err != nil {

‎pkg/api/v1/deep_copy_generated.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1764,6 +1764,14 @@ func deepCopy_v1_RecreateDeploymentStrategyParams(in deployapiv1.RecreateDeploym
17641764
} else {
17651765
out.Pre = nil
17661766
}
1767+
if in.Mid != nil {
1768+
out.Mid = new(deployapiv1.LifecycleHook)
1769+
if err := deepCopy_v1_LifecycleHook(*in.Mid, out.Mid, c); err != nil {
1770+
return err
1771+
}
1772+
} else {
1773+
out.Mid = nil
1774+
}
17671775
if in.Post != nil {
17681776
out.Post = new(deployapiv1.LifecycleHook)
17691777
if err := deepCopy_v1_LifecycleHook(*in.Post, out.Post, c); err != nil {

‎pkg/api/v1beta3/conversion_generated.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3149,6 +3149,14 @@ func autoconvert_api_RecreateDeploymentStrategyParams_To_v1beta3_RecreateDeploym
31493149
} else {
31503150
out.Pre = nil
31513151
}
3152+
if in.Mid != nil {
3153+
out.Mid = new(deployapiv1beta3.LifecycleHook)
3154+
if err := convert_api_LifecycleHook_To_v1beta3_LifecycleHook(in.Mid, out.Mid, s); err != nil {
3155+
return err
3156+
}
3157+
} else {
3158+
out.Mid = nil
3159+
}
31523160
if in.Post != nil {
31533161
out.Post = new(deployapiv1beta3.LifecycleHook)
31543162
if err := convert_api_LifecycleHook_To_v1beta3_LifecycleHook(in.Post, out.Post, s); err != nil {
@@ -3677,6 +3685,14 @@ func autoconvert_v1beta3_RecreateDeploymentStrategyParams_To_api_RecreateDeploym
36773685
} else {
36783686
out.Pre = nil
36793687
}
3688+
if in.Mid != nil {
3689+
out.Mid = new(deployapi.LifecycleHook)
3690+
if err := convert_v1beta3_LifecycleHook_To_api_LifecycleHook(in.Mid, out.Mid, s); err != nil {
3691+
return err
3692+
}
3693+
} else {
3694+
out.Mid = nil
3695+
}
36803696
if in.Post != nil {
36813697
out.Post = new(deployapi.LifecycleHook)
36823698
if err := convert_v1beta3_LifecycleHook_To_api_LifecycleHook(in.Post, out.Post, s); err != nil {

‎pkg/api/v1beta3/deep_copy_generated.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1772,6 +1772,14 @@ func deepCopy_v1beta3_RecreateDeploymentStrategyParams(in deployapiv1beta3.Recre
17721772
} else {
17731773
out.Pre = nil
17741774
}
1775+
if in.Mid != nil {
1776+
out.Mid = new(deployapiv1beta3.LifecycleHook)
1777+
if err := deepCopy_v1beta3_LifecycleHook(*in.Mid, out.Mid, c); err != nil {
1778+
return err
1779+
}
1780+
} else {
1781+
out.Mid = nil
1782+
}
17751783
if in.Post != nil {
17761784
out.Post = new(deployapiv1beta3.LifecycleHook)
17771785
if err := deepCopy_v1beta3_LifecycleHook(*in.Post, out.Post, c); err != nil {

‎pkg/cmd/cli/describe/deployments.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,10 +178,14 @@ func printStrategy(strategy deployapi.DeploymentStrategy, w *tabwriter.Writer) {
178178
case deployapi.DeploymentStrategyTypeRecreate:
179179
if strategy.RecreateParams != nil {
180180
pre := strategy.RecreateParams.Pre
181+
mid := strategy.RecreateParams.Mid
181182
post := strategy.RecreateParams.Post
182183
if pre != nil {
183184
printHook("Pre-deployment", pre, w)
184185
}
186+
if mid != nil {
187+
printHook("Mid-deployment", mid, w)
188+
}
185189
if post != nil {
186190
printHook("Post-deployment", post, w)
187191
}

‎pkg/deploy/api/types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ type RecreateDeploymentStrategyParams struct {
7070
// Pre is a lifecycle hook which is executed before the strategy manipulates
7171
// the deployment. All LifecycleHookFailurePolicy values are supported.
7272
Pre *LifecycleHook
73+
// Mid is a lifecycle hook which is executed while the deployment is scaled down to zero before the first new
74+
// pod is created. All LifecycleHookFailurePolicy values are supported.
75+
Mid *LifecycleHook
7376
// Post is a lifecycle hook which is executed after the strategy has
7477
// finished all deployment logic. The LifecycleHookFailurePolicyAbort policy
7578
// is NOT supported.

‎pkg/deploy/api/v1/types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ type RecreateDeploymentStrategyParams struct {
7070
// Pre is a lifecycle hook which is executed before the strategy manipulates
7171
// the deployment. All LifecycleHookFailurePolicy values are supported.
7272
Pre *LifecycleHook `json:"pre,omitempty" description:"a hook executed before the strategy starts the deployment"`
73+
// Mid is a lifecycle hook which is executed while the deployment is scaled down to zero before the first new
74+
// pod is created. All LifecycleHookFailurePolicy values are supported.
75+
Mid *LifecycleHook `json:"mid,omitempty" description:"a hook executed after the strategy scales down the deployment and before it scales up"`
7376
// Post is a lifecycle hook which is executed after the strategy has
7477
// finished all deployment logic. The LifecycleHookFailurePolicyAbort policy
7578
// is NOT supported.

‎pkg/deploy/api/v1beta3/types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ type RecreateDeploymentStrategyParams struct {
7070
// Pre is a lifecycle hook which is executed before the strategy manipulates
7171
// the deployment. All LifecycleHookFailurePolicy values are supported.
7272
Pre *LifecycleHook `json:"pre,omitempty" description:"a hook executed before the strategy starts the deployment"`
73+
// Mid is a lifecycle hook which is executed while the deployment is scaled down to zero before the first new
74+
// pod is created. All LifecycleHookFailurePolicy values are supported.
75+
Mid *LifecycleHook `json:"mid,omitempty" description:"a hook executed after the strategy scales down the deployment and before it scales up"`
7376
// Post is a lifecycle hook which is executed after the strategy has
7477
// finished all deployment logic. The LifecycleHookFailurePolicyAbort policy
7578
// is NOT supported.

‎pkg/deploy/api/validation/validation.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,9 @@ func validateRecreateParams(params *deployapi.RecreateDeploymentStrategyParams,
129129
if params.Pre != nil {
130130
errs = append(errs, validateLifecycleHook(params.Pre, fldPath.Child("pre"))...)
131131
}
132+
if params.Mid != nil {
133+
errs = append(errs, validateLifecycleHook(params.Mid, fldPath.Child("mid"))...)
134+
}
132135
if params.Post != nil {
133136
errs = append(errs, validateLifecycleHook(params.Post, fldPath.Child("post"))...)
134137
}

‎pkg/deploy/strategy/recreate/recreate.go

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,8 @@ func (s *RecreateDeploymentStrategy) DeployWithAcceptor(from *kapi.ReplicationCo
8383
if params != nil && params.Pre != nil {
8484
if err := s.hookExecutor.Execute(params.Pre, to, "prehook"); err != nil {
8585
return fmt.Errorf("Pre hook failed: %s", err)
86-
} else {
87-
glog.Infof("Pre hook finished")
8886
}
87+
glog.Infof("Pre hook finished")
8988
}
9089

9190
// Scale down the from deployment.
@@ -97,6 +96,13 @@ func (s *RecreateDeploymentStrategy) DeployWithAcceptor(from *kapi.ReplicationCo
9796
}
9897
}
9998

99+
if params != nil && params.Mid != nil {
100+
if err := s.hookExecutor.Execute(params.Mid, to, "mid"); err != nil {
101+
return fmt.Errorf("mid hook failed: %s", err)
102+
}
103+
glog.Infof("Mid hook finished")
104+
}
105+
100106
// Scale up the to deployment.
101107
if desiredReplicas > 0 {
102108
// If an UpdateAcceptor is provided, scale up to 1 and validate the replica,
@@ -128,9 +134,8 @@ func (s *RecreateDeploymentStrategy) DeployWithAcceptor(from *kapi.ReplicationCo
128134
if params != nil && params.Post != nil {
129135
if err := s.hookExecutor.Execute(params.Post, to, "posthook"); err != nil {
130136
util.HandleError(fmt.Errorf("post hook failed: %s", err))
131-
} else {
132-
glog.Infof("Post hook finished")
133137
}
138+
glog.Infof("Post hook finished")
134139
}
135140

136141
glog.Infof("Deployment %s successfully made active", to.Name)

‎pkg/deploy/strategy/recreate/recreate_test.go

Lines changed: 74 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func TestRecreate_initialDeployment(t *testing.T) {
4444

4545
func TestRecreate_deploymentPreHookSuccess(t *testing.T) {
4646
config := deploytest.OkDeploymentConfig(1)
47-
config.Spec.Strategy.RecreateParams = recreateParams(deployapi.LifecycleHookFailurePolicyAbort, "")
47+
config.Spec.Strategy.RecreateParams = recreateParams(deployapi.LifecycleHookFailurePolicyAbort, "", "")
4848
deployment, _ := deployutil.MakeDeployment(config, kapi.Codec)
4949
scaler := &scalertest.FakeScaler{}
5050

@@ -76,7 +76,7 @@ func TestRecreate_deploymentPreHookSuccess(t *testing.T) {
7676

7777
func TestRecreate_deploymentPreHookFail(t *testing.T) {
7878
config := deploytest.OkDeploymentConfig(1)
79-
config.Spec.Strategy.RecreateParams = recreateParams(deployapi.LifecycleHookFailurePolicyAbort, "")
79+
config.Spec.Strategy.RecreateParams = recreateParams(deployapi.LifecycleHookFailurePolicyAbort, "", "")
8080
deployment, _ := deployutil.MakeDeployment(config, kapi.Codec)
8181
scaler := &scalertest.FakeScaler{}
8282

@@ -104,9 +104,70 @@ func TestRecreate_deploymentPreHookFail(t *testing.T) {
104104
}
105105
}
106106

107+
func TestRecreate_deploymentMidHookSuccess(t *testing.T) {
108+
config := deploytest.OkDeploymentConfig(1)
109+
config.Spec.Strategy.RecreateParams = recreateParams("", deployapi.LifecycleHookFailurePolicyAbort, "")
110+
deployment, _ := deployutil.MakeDeployment(config, kapi.Codec)
111+
scaler := &scalertest.FakeScaler{}
112+
113+
hookExecuted := false
114+
strategy := &RecreateDeploymentStrategy{
115+
codec: api.Codec,
116+
retryTimeout: 1 * time.Second,
117+
retryPeriod: 1 * time.Millisecond,
118+
getReplicationController: func(namespace, name string) (*kapi.ReplicationController, error) {
119+
return deployment, nil
120+
},
121+
hookExecutor: &hookExecutorImpl{
122+
executeFunc: func(hook *deployapi.LifecycleHook, deployment *kapi.ReplicationController, label string) error {
123+
hookExecuted = true
124+
return nil
125+
},
126+
},
127+
scaler: scaler,
128+
}
129+
130+
err := strategy.Deploy(nil, deployment, 2)
131+
if err != nil {
132+
t.Fatalf("unexpected deploy error: %#v", err)
133+
}
134+
if !hookExecuted {
135+
t.Fatalf("expected hook execution")
136+
}
137+
}
138+
139+
func TestRecreate_deploymentMidHookFail(t *testing.T) {
140+
config := deploytest.OkDeploymentConfig(1)
141+
config.Spec.Strategy.RecreateParams = recreateParams("", deployapi.LifecycleHookFailurePolicyAbort, "")
142+
deployment, _ := deployutil.MakeDeployment(config, kapi.Codec)
143+
scaler := &scalertest.FakeScaler{}
144+
145+
strategy := &RecreateDeploymentStrategy{
146+
codec: api.Codec,
147+
retryTimeout: 1 * time.Second,
148+
retryPeriod: 1 * time.Millisecond,
149+
getReplicationController: func(namespace, name string) (*kapi.ReplicationController, error) {
150+
return deployment, nil
151+
},
152+
hookExecutor: &hookExecutorImpl{
153+
executeFunc: func(hook *deployapi.LifecycleHook, deployment *kapi.ReplicationController, label string) error {
154+
return fmt.Errorf("hook execution failure")
155+
},
156+
},
157+
scaler: scaler,
158+
}
159+
160+
err := strategy.Deploy(nil, deployment, 2)
161+
if err == nil {
162+
t.Fatalf("expected a deploy error")
163+
}
164+
if len(scaler.Events) > 0 {
165+
t.Fatalf("unexpected scaling events: %v", scaler.Events)
166+
}
167+
}
107168
func TestRecreate_deploymentPostHookSuccess(t *testing.T) {
108169
config := deploytest.OkDeploymentConfig(1)
109-
config.Spec.Strategy.RecreateParams = recreateParams("", deployapi.LifecycleHookFailurePolicyAbort)
170+
config.Spec.Strategy.RecreateParams = recreateParams("", "", deployapi.LifecycleHookFailurePolicyAbort)
110171
deployment, _ := deployutil.MakeDeployment(config, kapi.Codec)
111172
scaler := &scalertest.FakeScaler{}
112173

@@ -138,7 +199,7 @@ func TestRecreate_deploymentPostHookSuccess(t *testing.T) {
138199

139200
func TestRecreate_deploymentPostHookFail(t *testing.T) {
140201
config := deploytest.OkDeploymentConfig(1)
141-
config.Spec.Strategy.RecreateParams = recreateParams("", deployapi.LifecycleHookFailurePolicyAbort)
202+
config.Spec.Strategy.RecreateParams = recreateParams("", "", deployapi.LifecycleHookFailurePolicyAbort)
142203
deployment, _ := deployutil.MakeDeployment(config, kapi.Codec)
143204
scaler := &scalertest.FakeScaler{}
144205

@@ -246,16 +307,21 @@ func TestRecreate_acceptorFail(t *testing.T) {
246307
}
247308
}
248309

249-
func recreateParams(preFailurePolicy, postFailurePolicy deployapi.LifecycleHookFailurePolicy) *deployapi.RecreateDeploymentStrategyParams {
250-
var pre *deployapi.LifecycleHook
251-
var post *deployapi.LifecycleHook
310+
func recreateParams(preFailurePolicy, midFailurePolicy, postFailurePolicy deployapi.LifecycleHookFailurePolicy) *deployapi.RecreateDeploymentStrategyParams {
311+
var pre, mid, post *deployapi.LifecycleHook
252312

253313
if len(preFailurePolicy) > 0 {
254314
pre = &deployapi.LifecycleHook{
255315
FailurePolicy: preFailurePolicy,
256316
ExecNewPod: &deployapi.ExecNewPodHook{},
257317
}
258318
}
319+
if len(midFailurePolicy) > 0 {
320+
mid = &deployapi.LifecycleHook{
321+
FailurePolicy: midFailurePolicy,
322+
ExecNewPod: &deployapi.ExecNewPodHook{},
323+
}
324+
}
259325
if len(postFailurePolicy) > 0 {
260326
post = &deployapi.LifecycleHook{
261327
FailurePolicy: postFailurePolicy,
@@ -264,6 +330,7 @@ func recreateParams(preFailurePolicy, postFailurePolicy deployapi.LifecycleHookF
264330
}
265331
return &deployapi.RecreateDeploymentStrategyParams{
266332
Pre: pre,
333+
Mid: mid,
267334
Post: post,
268335
}
269336
}

‎test/end-to-end/core.sh

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,8 @@ cat ${LOG_DIR}/kubectl-with-token.log
176176
[ "$(cat ${LOG_DIR}/kubectl-with-token.log | grep 'Using in-cluster configuration')" ]
177177
[ "$(cat ${LOG_DIR}/kubectl-with-token.log | grep 'kubectl-with-token')" ]
178178

179-
echo "[INFO] Streaming the logs from a deployment a bunch of times..."
179+
echo "[INFO] Testing deployment logs and failing pre and mid hooks ..."
180+
# test the pre hook on a rolling deployment
180181
oc create -f test/fixtures/failing-dc.yaml
181182
tryuntil oc get rc/failing-dc-1
182183
oc logs -f dc/failing-dc
@@ -185,6 +186,16 @@ oc logs dc/failing-dc | grep 'test pre hook executed'
185186
oc deploy failing-dc --latest
186187
oc logs --version=1 dc/failing-dc | grep 'test pre hook executed'
187188
oc logs --previous dc/failing-dc | grep 'test pre hook executed'
189+
oc delete dc/failing-dc
190+
# test the mid hook on a recreate deployment and the health check
191+
oc create -f test/fixtures/failing-dc-mid.yaml
192+
tryuntil oc get rc/failing-dc-1
193+
oc logs -f dc/failing-dc
194+
wait_for_command "oc get rc/failing-dc-1 --template={{.metadata.annotations}} | grep openshift.io/deployment.phase:Failed" $((60*TIME_SEC))
195+
oc logs dc/failing-dc | grep 'test mid hook executed'
196+
oc deploy failing-dc --latest
197+
oc logs --version=1 dc/failing-dc | grep 'test mid hook executed'
198+
oc logs --previous dc/failing-dc | grep 'test mid hook executed'
188199

189200
echo "[INFO] Run pod diagnostics"
190201
# Requires a node to run the pod; uses origin-deployer pod, expects registry deployed

0 commit comments

Comments
 (0)