Skip to content

Commit 13326de

Browse files
committed
Add DC controllerRef to RC
1 parent eeaf7bc commit 13326de

File tree

4 files changed

+271
-9
lines changed

4 files changed

+271
-9
lines changed

Diff for: pkg/controller/controller_ref_manager.go

+175
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
package controller
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/golang/glog"
7+
kerrors "k8s.io/apimachinery/pkg/api/errors"
8+
kmetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
klabels "k8s.io/apimachinery/pkg/labels"
10+
kschema "k8s.io/apimachinery/pkg/runtime/schema"
11+
"k8s.io/apimachinery/pkg/types"
12+
kutilerrors "k8s.io/apimachinery/pkg/util/errors"
13+
"k8s.io/client-go/tools/record"
14+
kapi "k8s.io/kubernetes/pkg/api"
15+
kclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
16+
kcontroller "k8s.io/kubernetes/pkg/controller"
17+
)
18+
19+
// RSControlInterface is an interface that knows how to add or delete
20+
// ReplicationControllers, as well as increment or decrement them. It is used
21+
// by the DeploymentConfig controller to ease testing of actions that it takes.
22+
type RCControlInterface interface {
23+
PatchReplicationController(namespace, name string, data []byte) error
24+
}
25+
26+
// RealRCControl is the default implementation of RCControlInterface.
27+
type RealRCControl struct {
28+
KubeClient kclientset.Interface
29+
Recorder record.EventRecorder
30+
}
31+
32+
// To make sure RealRCControl implements RCControlInterface
33+
var _ RCControlInterface = &RealRCControl{}
34+
35+
// PatchReplicationController executes a strategic merge patch contained in 'data' on RC specified by 'namespace' and 'name'
36+
func (r RealRCControl) PatchReplicationController(namespace, name string, data []byte) error {
37+
_, err := r.KubeClient.Core().ReplicationControllers(namespace).Patch(name, types.StrategicMergePatchType, data)
38+
return err
39+
}
40+
41+
type RCControllerRefManager struct {
42+
kcontroller.BaseControllerRefManager
43+
controllerKind kschema.GroupVersionKind
44+
rcControl RCControlInterface
45+
}
46+
47+
// NewRCControllerRefManager returns a RCControllerRefManager that exposes
48+
// methods to manage the controllerRef of ReplicationControllers.
49+
//
50+
// The CanAdopt() function can be used to perform a potentially expensive check
51+
// (such as a live GET from the API server) prior to the first adoption.
52+
// It will only be called (at most once) if an adoption is actually attempted.
53+
// If CanAdopt() returns a non-nil error, all adoptions will fail.
54+
//
55+
// NOTE: Once CanAdopt() is called, it will not be called again by the same
56+
// RCControllerRefManager instance. Create a new instance if it
57+
// makes sense to check CanAdopt() again (e.g. in a different sync pass).
58+
func NewRCControllerRefManager(
59+
rcControl RCControlInterface,
60+
controller kmetav1.Object,
61+
selector klabels.Selector,
62+
controllerKind kschema.GroupVersionKind,
63+
canAdopt func() error,
64+
) *RCControllerRefManager {
65+
return &RCControllerRefManager{
66+
BaseControllerRefManager: kcontroller.BaseControllerRefManager{
67+
Controller: controller,
68+
Selector: selector,
69+
CanAdoptFunc: canAdopt,
70+
},
71+
controllerKind: controllerKind,
72+
rcControl: rcControl,
73+
}
74+
}
75+
76+
// ClaimReplicationController tries to take ownership of a ReplicationController.
77+
//
78+
// It will reconcile the following:
79+
// * Adopt the ReplicationController if it's an orphan.
80+
// * Release owned ReplicationController if the selector no longer matches.
81+
//
82+
// A non-nil error is returned if some form of reconciliation was attempted and
83+
// failed. Usually, controllers should try again later in case reconciliation
84+
// is still needed.
85+
//
86+
// If the error is nil, either the reconciliation succeeded, or no
87+
// reconciliation was necessary. The returned boolean indicates whether you now
88+
// own the object.
89+
func (m *RCControllerRefManager) ClaimReplicationController(rc *kapi.ReplicationController) (bool, error) {
90+
match := func(obj kmetav1.Object) bool {
91+
return m.Selector.Matches(klabels.Set(obj.GetLabels()))
92+
}
93+
adopt := func(obj kmetav1.Object) error {
94+
return m.AdoptReplicationController(obj.(*kapi.ReplicationController))
95+
}
96+
release := func(obj kmetav1.Object) error {
97+
return m.ReleaseReplicationController(obj.(*kapi.ReplicationController))
98+
}
99+
100+
return m.ClaimObject(rc, match, adopt, release)
101+
}
102+
103+
// ClaimReplicationControllers tries to take ownership of a list of ReplicationControllers.
104+
//
105+
// It will reconcile the following:
106+
// * Adopt orphans if the selector matches.
107+
// * Release owned objects if the selector no longer matches.
108+
//
109+
// A non-nil error is returned if some form of reconciliation was attempted and
110+
// failed. Usually, controllers should try again later in case reconciliation
111+
// is still needed.
112+
//
113+
// If the error is nil, either the reconciliation succeeded, or no
114+
// reconciliation was necessary. The list of ReplicationControllers that you now own is
115+
// returned.
116+
func (m *RCControllerRefManager) ClaimReplicationControllers(rcs []*kapi.ReplicationController) ([]*kapi.ReplicationController, error) {
117+
var claimed []*kapi.ReplicationController
118+
var errlist []error
119+
120+
for _, rc := range rcs {
121+
ok, err := m.ClaimReplicationController(rc)
122+
if err != nil {
123+
errlist = append(errlist, err)
124+
continue
125+
}
126+
if ok {
127+
claimed = append(claimed, rc)
128+
}
129+
}
130+
return claimed, kutilerrors.NewAggregate(errlist)
131+
}
132+
133+
// AdoptReplicationController sends a patch to take control of the ReplicationController. It returns the error if
134+
// the patching fails.
135+
func (m *RCControllerRefManager) AdoptReplicationController(rs *kapi.ReplicationController) error {
136+
if err := m.CanAdopt(); err != nil {
137+
return fmt.Errorf("can't adopt ReplicationController %s/%s (%s): %v", rs.Namespace, rs.Name, rs.UID, err)
138+
}
139+
// Note that ValidateOwnerReferences() will reject this patch if another
140+
// OwnerReference exists with controller=true.
141+
addControllerPatch := fmt.Sprintf(
142+
`{"metadata":{
143+
"ownerReferences":[{"apiVersion":"%s","kind":"%s","name":"%s","uid":"%s","controller":true,"blockOwnerDeletion":true}],
144+
"uid":"%s",
145+
"finalizers": ["%s"]
146+
}
147+
}`,
148+
m.controllerKind.GroupVersion(), m.controllerKind.Kind,
149+
m.Controller.GetName(), m.Controller.GetUID(), rs.UID,
150+
kmetav1.FinalizerDeleteDependents)
151+
return m.rcControl.PatchReplicationController(rs.Namespace, rs.Name, []byte(addControllerPatch))
152+
}
153+
154+
// ReleaseReplicationController sends a patch to free the ReplicationController from the control of the Deployment controller.
155+
// It returns the error if the patching fails. 404 and 422 errors are ignored.
156+
func (m *RCControllerRefManager) ReleaseReplicationController(rc *kapi.ReplicationController) error {
157+
glog.V(4).Infof("patching ReplicationController %s/%s to remove its controllerRef to %s/%s:%s",
158+
rc.Namespace, rc.Name, m.controllerKind.GroupVersion(), m.controllerKind.Kind, m.Controller.GetName())
159+
deleteOwnerRefPatch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"$patch":"delete","uid":"%s"}],"uid":"%s"}}`, m.Controller.GetUID(), rc.UID)
160+
err := m.rcControl.PatchReplicationController(rc.Namespace, rc.Name, []byte(deleteOwnerRefPatch))
161+
if err != nil {
162+
if kerrors.IsNotFound(err) {
163+
// If the ReplicationController no longer exists, ignore it.
164+
return nil
165+
}
166+
if kerrors.IsInvalid(err) {
167+
// Invalid error will be returned in two cases: 1. the ReplicationController
168+
// has no owner reference, 2. the uid of the ReplicationController doesn't
169+
// match, which means the ReplicationController is deleted and then recreated.
170+
// In both cases, the error can be ignored.
171+
return nil
172+
}
173+
}
174+
return err
175+
}

Diff for: pkg/deploy/controller/deploymentconfig/deploymentconfig_controller.go

+68-8
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
kapierrors "k8s.io/apimachinery/pkg/api/errors"
1010
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
"k8s.io/apimachinery/pkg/labels"
1112
"k8s.io/apimachinery/pkg/runtime"
1213
kutilerrors "k8s.io/apimachinery/pkg/util/errors"
1314
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -17,9 +18,11 @@ import (
1718
kcoreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
1819
kcorelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion"
1920
"k8s.io/kubernetes/pkg/client/retry"
21+
kcontroller "k8s.io/kubernetes/pkg/controller"
2022

2123
osclient "github.com/openshift/origin/pkg/client"
2224
oscache "github.com/openshift/origin/pkg/client/cache"
25+
oscontroller "github.com/openshift/origin/pkg/controller"
2326
deployapi "github.com/openshift/origin/pkg/deploy/api"
2427
deployutil "github.com/openshift/origin/pkg/deploy/util"
2528
)
@@ -68,6 +71,8 @@ type DeploymentConfigController struct {
6871
rcLister kcorelisters.ReplicationControllerLister
6972
// rcListerSynced makes sure the rc shared informer is synced before reconcling any deployment config.
7073
rcListerSynced func() bool
74+
// rcControl is used for adopting/releasing replication controllers.
75+
rcControl oscontroller.RCControlInterface
7176

7277
// codec is used to build deployments from configs.
7378
codec runtime.Codec
@@ -84,11 +89,29 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig)
8489
return c.updateStatus(config, []*kapi.ReplicationController{})
8590
}
8691

87-
// Find all deployments owned by the deployment config.
92+
// List all ReplicationControllers to find also those we own but that no longer match our selector.
93+
// They will be orphaned by ClaimReplicationControllers().
94+
rcList, err := c.rcLister.ReplicationControllers(config.Namespace).List(labels.Everything())
95+
if err != nil {
96+
return fmt.Errorf("error while deploymentConfigController listing replication controllers: %v", err)
97+
}
8898
selector := deployutil.ConfigSelector(config.Name)
89-
existingDeployments, err := c.rcLister.ReplicationControllers(config.Namespace).List(selector)
99+
// If any adoptions are attempted, we should first recheck for deletion with
100+
// an uncached quorum read sometime after listing ReplicationControllers (see Kubernetes #42639).
101+
canAdoptFunc := kcontroller.RecheckDeletionTimestamp(func() (metav1.Object, error) {
102+
fresh, err := c.dn.DeploymentConfigs(config.Namespace).Get(config.Name, metav1.GetOptions{})
103+
if err != nil {
104+
return nil, err
105+
}
106+
if fresh.UID != config.UID {
107+
return nil, fmt.Errorf("original DeploymentConfig %v/%v is gone: got uid %v, wanted %v", config.Namespace, config.Name, fresh.UID, config.UID)
108+
}
109+
return fresh, nil
110+
})
111+
cm := oscontroller.NewRCControllerRefManager(c.rcControl, config, selector, deployutil.ControllerKind, canAdoptFunc)
112+
existingDeployments, err := cm.ClaimReplicationControllers(rcList)
90113
if err != nil {
91-
return err
114+
return fmt.Errorf("error while deploymentConfigController claiming replication controllers: %v", err)
92115
}
93116

94117
// In case the deployment config has been marked for deletion, merely update its status with
@@ -125,6 +148,15 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig)
125148
if err != nil {
126149
return err
127150
}
151+
// We need to make sure we own that RC or adopt it if possible
152+
isOurs, err := cm.ClaimReplicationController(rc)
153+
if err != nil {
154+
return fmt.Errorf("error while deploymentConfigController claiming the replication controller %s/%s: %v", rc.Namespace, rc.Name, err)
155+
}
156+
if !isOurs {
157+
return nil
158+
}
159+
128160
copied, err := deployutil.DeploymentDeepCopy(rc)
129161
if err != nil {
130162
return err
@@ -157,7 +189,7 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig)
157189
return c.updateStatus(config, existingDeployments)
158190
}
159191

160-
return c.reconcileDeployments(existingDeployments, config)
192+
return c.reconcileDeployments(existingDeployments, config, cm)
161193
}
162194
// If the config is paused we shouldn't create new deployments for it.
163195
if config.Spec.Paused {
@@ -177,10 +209,26 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig)
177209
}
178210
created, err := c.rn.ReplicationControllers(config.Namespace).Create(deployment)
179211
if err != nil {
180-
// If the deployment was already created, just move on. The cache could be
181-
// stale, or another process could have already handled this update.
212+
// We need to find out if our controller owns that deployment and report error if not
182213
if kapierrors.IsAlreadyExists(err) {
183-
return c.updateStatus(config, existingDeployments)
214+
rc, err := c.rcLister.ReplicationControllers(deployment.Namespace).Get(deployment.Name)
215+
if err != nil {
216+
return fmt.Errorf("error while deploymentConfigController getting the replication controller %s/%s: %v", rc.Namespace, rc.Name, err)
217+
}
218+
// We need to make sure we own that RC or adopt it if possible
219+
isOurs, err := cm.ClaimReplicationController(rc)
220+
if err != nil {
221+
return fmt.Errorf("error while deploymentConfigController claiming the replication controller: %v", err)
222+
}
223+
if isOurs {
224+
// If the deployment was already created, just move on. The cache could be
225+
// stale, or another process could have already handled this update.
226+
return c.updateStatus(config, existingDeployments)
227+
} else {
228+
err = fmt.Errorf("replication controller %s already exists and deployment config is not allowed to claim it.", deployment.Name)
229+
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %v", config.Status.LatestVersion, err)
230+
return c.updateStatus(config, existingDeployments)
231+
}
184232
}
185233
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err)
186234
// We don't care about this error since we need to report the create failure.
@@ -208,7 +256,7 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig)
208256
// successful deployment, not necessarily the latest in terms of the config
209257
// version. The active deployment replica count should follow the config, and
210258
// all other deployments should be scaled to zero.
211-
func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []*kapi.ReplicationController, config *deployapi.DeploymentConfig) error {
259+
func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []*kapi.ReplicationController, config *deployapi.DeploymentConfig, cm *oscontroller.RCControllerRefManager) error {
212260
activeDeployment := deployutil.ActiveDeployment(existingDeployments)
213261

214262
// Reconcile deployments. The active deployment follows the config, and all
@@ -239,6 +287,18 @@ func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []
239287
if err != nil {
240288
return err
241289
}
290+
// We need to make sure we own that RC or adopt it if possible
291+
isOurs, err := cm.ClaimReplicationController(rc)
292+
if err != nil {
293+
return fmt.Errorf("error while deploymentConfigController claiming the replication controller %s/%s: %v", rc.Namespace, rc.Name, err)
294+
}
295+
if !isOurs {
296+
return fmt.Errorf("deployment config %s/%s (%v) no longer owns replication controller %s/%s (%v)",
297+
config.Namespace, config.Name, config.UID,
298+
deployment.Namespace, deployment.Name, deployment.UID,
299+
)
300+
}
301+
242302
copied, err = deployutil.DeploymentDeepCopy(rc)
243303
if err != nil {
244304
return err

Diff for: pkg/deploy/controller/deploymentconfig/factory.go

+5
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
kcontroller "k8s.io/kubernetes/pkg/controller"
2121

2222
osclient "github.com/openshift/origin/pkg/client"
23+
oscontroller "github.com/openshift/origin/pkg/controller"
2324
deployapi "github.com/openshift/origin/pkg/deploy/api"
2425
)
2526

@@ -51,6 +52,10 @@ func NewDeploymentConfigController(
5152

5253
rcLister: rcInformer.Lister(),
5354
rcListerSynced: rcInformer.Informer().HasSynced,
55+
rcControl: oscontroller.RealRCControl{
56+
KubeClient: internalKubeClientset,
57+
Recorder: recorder,
58+
},
5459

5560
recorder: recorder,
5661
codec: codec,

Diff for: pkg/deploy/util/util.go

+23-1
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,16 @@ import (
1818
kcoreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
1919
kdeplutil "k8s.io/kubernetes/pkg/controller/deployment/util"
2020

21+
osapiv1 "github.com/openshift/origin/pkg/api/v1"
2122
deployapi "github.com/openshift/origin/pkg/deploy/api"
2223
"github.com/openshift/origin/pkg/util/namer"
2324
)
2425

26+
var (
27+
// ControllerKind contains the schema.GroupVersionKind for this controller type.
28+
ControllerKind = osapiv1.SchemeGroupVersion.WithKind("DeploymentConfig")
29+
)
30+
2531
// NewDeploymentCondition creates a new deployment condition.
2632
func NewDeploymentCondition(condType deployapi.DeploymentConditionType, status api.ConditionStatus, reason deployapi.DeploymentConditionReason, message string) *deployapi.DeploymentCondition {
2733
return &deployapi.DeploymentCondition{
@@ -228,6 +234,19 @@ func EncodeDeploymentConfig(config *deployapi.DeploymentConfig, codec runtime.Co
228234
return string(bytes[:]), nil
229235
}
230236

237+
func NewControllerRef(config *deployapi.DeploymentConfig) *metav1.OwnerReference {
238+
blockOwnerDeletion := true
239+
isController := true
240+
return &metav1.OwnerReference{
241+
APIVersion: ControllerKind.Version,
242+
Kind: ControllerKind.Kind,
243+
Name: config.Name,
244+
UID: config.UID,
245+
BlockOwnerDeletion: &blockOwnerDeletion,
246+
Controller: &isController,
247+
}
248+
}
249+
231250
// MakeDeployment creates a deployment represented as a ReplicationController and based on the given
232251
// DeploymentConfig. The controller replica count will be zero.
233252
func MakeDeployment(config *deployapi.DeploymentConfig, codec runtime.Codec) (*api.ReplicationController, error) {
@@ -279,6 +298,7 @@ func MakeDeployment(config *deployapi.DeploymentConfig, codec runtime.Codec) (*a
279298
podAnnotations[deployapi.DeploymentConfigAnnotation] = config.Name
280299
podAnnotations[deployapi.DeploymentVersionAnnotation] = strconv.FormatInt(config.Status.LatestVersion, 10)
281300

301+
controllerRef := NewControllerRef(config)
282302
deployment := &api.ReplicationController{
283303
ObjectMeta: metav1.ObjectMeta{
284304
Name: deploymentName,
@@ -292,7 +312,9 @@ func MakeDeployment(config *deployapi.DeploymentConfig, codec runtime.Codec) (*a
292312
deployapi.DesiredReplicasAnnotation: strconv.Itoa(int(config.Spec.Replicas)),
293313
deployapi.DeploymentReplicasAnnotation: strconv.Itoa(0),
294314
},
295-
Labels: controllerLabels,
315+
Labels: controllerLabels,
316+
OwnerReferences: []metav1.OwnerReference{*controllerRef},
317+
Finalizers: []string{metav1.FinalizerDeleteDependents},
296318
},
297319
Spec: api.ReplicationControllerSpec{
298320
// The deployment should be inactive initially

0 commit comments

Comments
 (0)