Skip to content

Commit 1544d2d

Browse files
zicongmeizicongmei
committed
Support role annotation
1 parent 24070d9 commit 1544d2d

File tree

5 files changed

+218
-32
lines changed

5 files changed

+218
-32
lines changed

‎apis/core/v1alpha1/annotations.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,15 @@ const (
3535
// TODO(jaypipes): Link to documentation on cross-account resource
3636
// management
3737
AnnotationOwnerAccountID = AnnotationPrefix + "owner-account-id"
38+
// AnnotationRoleARN is an annotation whose value is the identifier
39+
// for the AWS role ARN to manage the resources. If this annotation
40+
// is set on a CR, the Kubernetes user is indicating that the ACK service
41+
// controller should create/patch/delete the resource in the specified AWS
42+
// role. In order for this cross-account resource management to succeed,
43+
// the AWS IAM Role that the ACK service controller runs as needs to have
44+
// the ability to call the AWS STS::AssumeRole API call and assume an IAM
45+
// Role in the target AWS Account.
46+
AnnotationRoleARN = AnnotationPrefix + "role-arn"
3847
// AnnotationRegion is an annotation whose value is the identifier for the
3948
// the AWS region in which the resources should be created. If this annotation
4049
// is set on a CR metadata, that means the user is indicating to the ACK service

‎pkg/runtime/adoption_reconciler.go

Lines changed: 45 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"context"
1818
"fmt"
1919

20+
"github.com/aws/aws-sdk-go/aws/arn"
2021
"github.com/go-logr/logr"
2122
"github.com/pkg/errors"
2223
corev1 "k8s.io/api/core/v1"
@@ -108,26 +109,40 @@ func (r *adoptionReconciler) reconcile(ctx context.Context, req ctrlrt.Request)
108109
return ackerr.NotAdoptable
109110
}
110111

111-
// If a user has specified a namespace that is annotated with the
112-
// an owner account ID, we need an appropriate role ARN to assume
113-
// in order to perform the reconciliation. The roles ARN are typically
114-
// stored in a ConfigMap in the ACK system namespace.
115-
// If the ConfigMap is not created, or not populated with an
116-
// accountID to roleARN mapping, we need to properly requeue with a
117-
// helpful message to the user.
118112
var roleARN ackv1alpha1.AWSResourceName
119-
acctID, needCARMLookup := r.getOwnerAccountID(res)
120-
if needCARMLookup {
121-
// This means that the user is specifying a namespace that is
122-
// annotated with an owner account ID. We need to retrieve the
123-
// roleARN from the ConfigMap and properly requeue if the roleARN
124-
// is not available.
125-
roleARN, err = r.getRoleARN(acctID)
113+
var acctID ackv1alpha1.AWSAccountID
114+
// If a user specified a namespace with role ARN annotation,
115+
// we need to get the role and set the accout ID to that role.
116+
roleARNFromAnnotation := r.getRoleARNFromAnnotation(res)
117+
if roleARNFromAnnotation != "" {
118+
roleARN = roleARNFromAnnotation
119+
parsedARN, err := arn.Parse(string(roleARNFromAnnotation))
126120
if err != nil {
127-
// r.getRoleARN errors are not terminal, we should requeue.
128-
return requeue.NeededAfter(err, roleARNNotAvailableRequeueDelay)
121+
return fmt.Errorf("failed to parsed role ARN %q from namespace annotation: %v", roleARNFromAnnotation, err)
122+
}
123+
acctID = ackv1alpha1.AWSAccountID(parsedARN.AccountID)
124+
} else {
125+
// If a user has specified a namespace that is annotated with the
126+
// an owner account ID, we need an appropriate role ARN to assume
127+
// in order to perform the reconciliation. The roles ARN are typically
128+
// stored in a ConfigMap in the ACK system namespace.
129+
// If the ConfigMap is not created, or not populated with an
130+
// accountID to roleARN mapping, we need to properly requeue with a
131+
// helpful message to the user.
132+
acctID, needCARMLookup := r.getOwnerAccountID(res)
133+
if needCARMLookup {
134+
// This means that the user is specifying a namespace that is
135+
// annotated with an owner account ID. We need to retrieve the
136+
// roleARN from the ConfigMap and properly requeue if the roleARN
137+
// is not available.
138+
roleARN, err = r.getRoleARN(acctID)
139+
if err != nil {
140+
// r.getRoleARN errors are not terminal, we should requeue.
141+
return requeue.NeededAfter(err, roleARNNotAvailableRequeueDelay)
142+
}
129143
}
130144
}
145+
131146
region := r.getRegion(res)
132147
targetDescriptor := rmf.ResourceDescriptor()
133148
endpointURL := r.getEndpointURL(res)
@@ -459,6 +474,20 @@ func (r *adoptionReconciler) getOwnerAccountID(
459474
return ackv1alpha1.AWSAccountID(r.cfg.AccountID), false
460475
}
461476

477+
// getRoleARNFromAnnotation gets the role ARN from the namespace
478+
// annotation.
479+
func (r *adoptionReconciler) getRoleARNFromAnnotation(
480+
res *ackv1alpha1.AdoptedResource,
481+
) ackv1alpha1.AWSResourceName {
482+
// look for role ARN in the namespace annotations
483+
namespace := res.GetNamespace()
484+
roleARN, ok := r.cache.Namespaces.GetRoleARN(namespace)
485+
if ok {
486+
return ackv1alpha1.AWSResourceName(roleARN)
487+
}
488+
return ""
489+
}
490+
462491
// getEndpointURL returns the AWS account that owns the supplied resource.
463492
// We look for the namespace associated endpoint url, if that is set we use it.
464493
// Otherwise if none of these annotations are set we use the endpoint url specified

‎pkg/runtime/cache/namespace.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ type namespaceInfo struct {
3636
endpointURL string
3737
// {service}.services.k8s.aws/deletion-policy Annotations (keyed by service)
3838
deletionPolicies map[string]string
39+
// services.k8s.aws/role-arn Annotation
40+
roleARN string
3941
}
4042

4143
// getDefaultRegion returns the default region value
@@ -54,6 +56,14 @@ func (n *namespaceInfo) getOwnerAccountID() string {
5456
return n.ownerAccountID
5557
}
5658

59+
// getRoleARN returns the namespace role ARN
60+
func (n *namespaceInfo) getRoleARN() string {
61+
if n == nil {
62+
return ""
63+
}
64+
return n.roleARN
65+
}
66+
5767
// getEndpointURL returns the namespace Endpoint URL
5868
func (n *namespaceInfo) getEndpointURL() string {
5969
if n == nil {
@@ -182,6 +192,16 @@ func (c *NamespaceCache) GetOwnerAccountID(namespace string) (string, bool) {
182192
return "", false
183193
}
184194

195+
// GetRoleARN returns the role ARN if it exists
196+
func (c *NamespaceCache) GetRoleARN(namespace string) (string, bool) {
197+
info, ok := c.getNamespaceInfo(namespace)
198+
if ok {
199+
a := info.getRoleARN()
200+
return a, a != ""
201+
}
202+
return "", false
203+
}
204+
185205
// GetEndpointURL returns the endpoint URL if it exists
186206
func (c *NamespaceCache) GetEndpointURL(namespace string) (string, bool) {
187207
info, ok := c.getNamespaceInfo(namespace)
@@ -229,6 +249,10 @@ func (c *NamespaceCache) setNamespaceInfoFromK8sObject(ns *corev1.Namespace) {
229249
if ok {
230250
nsInfo.endpointURL = EndpointURL
231251
}
252+
RoleARN, ok := nsa[ackv1alpha1.AnnotationRoleARN]
253+
if ok {
254+
nsInfo.roleARN = RoleARN
255+
}
232256

233257
nsInfo.deletionPolicies = map[string]string{}
234258
nsDeletionPolicySuffix := "." + ackv1alpha1.AnnotationDeletionPolicy

‎pkg/runtime/cache/namespace_test.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,101 @@ func TestNamespaceCache(t *testing.T) {
131131
require.False(t, ok)
132132
}
133133

134+
func TestNamespaceCacheWithRoleARN(t *testing.T) {
135+
// create a fake k8s client and fake watcher
136+
k8sClient := k8sfake.NewSimpleClientset()
137+
watcher := watch.NewFake()
138+
k8sClient.PrependWatchReactor("production", k8stesting.DefaultWatchReactor(watcher, nil))
139+
140+
// New logger writing to specific buffer
141+
zapOptions := ctrlrtzap.Options{
142+
Development: true,
143+
Level: zapcore.InfoLevel,
144+
}
145+
fakeLogger := ctrlrtzap.New(ctrlrtzap.UseFlagOptions(&zapOptions))
146+
147+
// initlizing account cache
148+
namespaceCache := ackrtcache.NewNamespaceCache(fakeLogger, []string{}, []string{})
149+
stopCh := make(chan struct{})
150+
151+
namespaceCache.Run(k8sClient, stopCh)
152+
153+
// Test create events
154+
_, err := k8sClient.CoreV1().Namespaces().Create(
155+
context.Background(),
156+
&corev1.Namespace{
157+
ObjectMeta: metav1.ObjectMeta{
158+
Name: "production",
159+
Annotations: map[string]string{
160+
ackv1alpha1.AnnotationDefaultRegion: "us-west-2",
161+
ackv1alpha1.AnnotationRoleARN: "arn:aws:iam::123456789012:role/some-role",
162+
ackv1alpha1.AnnotationEndpointURL: "https://amazon-service.region.amazonaws.com",
163+
},
164+
},
165+
},
166+
metav1.CreateOptions{},
167+
)
168+
require.Nil(t, err)
169+
170+
time.Sleep(time.Second)
171+
172+
defaultRegion, ok := namespaceCache.GetDefaultRegion("production")
173+
require.True(t, ok)
174+
require.Equal(t, "us-west-2", defaultRegion)
175+
176+
roleARN, ok := namespaceCache.GetRoleARN("production")
177+
require.True(t, ok)
178+
require.Equal(t, "arn:aws:iam::123456789012:role/some-role", roleARN)
179+
180+
endpointURL, ok := namespaceCache.GetEndpointURL("production")
181+
require.True(t, ok)
182+
require.Equal(t, "https://amazon-service.region.amazonaws.com", endpointURL)
183+
184+
// Test update events
185+
_, err = k8sClient.CoreV1().Namespaces().Update(
186+
context.Background(),
187+
&corev1.Namespace{
188+
ObjectMeta: metav1.ObjectMeta{
189+
Name: "production",
190+
Annotations: map[string]string{
191+
ackv1alpha1.AnnotationDefaultRegion: "us-est-1",
192+
ackv1alpha1.AnnotationRoleARN: "arn:aws:iam::223456789012:role/some-role",
193+
ackv1alpha1.AnnotationEndpointURL: "https://amazon-other-service.region.amazonaws.com",
194+
},
195+
},
196+
},
197+
metav1.UpdateOptions{},
198+
)
199+
require.Nil(t, err)
200+
201+
time.Sleep(time.Second)
202+
203+
defaultRegion, ok = namespaceCache.GetDefaultRegion("production")
204+
require.True(t, ok)
205+
require.Equal(t, "us-est-1", defaultRegion)
206+
207+
roleARN, ok = namespaceCache.GetRoleARN("production")
208+
require.True(t, ok)
209+
require.Equal(t, "arn:aws:iam::223456789012:role/some-role", roleARN)
210+
211+
endpointURL, ok = namespaceCache.GetEndpointURL("production")
212+
require.True(t, ok)
213+
require.Equal(t, "https://amazon-other-service.region.amazonaws.com", endpointURL)
214+
215+
// Test delete events
216+
err = k8sClient.CoreV1().Namespaces().Delete(
217+
context.Background(),
218+
"production",
219+
metav1.DeleteOptions{},
220+
)
221+
require.Nil(t, err)
222+
223+
time.Sleep(time.Second)
224+
225+
_, ok = namespaceCache.GetDefaultRegion(testNamespace1)
226+
require.False(t, ok)
227+
}
228+
134229
func TestScopedNamespaceCache(t *testing.T) {
135230
defaultConfig := ackrtcache.Config{
136231
WatchScope: []string{"watch-scope", "watch-scope-2"},

‎pkg/runtime/reconciler.go

Lines changed: 45 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"strings"
2121
"time"
2222

23+
"github.com/aws/aws-sdk-go/aws/arn"
2324
backoff "github.com/cenkalti/backoff/v4"
2425
"github.com/go-logr/logr"
2526
"github.com/pkg/errors"
@@ -158,26 +159,40 @@ func (r *resourceReconciler) Reconcile(ctx context.Context, req ctrlrt.Request)
158159
return ctrlrt.Result{}, err
159160
}
160161

161-
// If a user has specified a namespace that is annotated with the
162-
// an owner account ID, we need an appropriate role ARN to assume
163-
// in order to perform the reconciliation. The roles ARN are typically
164-
// stored in a ConfigMap in the ACK system namespace.
165-
// If the ConfigMap is not created, or not populated with an
166-
// accountID to roleARN mapping, we need to properly requeue with a
167-
// helpful message to the user.
168162
var roleARN ackv1alpha1.AWSResourceName
169-
acctID, needCARMLookup := r.getOwnerAccountID(desired)
170-
if needCARMLookup {
171-
// This means that the user is specifying a namespace that is
172-
// annotated with an owner account ID. We need to retrieve the
173-
// roleARN from the ConfigMap and properly requeue if the roleARN
174-
// is not available.
175-
roleARN, err = r.getRoleARN(acctID)
163+
var acctID ackv1alpha1.AWSAccountID
164+
// If a user specified a namespace with role ARN annotation,
165+
// we need to get the role and set the accout ID to that role.
166+
roleARNFromAnnotation := r.getRoleARNFromAnnotation(desired)
167+
if roleARNFromAnnotation != "" {
168+
roleARN = roleARNFromAnnotation
169+
parsedARN, err := arn.Parse(string(roleARNFromAnnotation))
176170
if err != nil {
177-
// r.getRoleARN errors are not terminal, we should requeue.
178-
return ctrlrt.Result{}, requeue.NeededAfter(err, roleARNNotAvailableRequeueDelay)
171+
return ctrlrt.Result{}, fmt.Errorf("failed to parsed role ARN %q from namespace annotation: %v", roleARNFromAnnotation, err)
172+
}
173+
acctID = ackv1alpha1.AWSAccountID(parsedARN.AccountID)
174+
} else {
175+
// If a user has specified a namespace that is annotated with the
176+
// an owner account ID, we need an appropriate role ARN to assume
177+
// in order to perform the reconciliation. The roles ARN are typically
178+
// stored in a ConfigMap in the ACK system namespace.
179+
// If the ConfigMap is not created, or not populated with an
180+
// accountID to roleARN mapping, we need to properly requeue with a
181+
// helpful message to the user.
182+
acctID, needCARMLookup := r.getOwnerAccountID(desired)
183+
if needCARMLookup {
184+
// This means that the user is specifying a namespace that is
185+
// annotated with an owner account ID. We need to retrieve the
186+
// roleARN from the ConfigMap and properly requeue if the roleARN
187+
// is not available.
188+
roleARN, err = r.getRoleARN(acctID)
189+
if err != nil {
190+
// r.getRoleARN errors are not terminal, we should requeue.
191+
return ctrlrt.Result{}, requeue.NeededAfter(err, roleARNNotAvailableRequeueDelay)
192+
}
179193
}
180194
}
195+
181196
region := r.getRegion(desired)
182197

183198
endpointURL := r.getEndpointURL(desired)
@@ -1025,6 +1040,20 @@ func (r *resourceReconciler) getOwnerAccountID(
10251040
return controllerAccountID, false
10261041
}
10271042

1043+
// getRoleARNFromAnnotation gets the role ARN from the namespace
1044+
// annotation.
1045+
func (r *resourceReconciler) getRoleARNFromAnnotation(
1046+
res acktypes.AWSResource,
1047+
) ackv1alpha1.AWSResourceName {
1048+
// look for role ARN in the namespace annotations
1049+
namespace := res.MetaObject().GetNamespace()
1050+
roleARN, ok := r.cache.Namespaces.GetRoleARN(namespace)
1051+
if ok {
1052+
return ackv1alpha1.AWSResourceName(roleARN)
1053+
}
1054+
return ""
1055+
}
1056+
10281057
// getRoleARN return the Role ARN that should be assumed in order to manage
10291058
// the resources.
10301059
func (r *resourceReconciler) getRoleARN(

0 commit comments

Comments
 (0)