Skip to content

Commit 0fefe84

Browse files
author
Serhii Zakharov
authored
Bug 2005816: make projectid and region anonymization consistent (openshift#534)
* make projectid and region anonymization consistent * anonymized the rest * cluster-config-v1 * test * anonymize service accounts * lint
1 parent f23f521 commit 0fefe84

File tree

11 files changed

+195
-12
lines changed

11 files changed

+195
-12
lines changed

‎docs/insights-archive-sample/config/configmaps/kube-system/cluster-config-v1.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,6 @@
2222
]
2323
},
2424
"data": {
25-
"install-config": "apiVersion: v1\nbaseDomain: xxxxxxxxxxxxxxxxxxx\ncompute:\n- hyperthreading: Enabled\n name: worker\n platform: {}\n replicas: 3\ncontrolPlane:\n hyperthreading: Enabled\n name: master\n platform: {}\n replicas: 3\nmetadata:\n creationTimestamp: null\n name: tremes-testing.tremes.2021.07.12.ccxdev.devshift.net\nnetworking:\n clusterNetwork:\n - cidr: 10.128.0.0/14\n hostPrefix: 23\n machineNetwork:\n - cidr: 10.0.0.0/16\n networkType: OpenShiftSDN\n serviceNetwork:\n - 172.30.0.0/16\nplatform:\n aws:\n region: us-east-2\npublish: External\npullSecret: \"\"\n"
25+
"install-config": "apiVersion: v1\nbaseDomain: xxxxxxxxxxxxxxxxxxx\ncompute:\n- hyperthreading: Enabled\n name: worker\n platform: {}\n replicas: 3\ncontrolPlane:\n hyperthreading: Enabled\n name: master\n platform: {}\n replicas: 3\nmetadata:\n creationTimestamp: null\n name: tremes-testing.tremes.2021.07.12.ccxdev.devshift.net\nnetworking:\n clusterNetwork:\n - cidr: 10.128.0.0/14\n hostPrefix: 23\n machineNetwork:\n - cidr: 10.0.0.0/16\n networkType: OpenShiftSDN\n serviceNetwork:\n - 172.30.0.0/16\nplatform:\n aws:\n region: xxxxxxxxx\npublish: External\npullSecret: \"\"\n"
2626
}
2727
}

‎docs/insights-archive-sample/machinesets/openshift-machine-api/ci-ln-fyvthbt-f76d1-nl2fh-worker-b.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@
6464
"subnetwork": "ci-ln-fyvthbt-f76d1-nl2fh-worker-subnet"
6565
}
6666
],
67-
"projectID": "openshift-gce-devel-ci",
68-
"region": "us-east1",
67+
"projectID": "xxxxxxxxxxxxxxxxxxxxxx",
68+
"region": "xxxxxxxx",
6969
"serviceAccounts": [
7070
{
7171
"email": "ci-ln-fyvthbt-f76d1-nl2fh-w@openshift-gce-devel-ci.iam.gserviceaccount.com",

‎docs/insights-archive-sample/machinesets/openshift-machine-api/ci-ln-fyvthbt-f76d1-nl2fh-worker-c.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@
6464
"subnetwork": "ci-ln-fyvthbt-f76d1-nl2fh-worker-subnet"
6565
}
6666
],
67-
"projectID": "openshift-gce-devel-ci",
68-
"region": "us-east1",
67+
"projectID": "xxxxxxxxxxxxxxxxxxxxxx",
68+
"region": "xxxxxxxx",
6969
"serviceAccounts": [
7070
{
7171
"email": "ci-ln-fyvthbt-f76d1-nl2fh-w@openshift-gce-devel-ci.iam.gserviceaccount.com",

‎docs/insights-archive-sample/machinesets/openshift-machine-api/ci-ln-fyvthbt-f76d1-nl2fh-worker-d.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@
6464
"subnetwork": "ci-ln-fyvthbt-f76d1-nl2fh-worker-subnet"
6565
}
6666
],
67-
"projectID": "openshift-gce-devel-ci",
68-
"region": "us-east1",
67+
"projectID": "xxxxxxxxxxxxxxxxxxxxxx",
68+
"region": "xxxxxxxx",
6969
"serviceAccounts": [
7070
{
7171
"email": "ci-ln-fyvthbt-f76d1-nl2fh-w@openshift-gce-devel-ci.iam.gserviceaccount.com",

‎pkg/gatherers/clusterconfig/cluster_config_v1_config_map.go

+20
Original file line numberDiff line numberDiff line change
@@ -54,5 +54,25 @@ func anonymizeInstallConfig(installConfig *installertypes.InstallConfig) *instal
5454
// we don't use it
5555
installConfig.BaseDomain = anonymize.String(installConfig.BaseDomain)
5656

57+
if installConfig.Platform.AWS != nil {
58+
installConfig.Platform.AWS.Region = anonymize.String(installConfig.Platform.AWS.Region)
59+
}
60+
if installConfig.Platform.Azure != nil {
61+
installConfig.Platform.Azure.Region = anonymize.String(installConfig.Platform.Azure.Region)
62+
}
63+
if installConfig.Platform.GCP != nil {
64+
installConfig.Platform.GCP.Region = anonymize.String(installConfig.Platform.GCP.Region)
65+
installConfig.Platform.GCP.ProjectID = anonymize.String(installConfig.Platform.GCP.ProjectID)
66+
}
67+
if installConfig.Platform.VSphere != nil {
68+
installConfig.Platform.VSphere.Datacenter = anonymize.String(installConfig.Platform.VSphere.Datacenter)
69+
installConfig.Platform.VSphere.Username = anonymize.String(installConfig.Platform.VSphere.Username)
70+
installConfig.Platform.VSphere.Password = anonymize.String(installConfig.Platform.VSphere.Password)
71+
}
72+
if installConfig.Platform.OpenStack != nil {
73+
installConfig.Platform.OpenStack.Region = anonymize.String(installConfig.Platform.OpenStack.Region)
74+
installConfig.Platform.OpenStack.Cloud = anonymize.String(installConfig.Platform.OpenStack.Cloud)
75+
}
76+
5777
return installConfig
5878
}

‎pkg/gatherers/clusterconfig/cluster_config_v1_config_map_test.go

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package clusterconfig
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
"github.com/stretchr/testify/assert"
8+
corev1 "k8s.io/api/core/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
kubefake "k8s.io/client-go/kubernetes/fake"
11+
)
12+
13+
func Test_gatherClusterConfigV1(t *testing.T) {
14+
coreClient := kubefake.NewSimpleClientset()
15+
16+
_, err := coreClient.CoreV1().ConfigMaps("kube-system").Create(context.Background(), &corev1.ConfigMap{
17+
TypeMeta: metav1.TypeMeta{},
18+
ObjectMeta: metav1.ObjectMeta{
19+
Name: "cluster-config-v1",
20+
},
21+
Immutable: nil,
22+
Data: map[string]string{
23+
"install-config": "{}",
24+
},
25+
BinaryData: nil,
26+
}, metav1.CreateOptions{})
27+
assert.NoError(t, err)
28+
29+
records, errs := gatherClusterConfigV1(context.Background(), coreClient.CoreV1())
30+
assert.Empty(t, errs)
31+
32+
assert.Len(t, records, 1)
33+
assert.Equal(t, "config/configmaps/kube-system/cluster-config-v1", records[0].Name)
34+
35+
data, err := records[0].Item.Marshal(context.Background())
36+
assert.NoError(t, err)
37+
38+
installConfig := `baseDomain: \"\"\nmetadata:\n creationTimestamp: null\nplatform: {}\npullSecret: \"\"\n`
39+
40+
assert.JSONEq(t, `{
41+
"metadata": {
42+
"name": "cluster-config-v1",
43+
"namespace": "kube-system",
44+
"creationTimestamp": null
45+
},
46+
"data": {
47+
"install-config": "`+installConfig+`"
48+
}
49+
}`, string(data))
50+
}

‎pkg/gatherers/clusterconfig/image_registries.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -137,21 +137,23 @@ func anonymizeImageRegistry(config *registryv1.Config) *registryv1.Config {
137137
if config.Spec.Storage.IBMCOS != nil {
138138
anonymizeIBMCOSStorage(config.Spec.Storage.IBMCOS)
139139
}
140+
140141
if config.Status.Storage.S3 != nil {
141142
anonymizeS3Storage(config.Status.Storage.S3)
142143
}
143-
if config.Status.Storage.GCS != nil {
144-
anonymizeGCSStorage(config.Status.Storage.GCS)
145-
}
146144
if config.Status.Storage.Azure != nil {
147145
anonymizeAzureStorage(config.Status.Storage.Azure)
148146
}
147+
if config.Status.Storage.GCS != nil {
148+
anonymizeGCSStorage(config.Status.Storage.GCS)
149+
}
149150
if config.Status.Storage.Swift != nil {
150151
anonymizeSwiftStorage(config.Status.Storage.Swift)
151152
}
152153
if config.Status.Storage.IBMCOS != nil {
153154
anonymizeIBMCOSStorage(config.Status.Storage.IBMCOS)
154155
}
156+
155157
// kubectl.kubernetes.io/last-applied-configuration annotation contains complete previous resource definition
156158
// including the sensitive information as bucket, keyIDs, etc.
157159
if lac, ok := config.Annotations[lacAnnotation]; ok {

‎pkg/gatherers/clusterconfig/infrastructures.go

+20
Original file line numberDiff line numberDiff line change
@@ -42,5 +42,25 @@ func gatherClusterInfrastructure(ctx context.Context, configClient configv1clien
4242

4343
func anonymizeInfrastructure(config *configv1.Infrastructure) *configv1.Infrastructure {
4444
config.Status.InfrastructureName = anonymize.URL(config.Status.InfrastructureName)
45+
46+
if config.Status.PlatformStatus.AWS != nil {
47+
config.Status.PlatformStatus.AWS.Region = anonymize.String(config.Status.PlatformStatus.AWS.Region)
48+
}
49+
if config.Status.PlatformStatus.Azure != nil {
50+
config.Status.PlatformStatus.Azure.CloudName = configv1.AzureCloudEnvironment(
51+
anonymize.String(string(config.Status.PlatformStatus.Azure.CloudName)),
52+
)
53+
}
54+
if config.Status.PlatformStatus.GCP != nil {
55+
config.Status.PlatformStatus.GCP.Region = anonymize.String(config.Status.PlatformStatus.GCP.Region)
56+
config.Status.PlatformStatus.GCP.ProjectID = anonymize.String(config.Status.PlatformStatus.GCP.ProjectID)
57+
}
58+
if config.Status.PlatformStatus.IBMCloud != nil {
59+
config.Status.PlatformStatus.IBMCloud.Location = anonymize.String(config.Status.PlatformStatus.IBMCloud.Location)
60+
}
61+
if config.Status.PlatformStatus.OpenStack != nil {
62+
config.Status.PlatformStatus.OpenStack.CloudName = anonymize.String(config.Status.PlatformStatus.OpenStack.CloudName)
63+
}
64+
4565
return config
4666
}

‎pkg/gatherers/clusterconfig/machine_sets.go

+61-1
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,13 @@ import (
66

77
"k8s.io/apimachinery/pkg/api/errors"
88
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
910
"k8s.io/apimachinery/pkg/runtime/schema"
1011
"k8s.io/client-go/dynamic"
12+
"k8s.io/klog/v2"
1113

1214
"github.com/openshift/insights-operator/pkg/record"
15+
"github.com/openshift/insights-operator/pkg/utils/anonymize"
1316
)
1417

1518
// GatherMachineSet collects MachineSet information
@@ -52,9 +55,66 @@ func gatherMachineSet(ctx context.Context, dynamicClient dynamic.Interface) ([]r
5255
}
5356
records = append(records, record.Record{
5457
Name: recordName,
55-
Item: record.ResourceMarshaller{Resource: &machineSets.Items[i]},
58+
Item: record.ResourceMarshaller{Resource: anonymizeMachineset(&machineSets.Items[i])},
5659
})
5760
}
5861

5962
return records, nil
6063
}
64+
65+
func anonymizeMachineset(data *unstructured.Unstructured) *unstructured.Unstructured {
66+
fieldsToAnonymize := [][]string{
67+
{"spec", "template", "spec", "providerSpec", "value", "projectID"},
68+
{"spec", "template", "spec", "providerSpec", "value", "region"},
69+
{"spec", "template", "spec", "providerSpec", "value", "placement", "availabilityZone"},
70+
{"spec", "template", "spec", "providerSpec", "value", "placement", "region"},
71+
}
72+
73+
for _, fieldToAnonymize := range fieldsToAnonymize {
74+
err := anonymize.UnstructuredNestedStringField(data.Object, fieldToAnonymize...)
75+
if err != nil {
76+
klog.Infof("error during anonymizing machineset: %v", err)
77+
}
78+
}
79+
80+
return anonymizeServiceAccounts(data)
81+
}
82+
83+
func anonymizeServiceAccounts(data *unstructured.Unstructured) *unstructured.Unstructured {
84+
serviceAccounts, found, err := unstructured.NestedSlice(
85+
data.Object, "spec", "template", "spec", "providerSpec", "value", "serviceAccounts",
86+
)
87+
if !found || err != nil {
88+
klog.Infof("error during anonymizing machineset: unable to find service accounts %v %v", found, err)
89+
return data
90+
}
91+
92+
for i := range serviceAccounts {
93+
serviceAccount, ok := serviceAccounts[i].(map[string]interface{})
94+
if !ok {
95+
klog.Infof("error during anonymizing machineset: service account is not a map")
96+
continue
97+
}
98+
99+
emailI, found := serviceAccount["email"]
100+
if !found {
101+
klog.Infof("error during anonymizing machineset: email was not found in service account map")
102+
continue
103+
}
104+
105+
email, ok := emailI.(string)
106+
if !ok {
107+
klog.Infof("error during anonymizing machineset: email was not a string")
108+
continue
109+
}
110+
111+
serviceAccount["email"] = anonymize.String(email)
112+
}
113+
114+
err = unstructured.SetNestedSlice(data.Object, serviceAccounts, "spec", "template", "spec", "providerSpec", "value", "serviceAccounts")
115+
if err != nil {
116+
klog.Infof("error during anonymizing machineset: unable to set anonymized service accounts: %v", err.Error())
117+
}
118+
119+
return data
120+
}

‎pkg/gatherers/clusterconfig/nodes.go

+10-1
Original file line numberDiff line numberDiff line change
@@ -48,21 +48,30 @@ func anonymizeNode(node *corev1.Node) *corev1.Node {
4848
if isProductNamespacedKey(k) {
4949
continue
5050
}
51+
5152
node.Annotations[k] = ""
5253
}
54+
5355
for k, v := range node.Labels {
54-
if isProductNamespacedKey(k) {
56+
if isProductNamespacedKey(k) && !isRegionLabel(k) {
5557
continue
5658
}
59+
5760
node.Labels[k] = anonymize.String(v)
5861
}
62+
5963
node.Status.NodeInfo.BootID = anonymize.String(node.Status.NodeInfo.BootID)
6064
node.Status.NodeInfo.SystemUUID = anonymize.String(node.Status.NodeInfo.SystemUUID)
6165
node.Status.NodeInfo.MachineID = anonymize.String(node.Status.NodeInfo.MachineID)
6266
node.Status.Images = nil
67+
6368
return node
6469
}
6570

6671
func isProductNamespacedKey(key string) bool {
6772
return strings.Contains(key, "openshift.io/") || strings.Contains(key, "k8s.io/") || strings.Contains(key, "kubernetes.io/")
6873
}
74+
75+
func isRegionLabel(key string) bool {
76+
return key == "failure-domain.beta.kubernetes.io/region" || key == "topology.kubernetes.io/region"
77+
}

‎pkg/utils/anonymize/unstructured.go

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package anonymize
2+
3+
import (
4+
"fmt"
5+
6+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
7+
)
8+
9+
// UnstructuredNestedStringField anonymizes the nested field in the unstructured object
10+
// or returns error if it is not possible
11+
func UnstructuredNestedStringField(data map[string]interface{}, fields ...string) error {
12+
value, found, err := unstructured.NestedFieldCopy(data, fields...)
13+
if err != nil {
14+
return err
15+
}
16+
if !found {
17+
return fmt.Errorf("unable to find field '%v'", fields)
18+
}
19+
20+
valueStr, _ := value.(string)
21+
return unstructured.SetNestedField(data, String(valueStr), fields...)
22+
}

0 commit comments

Comments
 (0)