diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 11fc8cb81..71832a7b0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -154,40 +154,6 @@ rules: - list - update - watch -- apiGroups: - - replication.storage.openshift.io - resources: - - volumereplicationclasses - verbs: - - get - - list - - watch -- apiGroups: - - replication.storage.openshift.io - resources: - - volumereplications - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - replication.storage.openshift.io - resources: - - volumereplications/finalizers - verbs: - - update -- apiGroups: - - replication.storage.openshift.io - resources: - - volumereplications/status - verbs: - - get - - patch - - update - apiGroups: - security.openshift.io resourceNames: @@ -275,3 +241,29 @@ rules: - volumeattachments/status verbs: - patch +- apiGroups: + - csiaddons.openshift.io + resources: + - csiaddonsnodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - csiaddons.openshift.io + resources: + - csiaddonsnodes/finalizers + verbs: + - update +- apiGroups: + - csiaddons.openshift.io + resources: + - csiaddonsnodes/status + verbs: + - get + - patch + - update diff --git a/controllers/ibmblockcsi_controller.go b/controllers/ibmblockcsi_controller.go index 3f150ce6c..b1c49bc79 100644 --- a/controllers/ibmblockcsi_controller.go +++ b/controllers/ibmblockcsi_controller.go @@ -102,10 +102,9 @@ type IBMBlockCSIReconciler struct { // +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;watch;list;create;update;delete // +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents/status,verbs=update // +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshots,verbs=get;watch;list;update -// +kubebuilder:rbac:groups=replication.storage.openshift.io,resources=volumereplicationclasses,verbs=get;list;watch -// +kubebuilder:rbac:groups=replication.storage.openshift.io,resources=volumereplications,verbs=create;delete;get;list;patch;update;watch -// +kubebuilder:rbac:groups=replication.storage.openshift.io,resources=volumereplications/finalizers,verbs=update -// +kubebuilder:rbac:groups=replication.storage.openshift.io,resources=volumereplications/status,verbs=get;patch;update +// +kubebuilder:rbac:groups=csiaddons.openshift.io,resources=csiaddonsnodes,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=csiaddons.openshift.io,resources=csiaddonsnodes/finalizers,verbs=update +// +kubebuilder:rbac:groups=csiaddons.openshift.io,resources=csiaddonsnodes/status,verbs=get;patch;update func (r *IBMBlockCSIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (reconcile.Result, error) { reqLogger := log.WithValues("Request.Namespace", req.Namespace, "Request.Name", req.Name) reqLogger.Info("Reconciling IBMBlockCSI") @@ -360,7 +359,7 @@ func (r *IBMBlockCSIReconciler) restartControllerPodfromStatefulSet(logger logr. } func (r *IBMBlockCSIReconciler) getControllerPod(controllerStatefulset *appsv1.StatefulSet, controllerPod *corev1.Pod) error { - controllerPodName := fmt.Sprintf("%s-0", controllerStatefulset.Name) + controllerPodName := oconfig.GetControllerPodName(controllerStatefulset.Name) err := r.Get(context.TODO(), types.NamespacedName{ Name: controllerPodName, Namespace: controllerStatefulset.Namespace, diff --git a/controllers/internal/crutils/static_resource_generator.go b/controllers/internal/crutils/static_resource_generator.go index c34ececeb..0094d3088 100644 --- a/controllers/internal/crutils/static_resource_generator.go +++ b/controllers/internal/crutils/static_resource_generator.go @@ -31,7 +31,7 @@ const ( volumeGroupApiGroup string = "csi.ibm.com" storageApiGroup string = "storage.k8s.io" rbacAuthorizationApiGroup string = "rbac.authorization.k8s.io" - replicationStorageOpenshiftApiGroup string = "replication.storage.openshift.io" + csiAddonsApiGroup string = "csiaddons.openshift.io" storageClassesResource string = "storageclasses" persistentVolumesResource string = "persistentvolumes" persistentVolumeClaimsResource string = "persistentvolumeclaims" @@ -50,10 +50,9 @@ const ( volumeSnapshotsResource string = "volumesnapshots" volumeSnapshotContentsResource string = "volumesnapshotcontents" volumeSnapshotContentsStatusResource string = "volumesnapshotcontents/status" - volumeReplicationClassesResource string = "volumereplicationclasses" - volumeReplicationsResource string = "volumereplications" - volumeReplicationsFinalizersResource string = "volumereplications/finalizers" - volumeReplicationsStatusResource string = "volumereplications/status" + csiAddonsNodesResource string = "csiaddonsnodes" + csiAddonsNodesFinalizersResource string = "csiaddonsnodes/finalizers" + csiAddonsNodesStatusResource string = "csiaddonsnodes/status" eventsResource string = "events" nodesResource string = "nodes" csiNodesResource string = "csinodes" @@ -357,30 +356,25 @@ func (c *IBMBlockCSI) GenerateCSIAddonsReplicatorClusterRole() *rbacv1.ClusterRo }, Rules: []rbacv1.PolicyRule{ { - APIGroups: []string{replicationStorageOpenshiftApiGroup}, - Resources: []string{volumeReplicationClassesResource}, - Verbs: []string{verbGet, verbList, verbWatch}, + APIGroups: []string{""}, + Resources: []string{secretsResource}, + Verbs: []string{verbGet}, }, { - APIGroups: []string{replicationStorageOpenshiftApiGroup}, - Resources: []string{volumeReplicationsResource}, + APIGroups: []string{csiAddonsApiGroup}, + Resources: []string{csiAddonsNodesResource}, Verbs: []string{verbCreate, verbDelete, verbGet, verbList, verbPatch, verbUpdate, verbWatch}, }, { - APIGroups: []string{replicationStorageOpenshiftApiGroup}, - Resources: []string{volumeReplicationsFinalizersResource}, + APIGroups: []string{csiAddonsApiGroup}, + Resources: []string{csiAddonsNodesFinalizersResource}, Verbs: []string{verbUpdate}, }, { - APIGroups: []string{replicationStorageOpenshiftApiGroup}, - Resources: []string{volumeReplicationsStatusResource}, + APIGroups: []string{csiAddonsApiGroup}, + Resources: []string{csiAddonsNodesStatusResource}, Verbs: []string{verbGet, verbPatch, verbUpdate}, }, - { - APIGroups: []string{""}, - Resources: []string{secretsResource}, - Verbs: []string{verbGet}, - }, }, } } diff --git a/controllers/syncer/csi_controller.go b/controllers/syncer/csi_controller.go index df23f815a..78fdf2a74 100644 --- a/controllers/syncer/csi_controller.go +++ b/controllers/syncer/csi_controller.go @@ -136,7 +136,7 @@ func (s *csiControllerSyncer) ensurePodSpec() corev1.PodSpec { func (s *csiControllerSyncer) ensureContainersSpec() []corev1.Container { controllerPlugin := s.ensureContainer(ControllerContainerName, s.driver.GetCSIControllerImage(), - []string{"--csi-endpoint=$(CSI_ENDPOINT)"}, + []string{"--csi-endpoint=$(CSI_ENDPOINT)", "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"}, ) controllerPlugin.Resources = ensureResources("40m", "800m", "40Mi", "400Mi") @@ -208,12 +208,17 @@ func (s *csiControllerSyncer) ensureContainersSpec() []corev1.Container { ) resizer.ImagePullPolicy = s.getCSIResizerPullPolicy() - leaderElectionNamespaceFlag := fmt.Sprintf("--leader-election-namespace=%s", s.driver.Namespace) driverNameFlag := fmt.Sprintf("--driver-name=%s", config.DriverName) + statfulSetName := config.GetNameForResource(config.CSIController, s.driver.Name) + controllerPodName := fmt.Sprintf("--pod=%s", config.GetControllerPodName(statfulSetName)) + controllerPodNamespace := fmt.Sprintf("--namespace=%s", s.driver.Namespace) + controllerPort := fmt.Sprintf("--controller-port=%s", "9087") replicator := s.ensureContainer(replicatorContainerName, s.getCSIAddonsReplicatorImage(), - []string{leaderElectionNamespaceFlag, driverNameFlag, - "--csi-address=$(ADDRESS)", "--zap-log-level=5", "--rpc-timeout=30s"}, + []string{controllerPodName, controllerPodNamespace, controllerPort, + "--csi-addons-address=$(CSI_ADDONS_ENDPOINT)", + "--node-id=$(NODE_ID)", "--pod-uid=$(POD_UID)", + "--controller-ip=$(POD_IP)"}, ) replicator.ImagePullPolicy = s.getCSIAddonsReplicatorPullPolicy() @@ -328,6 +333,18 @@ func (s *csiControllerSyncer) getEnvFor(name string) []corev1.EnvVar { Name: "CSI_ENDPOINT", Value: config.CSIEndpoint, }, + { + Name: "CSI_ADDONS_ENDPOINT", + Value: config.CSIAddonsEndpoint, + }, + { + Name: "NODE_ID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, { Name: "CSI_LOGLEVEL", Value: config.DefaultLogLevel, @@ -337,9 +354,40 @@ func (s *csiControllerSyncer) getEnvFor(name string) []corev1.EnvVar { Value: strconv.FormatBool(s.driver.Spec.EnableCallHome), }, } + case replicatorContainerName: + return []corev1.EnvVar{ + { + Name: "CSI_ADDONS_ENDPOINT", + Value: config.CSIAddonsEndpoint, + }, + { + Name: "NODE_ID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + Name: "POD_UID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, + }, + }, + } case provisionerContainerName, attacherContainerName, snapshotterContainerName, - resizerContainerName, replicatorContainerName, volumeGroupContainerName: + resizerContainerName, volumeGroupContainerName: return []corev1.EnvVar{ { Name: "ADDRESS", diff --git a/deploy/installer/generated/ibm-block-csi-operator.yaml b/deploy/installer/generated/ibm-block-csi-operator.yaml index 0e52b66d3..7770c6188 100644 --- a/deploy/installer/generated/ibm-block-csi-operator.yaml +++ b/deploy/installer/generated/ibm-block-csi-operator.yaml @@ -3224,40 +3224,6 @@ rules: - list - update - watch -- apiGroups: - - replication.storage.openshift.io - resources: - - volumereplicationclasses - verbs: - - get - - list - - watch -- apiGroups: - - replication.storage.openshift.io - resources: - - volumereplications - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - replication.storage.openshift.io - resources: - - volumereplications/finalizers - verbs: - - update -- apiGroups: - - replication.storage.openshift.io - resources: - - volumereplications/status - verbs: - - get - - patch - - update - apiGroups: - security.openshift.io resourceNames: @@ -3345,6 +3311,32 @@ rules: - volumeattachments/status verbs: - patch +- apiGroups: + - csiaddons.openshift.io + resources: + - csiaddonsnodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - csiaddons.openshift.io + resources: + - csiaddonsnodes/finalizers + verbs: + - update +- apiGroups: + - csiaddons.openshift.io + resources: + - csiaddonsnodes/status + verbs: + - get + - patch + - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/deploy/olm-catalog/ibm-block-csi-operator-community/1.12.0/manifests/ibm-block-csi-operator.v1.12.0.clusterserviceversion.yaml b/deploy/olm-catalog/ibm-block-csi-operator-community/1.12.0/manifests/ibm-block-csi-operator.v1.12.0.clusterserviceversion.yaml index 24bff9cbd..6987c906c 100644 --- a/deploy/olm-catalog/ibm-block-csi-operator-community/1.12.0/manifests/ibm-block-csi-operator.v1.12.0.clusterserviceversion.yaml +++ b/deploy/olm-catalog/ibm-block-csi-operator-community/1.12.0/manifests/ibm-block-csi-operator.v1.12.0.clusterserviceversion.yaml @@ -488,17 +488,9 @@ spec: - update - watch - apiGroups: - - replication.storage.openshift.io + - csiaddons.openshift.io resources: - - volumereplicationclasses - verbs: - - get - - list - - watch - - apiGroups: - - replication.storage.openshift.io - resources: - - volumereplications + - csiaddonsnodes verbs: - create - delete @@ -508,15 +500,15 @@ spec: - update - watch - apiGroups: - - replication.storage.openshift.io + - csiaddons.openshift.io resources: - - volumereplications/finalizers + - csiaddonsnodes/finalizers verbs: - update - apiGroups: - - replication.storage.openshift.io + - csiaddons.openshift.io resources: - - volumereplications/status + - csiaddonsnodes/status verbs: - get - patch diff --git a/deploy/olm-catalog/ibm-block-csi-operator/1.12.0/manifests/ibm-block-csi-operator.clusterserviceversion.yaml b/deploy/olm-catalog/ibm-block-csi-operator/1.12.0/manifests/ibm-block-csi-operator.clusterserviceversion.yaml index 57daf012e..283734b22 100644 --- a/deploy/olm-catalog/ibm-block-csi-operator/1.12.0/manifests/ibm-block-csi-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/ibm-block-csi-operator/1.12.0/manifests/ibm-block-csi-operator.clusterserviceversion.yaml @@ -478,17 +478,9 @@ spec: - update - watch - apiGroups: - - replication.storage.openshift.io + - csiaddons.openshift.io resources: - - volumereplicationclasses - verbs: - - get - - list - - watch - - apiGroups: - - replication.storage.openshift.io - resources: - - volumereplications + - csiaddonsnodes verbs: - create - delete @@ -498,15 +490,15 @@ spec: - update - watch - apiGroups: - - replication.storage.openshift.io + - csiaddons.openshift.io resources: - - volumereplications/finalizers + - csiaddonsnodes/finalizers verbs: - update - apiGroups: - - replication.storage.openshift.io + - csiaddons.openshift.io resources: - - volumereplications/status + - csiaddonsnodes/status verbs: - get - patch diff --git a/pkg/config/constants.go b/pkg/config/constants.go index 3efa696d4..b0f75d8b5 100644 --- a/pkg/config/constants.go +++ b/pkg/config/constants.go @@ -61,4 +61,5 @@ const ( NodeRegistrarSocketPath = "/var/lib/kubelet/plugins/block.csi.ibm.com/csi.sock" CSIEndpoint = "unix:///var/lib/csi/sockets/pluginproxy/csi.sock" CSINodeEndpoint = "unix:///csi/csi.sock" + CSIAddonsEndpoint = "unix:///var/lib/csi/sockets/pluginproxy/csi-addons.sock" ) diff --git a/pkg/config/resources.go b/pkg/config/resources.go index 540d10f8d..de5b0fc1b 100644 --- a/pkg/config/resources.go +++ b/pkg/config/resources.go @@ -74,3 +74,7 @@ func GetNameForResource(name ResourceName, driverName string) string { return fmt.Sprintf("%s-%s", driverName, name) } } + +func GetControllerPodName(statefulSetName string) string { + return fmt.Sprintf("%s-0", statefulSetName) +}