diff --git a/config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml b/config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml index 1a3bb00f9b..0550a17b94 100644 --- a/config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml +++ b/config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml @@ -4334,8 +4334,6 @@ spec: required: - volumeSnapshotClassName type: object - required: - - pgbackrest type: object config: properties: @@ -16873,7 +16871,6 @@ spec: - name x-kubernetes-list-type: map required: - - backups - instances - postgresVersion type: object diff --git a/config/rbac/cluster/role.yaml b/config/rbac/cluster/role.yaml index 64c58a134c..1119eb0d5a 100644 --- a/config/rbac/cluster/role.yaml +++ b/config/rbac/cluster/role.yaml @@ -10,6 +10,7 @@ rules: - configmaps - persistentvolumeclaims - secrets + - serviceaccounts - services verbs: - create @@ -54,16 +55,6 @@ rules: - list - patch - watch -- apiGroups: - - '' - resources: - - serviceaccounts - verbs: - - create - - get - - list - - patch - - watch - apiGroups: - apps resources: @@ -167,6 +158,7 @@ rules: - roles verbs: - create + - delete - get - list - patch diff --git a/config/rbac/namespace/role.yaml b/config/rbac/namespace/role.yaml index 2193a7b674..d4ede32c6c 100644 --- a/config/rbac/namespace/role.yaml +++ b/config/rbac/namespace/role.yaml @@ -10,6 +10,7 @@ rules: - configmaps - persistentvolumeclaims - secrets + - serviceaccounts - services verbs: - create @@ -54,16 +55,6 @@ rules: - list - patch - watch -- apiGroups: - - '' - resources: - - serviceaccounts - verbs: - - create - - get - - list - - patch - - watch - apiGroups: - apps resources: @@ -167,6 +158,7 @@ rules: - roles verbs: - create + - delete - get - list - patch diff --git a/internal/controller/postgrescluster/cluster.go b/internal/controller/postgrescluster/cluster.go index 8d32679db3..2018dc3f95 100644 --- a/internal/controller/postgrescluster/cluster.go +++ b/internal/controller/postgrescluster/cluster.go @@ -290,7 +290,9 @@ func (r *Reconciler) reconcileClusterReplicaService( func (r *Reconciler) reconcileDataSource(ctx context.Context, cluster *v1beta1.PostgresCluster, observed *observedInstances, clusterVolumes []corev1.PersistentVolumeClaim, - rootCA *pki.RootCertificateAuthority) (bool, error) { + rootCA *pki.RootCertificateAuthority, + backupsSpecFound bool, +) (bool, error) { // a hash func to hash the pgBackRest restore options hashFunc := func(jobConfigs []string) (string, error) { @@ -413,7 +415,8 @@ func (r *Reconciler) reconcileDataSource(ctx context.Context, switch { case dataSource != nil: if err := r.reconcilePostgresClusterDataSource(ctx, cluster, dataSource, - configHash, clusterVolumes, rootCA); err != nil { + configHash, clusterVolumes, rootCA, + backupsSpecFound); err != nil { return true, err } case cloudDataSource != nil: diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go index 098b38b30d..c038d36e68 100644 --- a/internal/controller/postgrescluster/controller.go +++ b/internal/controller/postgrescluster/controller.go @@ -162,21 +162,23 @@ func (r *Reconciler) Reconcile( } var ( - clusterConfigMap *corev1.ConfigMap - clusterReplicationSecret *corev1.Secret - clusterPodService *corev1.Service - clusterVolumes []corev1.PersistentVolumeClaim - instanceServiceAccount *corev1.ServiceAccount - instances *observedInstances - patroniLeaderService *corev1.Service - primaryCertificate *corev1.SecretProjection - primaryService *corev1.Service - replicaService *corev1.Service - rootCA *pki.RootCertificateAuthority - monitoringSecret *corev1.Secret - exporterQueriesConfig *corev1.ConfigMap - exporterWebConfig *corev1.ConfigMap - err error + clusterConfigMap *corev1.ConfigMap + clusterReplicationSecret *corev1.Secret + clusterPodService *corev1.Service + clusterVolumes []corev1.PersistentVolumeClaim + instanceServiceAccount *corev1.ServiceAccount + instances *observedInstances + patroniLeaderService *corev1.Service + primaryCertificate *corev1.SecretProjection + primaryService *corev1.Service + replicaService *corev1.Service + rootCA *pki.RootCertificateAuthority + monitoringSecret *corev1.Secret + exporterQueriesConfig *corev1.ConfigMap + exporterWebConfig *corev1.ConfigMap + err error + backupsSpecFound bool + backupsReconciliationAllowed bool ) patchClusterStatus := func() error { @@ -214,13 +216,34 @@ func (r *Reconciler) Reconcile( meta.RemoveStatusCondition(&cluster.Status.Conditions, v1beta1.PostgresClusterProgressing) } + if err == nil { + backupsSpecFound, backupsReconciliationAllowed, err = r.BackupsEnabled(ctx, cluster) + + // If we cannot reconcile because the backup reconciliation is paused, set a condition and exit + if !backupsReconciliationAllowed { + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: v1beta1.PostgresClusterProgressing, + Status: metav1.ConditionFalse, + Reason: "Paused", + Message: "Reconciliation is paused: please fill in spec.backups " + + "or add the postgres-operator.crunchydata.com/authorizeBackupRemoval " + + "annotation to authorize backup removal.", + + ObservedGeneration: cluster.GetGeneration(), + }) + return runtime.ErrorWithBackoff(patchClusterStatus()) + } else { + meta.RemoveStatusCondition(&cluster.Status.Conditions, v1beta1.PostgresClusterProgressing) + } + } + pgHBAs := postgres.NewHBAs() pgmonitor.PostgreSQLHBAs(cluster, &pgHBAs) pgbouncer.PostgreSQL(cluster, &pgHBAs) pgParameters := postgres.NewParameters() pgaudit.PostgreSQLParameters(&pgParameters) - pgbackrest.PostgreSQL(cluster, &pgParameters) + pgbackrest.PostgreSQL(cluster, &pgParameters, backupsSpecFound) pgmonitor.PostgreSQLParameters(cluster, &pgParameters) // Set huge_pages = try if a hugepages resource limit > 0, otherwise set "off" @@ -287,7 +310,7 @@ func (r *Reconciler) Reconcile( // the controller should return early while data initialization is in progress, after // which it will indicate that an early return is no longer needed, and reconciliation // can proceed normally. - returnEarly, err := r.reconcileDataSource(ctx, cluster, instances, clusterVolumes, rootCA) + returnEarly, err := r.reconcileDataSource(ctx, cluster, instances, clusterVolumes, rootCA, backupsSpecFound) if err != nil || returnEarly { return runtime.ErrorWithBackoff(errors.Join(err, patchClusterStatus())) } @@ -329,7 +352,9 @@ func (r *Reconciler) Reconcile( err = r.reconcileInstanceSets( ctx, cluster, clusterConfigMap, clusterReplicationSecret, rootCA, clusterPodService, instanceServiceAccount, instances, patroniLeaderService, - primaryCertificate, clusterVolumes, exporterQueriesConfig, exporterWebConfig) + primaryCertificate, clusterVolumes, exporterQueriesConfig, exporterWebConfig, + backupsSpecFound, + ) } if err == nil { @@ -341,7 +366,8 @@ func (r *Reconciler) Reconcile( if err == nil { var next reconcile.Result - if next, err = r.reconcilePGBackRest(ctx, cluster, instances, rootCA); err == nil && !next.IsZero() { + if next, err = r.reconcilePGBackRest(ctx, cluster, + instances, rootCA, backupsSpecFound); err == nil && !next.IsZero() { result.Requeue = result.Requeue || next.Requeue if next.RequeueAfter > 0 { result.RequeueAfter = next.RequeueAfter diff --git a/internal/controller/postgrescluster/instance.go b/internal/controller/postgrescluster/instance.go index beaaabcced..fceeee9d6d 100644 --- a/internal/controller/postgrescluster/instance.go +++ b/internal/controller/postgrescluster/instance.go @@ -346,7 +346,7 @@ func (r *Reconciler) observeInstances( status.DesiredPGDataVolume = make(map[string]string) for _, instance := range observed.bySet[name] { - status.Replicas += int32(len(instance.Pods)) + status.Replicas += int32(len(instance.Pods)) //nolint:gosec if ready, known := instance.IsReady(); known && ready { status.ReadyReplicas++ @@ -604,6 +604,7 @@ func (r *Reconciler) reconcileInstanceSets( primaryCertificate *corev1.SecretProjection, clusterVolumes []corev1.PersistentVolumeClaim, exporterQueriesConfig, exporterWebConfig *corev1.ConfigMap, + backupsSpecFound bool, ) error { // Go through the observed instances and check if a primary has been determined. @@ -640,7 +641,9 @@ func (r *Reconciler) reconcileInstanceSets( rootCA, clusterPodService, instanceServiceAccount, patroniLeaderService, primaryCertificate, findAvailableInstanceNames(*set, instances, clusterVolumes), - numInstancePods, clusterVolumes, exporterQueriesConfig, exporterWebConfig) + numInstancePods, clusterVolumes, exporterQueriesConfig, exporterWebConfig, + backupsSpecFound, + ) if err == nil { err = r.reconcileInstanceSetPodDisruptionBudget(ctx, cluster, set) @@ -1079,6 +1082,7 @@ func (r *Reconciler) scaleUpInstances( numInstancePods int, clusterVolumes []corev1.PersistentVolumeClaim, exporterQueriesConfig, exporterWebConfig *corev1.ConfigMap, + backupsSpecFound bool, ) ([]*appsv1.StatefulSet, error) { log := logging.FromContext(ctx) @@ -1123,6 +1127,7 @@ func (r *Reconciler) scaleUpInstances( rootCA, clusterPodService, instanceServiceAccount, patroniLeaderService, primaryCertificate, instances[i], numInstancePods, clusterVolumes, exporterQueriesConfig, exporterWebConfig, + backupsSpecFound, ) } if err == nil { @@ -1152,6 +1157,7 @@ func (r *Reconciler) reconcileInstance( numInstancePods int, clusterVolumes []corev1.PersistentVolumeClaim, exporterQueriesConfig, exporterWebConfig *corev1.ConfigMap, + backupsSpecFound bool, ) error { log := logging.FromContext(ctx).WithValues("instance", instance.Name) ctx = logging.NewContext(ctx, log) @@ -1198,8 +1204,10 @@ func (r *Reconciler) reconcileInstance( postgresDataVolume, postgresWALVolume, tablespaceVolumes, &instance.Spec.Template.Spec) - addPGBackRestToInstancePodSpec( - ctx, cluster, instanceCertificates, &instance.Spec.Template.Spec) + if backupsSpecFound { + addPGBackRestToInstancePodSpec( + ctx, cluster, instanceCertificates, &instance.Spec.Template.Spec) + } err = patroni.InstancePod( ctx, cluster, clusterConfigMap, clusterPodService, patroniLeaderService, diff --git a/internal/controller/postgrescluster/pgbackrest.go b/internal/controller/postgrescluster/pgbackrest.go index 85465ddbf2..34414fe2cd 100644 --- a/internal/controller/postgrescluster/pgbackrest.go +++ b/internal/controller/postgrescluster/pgbackrest.go @@ -19,6 +19,7 @@ import ( "context" "fmt" "io" + "reflect" "regexp" "sort" "strings" @@ -116,11 +117,14 @@ var regexRepoIndex = regexp.MustCompile(`\d+`) // RepoResources is used to store various resources for pgBackRest repositories and // repository hosts type RepoResources struct { + hosts []*appsv1.StatefulSet cronjobs []*batchv1.CronJob manualBackupJobs []*batchv1.Job replicaCreateBackupJobs []*batchv1.Job - hosts []*appsv1.StatefulSet pvcs []*corev1.PersistentVolumeClaim + sas []*corev1.ServiceAccount + roles []*rbacv1.Role + rolebindings []*rbacv1.RoleBinding } // applyRepoHostIntent ensures the pgBackRest repository host StatefulSet is synchronized with the @@ -191,24 +195,44 @@ func (r *Reconciler) applyRepoVolumeIntent(ctx context.Context, return repo, nil } +// +kubebuilder:rbac:groups="apps",resources="statefulsets",verbs={list} +// +kubebuilder:rbac:groups="batch",resources="cronjobs",verbs={list} +// +kubebuilder:rbac:groups="batch",resources="jobs",verbs={list} +// +kubebuilder:rbac:groups="",resources="configmaps",verbs={list} +// +kubebuilder:rbac:groups="",resources="persistentvolumeclaims",verbs={list} +// +kubebuilder:rbac:groups="",resources="secrets",verbs={list} +// +kubebuilder:rbac:groups="",resources="serviceaccounts",verbs={list} +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources="roles",verbs={list} +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources="rolebindings",verbs={list} + // getPGBackRestResources returns the existing pgBackRest resources that should utilized by the // PostgresCluster controller during reconciliation. Any items returned are verified to be owned // by the PostgresCluster controller and still applicable per the current PostgresCluster spec. // Additionally, and resources identified that no longer correspond to any current configuration // are deleted. func (r *Reconciler) getPGBackRestResources(ctx context.Context, - postgresCluster *v1beta1.PostgresCluster) (*RepoResources, error) { + postgresCluster *v1beta1.PostgresCluster, + backupsSpecFound bool, +) (*RepoResources, error) { repoResources := &RepoResources{} gvks := []schema.GroupVersionKind{{ - Group: corev1.SchemeGroupVersion.Group, - Version: corev1.SchemeGroupVersion.Version, - Kind: "ConfigMapList", + Group: appsv1.SchemeGroupVersion.Group, + Version: appsv1.SchemeGroupVersion.Version, + Kind: "StatefulSetList", + }, { + Group: batchv1.SchemeGroupVersion.Group, + Version: batchv1.SchemeGroupVersion.Version, + Kind: "CronJobList", }, { Group: batchv1.SchemeGroupVersion.Group, Version: batchv1.SchemeGroupVersion.Version, Kind: "JobList", + }, { + Group: corev1.SchemeGroupVersion.Group, + Version: corev1.SchemeGroupVersion.Version, + Kind: "ConfigMapList", }, { Group: corev1.SchemeGroupVersion.Group, Version: corev1.SchemeGroupVersion.Version, @@ -218,13 +242,17 @@ func (r *Reconciler) getPGBackRestResources(ctx context.Context, Version: corev1.SchemeGroupVersion.Version, Kind: "SecretList", }, { - Group: appsv1.SchemeGroupVersion.Group, - Version: appsv1.SchemeGroupVersion.Version, - Kind: "StatefulSetList", + Group: corev1.SchemeGroupVersion.Group, + Version: corev1.SchemeGroupVersion.Version, + Kind: "ServiceAccountList", }, { - Group: batchv1.SchemeGroupVersion.Group, - Version: batchv1.SchemeGroupVersion.Version, - Kind: "CronJobList", + Group: rbacv1.SchemeGroupVersion.Group, + Version: rbacv1.SchemeGroupVersion.Version, + Kind: "RoleList", + }, { + Group: rbacv1.SchemeGroupVersion.Group, + Version: rbacv1.SchemeGroupVersion.Version, + Kind: "RoleBindingList", }} selector := naming.PGBackRestSelector(postgresCluster.GetName()) @@ -240,7 +268,7 @@ func (r *Reconciler) getPGBackRestResources(ctx context.Context, continue } - owned, err := r.cleanupRepoResources(ctx, postgresCluster, uList.Items) + owned, err := r.cleanupRepoResources(ctx, postgresCluster, uList.Items, backupsSpecFound) if err != nil { return nil, errors.WithStack(err) } @@ -262,8 +290,11 @@ func (r *Reconciler) getPGBackRestResources(ctx context.Context, } // +kubebuilder:rbac:groups="",resources="persistentvolumeclaims",verbs={delete} +// +kubebuilder:rbac:groups="",resources="serviceaccounts",verbs={delete} // +kubebuilder:rbac:groups="apps",resources="statefulsets",verbs={delete} // +kubebuilder:rbac:groups="batch",resources="cronjobs",verbs={delete} +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources="roles",verbs={delete} +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources="rolebindings",verbs={delete} // cleanupRepoResources cleans up pgBackRest repository resources that should no longer be // reconciled by deleting them. This includes deleting repos (i.e. PersistentVolumeClaims) that @@ -271,7 +302,9 @@ func (r *Reconciler) getPGBackRestResources(ctx context.Context, // pgBackRest repository host resources if a repository host is no longer configured. func (r *Reconciler) cleanupRepoResources(ctx context.Context, postgresCluster *v1beta1.PostgresCluster, - ownedResources []unstructured.Unstructured) ([]unstructured.Unstructured, error) { + ownedResources []unstructured.Unstructured, + backupsSpecFound bool, +) ([]unstructured.Unstructured, error) { // stores the resources that should not be deleted ownedNoDelete := []unstructured.Unstructured{} @@ -286,11 +319,17 @@ func (r *Reconciler) cleanupRepoResources(ctx context.Context, // spec switch { case hasLabel(naming.LabelPGBackRestConfig): + if !backupsSpecFound { + break + } // Simply add the things we never want to delete (e.g. the pgBackRest configuration) // to the slice and do not delete ownedNoDelete = append(ownedNoDelete, owned) delete = false case hasLabel(naming.LabelPGBackRestDedicated): + if !backupsSpecFound { + break + } // Any resources from before 5.1 that relate to the previously required // SSH configuration should be deleted. // TODO(tjmoore4): This can be removed once 5.0 is EOL. @@ -302,6 +341,9 @@ func (r *Reconciler) cleanupRepoResources(ctx context.Context, delete = false } case hasLabel(naming.LabelPGBackRestRepoVolume): + if !backupsSpecFound { + break + } // If a volume (PVC) is identified for a repo that no longer exists in the // spec then delete it. Otherwise add it to the slice and continue. for _, repo := range postgresCluster.Spec.Backups.PGBackRest.Repos { @@ -314,6 +356,9 @@ func (r *Reconciler) cleanupRepoResources(ctx context.Context, } } case hasLabel(naming.LabelPGBackRestBackup): + if !backupsSpecFound { + break + } // If a Job is identified for a repo that no longer exists in the spec then // delete it. Otherwise add it to the slice and continue. for _, repo := range postgresCluster.Spec.Backups.PGBackRest.Repos { @@ -323,6 +368,9 @@ func (r *Reconciler) cleanupRepoResources(ctx context.Context, } } case hasLabel(naming.LabelPGBackRestCronJob): + if !backupsSpecFound { + break + } for _, repo := range postgresCluster.Spec.Backups.PGBackRest.Repos { if repo.Name == owned.GetLabels()[naming.LabelPGBackRestRepo] { if backupScheduleFound(repo, @@ -334,6 +382,9 @@ func (r *Reconciler) cleanupRepoResources(ctx context.Context, } } case hasLabel(naming.LabelPGBackRestRestore): + if !backupsSpecFound { + break + } // When a cluster is prepared for restore, the system identifier is removed from status // and the cluster is therefore no longer bootstrapped. Only once the restore Job is // complete will the cluster then be bootstrapped again, which means by the time we @@ -343,6 +394,12 @@ func (r *Reconciler) cleanupRepoResources(ctx context.Context, ownedNoDelete = append(ownedNoDelete, owned) delete = false } + case hasLabel(naming.LabelPGBackRest): + if !backupsSpecFound { + break + } + ownedNoDelete = append(ownedNoDelete, owned) + delete = false } // If nothing has specified that the resource should not be deleted, then delete @@ -382,6 +439,24 @@ func unstructuredToRepoResources(kind string, repoResources *RepoResources, uList *unstructured.UnstructuredList) error { switch kind { + case "StatefulSetList": + var stsList appsv1.StatefulSetList + if err := runtime.DefaultUnstructuredConverter. + FromUnstructured(uList.UnstructuredContent(), &stsList); err != nil { + return errors.WithStack(err) + } + for i := range stsList.Items { + repoResources.hosts = append(repoResources.hosts, &stsList.Items[i]) + } + case "CronJobList": + var cronList batchv1.CronJobList + if err := runtime.DefaultUnstructuredConverter. + FromUnstructured(uList.UnstructuredContent(), &cronList); err != nil { + return errors.WithStack(err) + } + for i := range cronList.Items { + repoResources.cronjobs = append(repoResources.cronjobs, &cronList.Items[i]) + } case "JobList": var jobList batchv1.JobList if err := runtime.DefaultUnstructuredConverter. @@ -399,6 +474,9 @@ func unstructuredToRepoResources(kind string, repoResources *RepoResources, append(repoResources.manualBackupJobs, &jobList.Items[i]) } } + case "ConfigMapList": + // Repository host now uses mTLS for encryption, authentication, and authorization. + // Configmaps for SSHD are no longer managed here. case "PersistentVolumeClaimList": var pvcList corev1.PersistentVolumeClaimList if err := runtime.DefaultUnstructuredConverter. @@ -408,32 +486,38 @@ func unstructuredToRepoResources(kind string, repoResources *RepoResources, for i := range pvcList.Items { repoResources.pvcs = append(repoResources.pvcs, &pvcList.Items[i]) } - case "StatefulSetList": - var stsList appsv1.StatefulSetList + case "SecretList": + // Repository host now uses mTLS for encryption, authentication, and authorization. + // Secrets for SSHD are no longer managed here. + // TODO(tjmoore4): Consider adding all pgBackRest secrets to RepoResources to + // observe all pgBackRest secrets in one place. + case "ServiceAccountList": + var saList corev1.ServiceAccountList if err := runtime.DefaultUnstructuredConverter. - FromUnstructured(uList.UnstructuredContent(), &stsList); err != nil { + FromUnstructured(uList.UnstructuredContent(), &saList); err != nil { return errors.WithStack(err) } - for i := range stsList.Items { - repoResources.hosts = append(repoResources.hosts, &stsList.Items[i]) + for i := range saList.Items { + repoResources.sas = append(repoResources.sas, &saList.Items[i]) } - case "CronJobList": - var cronList batchv1.CronJobList + case "RoleList": + var roleList rbacv1.RoleList if err := runtime.DefaultUnstructuredConverter. - FromUnstructured(uList.UnstructuredContent(), &cronList); err != nil { + FromUnstructured(uList.UnstructuredContent(), &roleList); err != nil { return errors.WithStack(err) } - for i := range cronList.Items { - repoResources.cronjobs = append(repoResources.cronjobs, &cronList.Items[i]) + for i := range roleList.Items { + repoResources.roles = append(repoResources.roles, &roleList.Items[i]) + } + case "RoleBindingList": + var rb rbacv1.RoleBindingList + if err := runtime.DefaultUnstructuredConverter. + FromUnstructured(uList.UnstructuredContent(), &rb); err != nil { + return errors.WithStack(err) + } + for i := range rb.Items { + repoResources.rolebindings = append(repoResources.rolebindings, &rb.Items[i]) } - case "ConfigMapList": - // Repository host now uses mTLS for encryption, authentication, and authorization. - // Configmaps for SSHD are no longer managed here. - case "SecretList": - // Repository host now uses mTLS for encryption, authentication, and authorization. - // Secrets for SSHD are no longer managed here. - // TODO(tjmoore4): Consider adding all pgBackRest secrets to RepoResources to - // observe all pgBackRest secrets in one place. default: return fmt.Errorf("unexpected kind %q", kind) } @@ -1265,13 +1349,15 @@ func (r *Reconciler) generateRestoreJobIntent(cluster *v1beta1.PostgresCluster, func (r *Reconciler) reconcilePGBackRest(ctx context.Context, postgresCluster *v1beta1.PostgresCluster, instances *observedInstances, - rootCA *pki.RootCertificateAuthority) (reconcile.Result, error) { + rootCA *pki.RootCertificateAuthority, + backupsSpecFound bool, +) (reconcile.Result, error) { // add some additional context about what component is being reconciled log := logging.FromContext(ctx).WithValues("reconciler", "pgBackRest") - // if nil, create the pgBackRest status that will be updated when reconciling various - // pgBackRest resources + // if nil, create the pgBackRest status that will be updated when + // reconciling various pgBackRest resources if postgresCluster.Status.PGBackRest == nil { postgresCluster.Status.PGBackRest = &v1beta1.PGBackRestStatus{} } @@ -1282,12 +1368,19 @@ func (r *Reconciler) reconcilePGBackRest(ctx context.Context, // Get all currently owned pgBackRest resources in the environment as needed for // reconciliation. This includes deleting resources that should no longer exist per the // current spec (e.g. if repos, repo hosts, etc. have been removed). - repoResources, err := r.getPGBackRestResources(ctx, postgresCluster) + repoResources, err := r.getPGBackRestResources(ctx, postgresCluster, backupsSpecFound) if err != nil { // exit early if can't get and clean existing resources as needed to reconcile return reconcile.Result{}, errors.WithStack(err) } + // At this point, reconciliation is allowed, so if no backups spec is found + // clear the status and exit + if !backupsSpecFound { + postgresCluster.Status.PGBackRest = &v1beta1.PGBackRestStatus{} + return result, nil + } + var repoHost *appsv1.StatefulSet var repoHostName string // reconcile the pgbackrest repository host @@ -1408,7 +1501,9 @@ func (r *Reconciler) reconcilePGBackRest(ctx context.Context, func (r *Reconciler) reconcilePostgresClusterDataSource(ctx context.Context, cluster *v1beta1.PostgresCluster, dataSource *v1beta1.PostgresClusterDataSource, configHash string, clusterVolumes []corev1.PersistentVolumeClaim, - rootCA *pki.RootCertificateAuthority) error { + rootCA *pki.RootCertificateAuthority, + backupsSpecFound bool, +) error { // grab cluster, namespaces and repo name information from the data source sourceClusterName := dataSource.ClusterName @@ -1490,7 +1585,7 @@ func (r *Reconciler) reconcilePostgresClusterDataSource(ctx context.Context, // Note that function reconcilePGBackRest only uses forCluster in observedInstances. result, err := r.reconcilePGBackRest(ctx, cluster, &observedInstances{ forCluster: []*Instance{instance}, - }, rootCA) + }, rootCA, backupsSpecFound) if err != nil || result != (reconcile.Result{}) { return fmt.Errorf("unable to reconcile pgBackRest as needed to initialize "+ "PostgreSQL data for the cluster: %w", err) @@ -2915,3 +3010,94 @@ func (r *Reconciler) reconcilePGBackRestCronJob( } return err } + +// BackupsEnabled checks the state of the backups (i.e., if backups are in the spec, +// if a repo-host StatefulSet exists, if the annotation permitting backup deletion exists) +// and determines whether reconciliation is allowed. +// Reconciliation of backup-related Kubernetes objects is paused if +// - a user created a cluster with backups; +// - the cluster is updated to remove backups; +// - the annotation authorizing that removal is missing. +// +// This function also returns whether the spec has a defined backups or not. +func (r *Reconciler) BackupsEnabled( + ctx context.Context, + postgresCluster *v1beta1.PostgresCluster, +) ( + backupsSpecFound bool, + backupsReconciliationAllowed bool, + err error, +) { + specFound, stsNotFound, annotationFound, err := r.ObserveBackupUniverse(ctx, postgresCluster) + + switch { + case err != nil: + case specFound: + backupsSpecFound = true + backupsReconciliationAllowed = true + case annotationFound || stsNotFound: + backupsReconciliationAllowed = true + case !annotationFound && !stsNotFound: + // Destroying backups is a two key operation: + // 1. You must remove the backups section of the spec. + // 2. You must apply an annotation to the cluster. + // The existence of a StatefulSet without the backups spec is + // evidence of key 1 being turned without key 2 being turned + // -- block reconciliation until the annotation is added. + backupsReconciliationAllowed = false + default: + backupsReconciliationAllowed = false + } + return backupsSpecFound, backupsReconciliationAllowed, err +} + +// ObserveBackupUniverse returns +// - whether the spec has backups defined; +// - whether the repo-host statefulset exists; +// - whether the cluster has the annotation authorizing backup removal. +func (r *Reconciler) ObserveBackupUniverse(ctx context.Context, + postgresCluster *v1beta1.PostgresCluster, +) ( + backupsSpecFound bool, + repoHostStatefulSetNotFound bool, + backupsRemovalAnnotationFound bool, + err error, +) { + + // Does the cluster have a blank Backups section + backupsSpecFound = !reflect.DeepEqual(postgresCluster.Spec.Backups, v1beta1.Backups{PGBackRest: v1beta1.PGBackRestArchive{}}) + + // Does the repo-host StatefulSet exist? + name := fmt.Sprintf("%s-%s", postgresCluster.GetName(), "repo-host") + existing := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: postgresCluster.Namespace, + Name: name, + }, + } + err = errors.WithStack( + r.Client.Get(ctx, client.ObjectKeyFromObject(existing), existing)) + repoHostStatefulSetNotFound = apierrors.IsNotFound(err) + + // If we have an error that is not related to a missing repo-host StatefulSet, + // we return an error and expect the calling function to correctly stop processing. + if err != nil && !repoHostStatefulSetNotFound { + return true, false, false, err + } + + backupsRemovalAnnotationFound = authorizeBackupRemovalAnnotationPresent(postgresCluster) + + // If we have reached this point, the err is either nil or an IsNotFound error + // which we do not care about; hence, pass nil rather than the err + return backupsSpecFound, repoHostStatefulSetNotFound, backupsRemovalAnnotationFound, nil +} + +func authorizeBackupRemovalAnnotationPresent(postgresCluster *v1beta1.PostgresCluster) bool { + annotations := postgresCluster.GetAnnotations() + for annotation := range annotations { + if annotation == naming.AuthorizeBackupRemovalAnnotation { + return annotations[naming.AuthorizeBackupRemovalAnnotation] == "true" + } + } + return false +} diff --git a/internal/controller/postgrescluster/pgbackrest_test.go b/internal/controller/postgrescluster/pgbackrest_test.go index 5b67da0bca..5cf331909f 100644 --- a/internal/controller/postgrescluster/pgbackrest_test.go +++ b/internal/controller/postgrescluster/pgbackrest_test.go @@ -197,137 +197,137 @@ func TestReconcilePGBackRest(t *testing.T) { }) t.Cleanup(func() { teardownManager(cancel, t) }) - clusterName := "hippocluster" - clusterUID := "hippouid" + t.Run("run reconcile with backups defined", func(t *testing.T) { + clusterName := "hippocluster" + clusterUID := "hippouid" - ns := setupNamespace(t, tClient) - - // create a PostgresCluster to test with - postgresCluster := fakePostgresCluster(clusterName, ns.GetName(), clusterUID, true) + ns := setupNamespace(t, tClient) + // create a PostgresCluster to test with + postgresCluster := fakePostgresCluster(clusterName, ns.GetName(), clusterUID, true) - // create a service account to test with - serviceAccount, err := r.reconcilePGBackRestRBAC(ctx, postgresCluster) - assert.NilError(t, err) - assert.Assert(t, serviceAccount != nil) + // create a service account to test with + serviceAccount, err := r.reconcilePGBackRestRBAC(ctx, postgresCluster) + assert.NilError(t, err) + assert.Assert(t, serviceAccount != nil) - // create the 'observed' instances and set the leader - instances := &observedInstances{ - forCluster: []*Instance{{Name: "instance1", - Pods: []*corev1.Pod{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{naming.LabelRole: naming.RolePatroniLeader}, - }, - Spec: corev1.PodSpec{}, - }}, - }, {Name: "instance2"}, {Name: "instance3"}}, - } + // create the 'observed' instances and set the leader + instances := &observedInstances{ + forCluster: []*Instance{{Name: "instance1", + Pods: []*corev1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{naming.LabelRole: naming.RolePatroniLeader}, + }, + Spec: corev1.PodSpec{}, + }}, + }, {Name: "instance2"}, {Name: "instance3"}}, + } - // set status - postgresCluster.Status = v1beta1.PostgresClusterStatus{ - Patroni: v1beta1.PatroniStatus{SystemIdentifier: "12345abcde"}, - PGBackRest: &v1beta1.PGBackRestStatus{ - RepoHost: &v1beta1.RepoHostStatus{Ready: true}, - Repos: []v1beta1.RepoStatus{{Name: "repo1", StanzaCreated: true}}}, - } + // set status + postgresCluster.Status = v1beta1.PostgresClusterStatus{ + Patroni: v1beta1.PatroniStatus{SystemIdentifier: "12345abcde"}, + PGBackRest: &v1beta1.PGBackRestStatus{ + RepoHost: &v1beta1.RepoHostStatus{Ready: true}, + Repos: []v1beta1.RepoStatus{{Name: "repo1", StanzaCreated: true}}}, + } - // set conditions - clusterConditions := map[string]metav1.ConditionStatus{ - ConditionRepoHostReady: metav1.ConditionTrue, - ConditionReplicaCreate: metav1.ConditionTrue, - } - for condition, status := range clusterConditions { - meta.SetStatusCondition(&postgresCluster.Status.Conditions, metav1.Condition{ - Type: condition, Reason: "testing", Status: status}) - } + // set conditions + clusterConditions := map[string]metav1.ConditionStatus{ + ConditionRepoHostReady: metav1.ConditionTrue, + ConditionReplicaCreate: metav1.ConditionTrue, + } + for condition, status := range clusterConditions { + meta.SetStatusCondition(&postgresCluster.Status.Conditions, metav1.Condition{ + Type: condition, Reason: "testing", Status: status}) + } - rootCA, err := pki.NewRootCertificateAuthority() - assert.NilError(t, err) + rootCA, err := pki.NewRootCertificateAuthority() + assert.NilError(t, err) - result, err := r.reconcilePGBackRest(ctx, postgresCluster, instances, rootCA) - if err != nil || result != (reconcile.Result{}) { - t.Errorf("unable to reconcile pgBackRest: %v", err) - } + result, err := r.reconcilePGBackRest(ctx, postgresCluster, instances, rootCA, true) + if err != nil || result != (reconcile.Result{}) { + t.Errorf("unable to reconcile pgBackRest: %v", err) + } - // repo is the first defined repo - repo := postgresCluster.Spec.Backups.PGBackRest.Repos[0] + // repo is the first defined repo + repo := postgresCluster.Spec.Backups.PGBackRest.Repos[0] - // test that the repo was created properly - t.Run("verify pgbackrest dedicated repo StatefulSet", func(t *testing.T) { + // test that the repo was created properly + t.Run("verify pgbackrest dedicated repo StatefulSet", func(t *testing.T) { - // get the pgBackRest repo sts using the labels we expect it to have - dedicatedRepos := &appsv1.StatefulSetList{} - if err := tClient.List(ctx, dedicatedRepos, client.InNamespace(ns.Name), - client.MatchingLabels{ - naming.LabelCluster: clusterName, - naming.LabelPGBackRest: "", - naming.LabelPGBackRestDedicated: "", - }); err != nil { - t.Fatal(err) - } + // get the pgBackRest repo sts using the labels we expect it to have + dedicatedRepos := &appsv1.StatefulSetList{} + if err := tClient.List(ctx, dedicatedRepos, client.InNamespace(ns.Name), + client.MatchingLabels{ + naming.LabelCluster: clusterName, + naming.LabelPGBackRest: "", + naming.LabelPGBackRestDedicated: "", + }); err != nil { + t.Fatal(err) + } - repo := appsv1.StatefulSet{} - // verify that we found a repo sts as expected - if len(dedicatedRepos.Items) == 0 { - t.Fatal("Did not find a dedicated repo sts") - } else if len(dedicatedRepos.Items) > 1 { - t.Fatal("Too many dedicated repo sts's found") - } else { - repo = dedicatedRepos.Items[0] - } + repo := appsv1.StatefulSet{} + // verify that we found a repo sts as expected + if len(dedicatedRepos.Items) == 0 { + t.Fatal("Did not find a dedicated repo sts") + } else if len(dedicatedRepos.Items) > 1 { + t.Fatal("Too many dedicated repo sts's found") + } else { + repo = dedicatedRepos.Items[0] + } - // verify proper number of replicas - if *repo.Spec.Replicas != 1 { - t.Errorf("%v replicas found for dedicated repo sts, expected %v", - repo.Spec.Replicas, 1) - } + // verify proper number of replicas + if *repo.Spec.Replicas != 1 { + t.Errorf("%v replicas found for dedicated repo sts, expected %v", + repo.Spec.Replicas, 1) + } - // verify proper ownership - var foundOwnershipRef bool - for _, r := range repo.GetOwnerReferences() { - if r.Kind == "PostgresCluster" && r.Name == clusterName && - r.UID == types.UID(clusterUID) { + // verify proper ownership + var foundOwnershipRef bool + for _, r := range repo.GetOwnerReferences() { + if r.Kind == "PostgresCluster" && r.Name == clusterName && + r.UID == types.UID(clusterUID) { - foundOwnershipRef = true - break + foundOwnershipRef = true + break + } } - } - if !foundOwnershipRef { - t.Errorf("did not find expected ownership references") - } + if !foundOwnershipRef { + t.Errorf("did not find expected ownership references") + } - // verify proper matching labels - expectedLabels := map[string]string{ - naming.LabelCluster: clusterName, - naming.LabelPGBackRest: "", - naming.LabelPGBackRestDedicated: "", - } - expectedLabelsSelector, err := metav1.LabelSelectorAsSelector( - metav1.SetAsLabelSelector(expectedLabels)) - if err != nil { - t.Error(err) - } - if !expectedLabelsSelector.Matches(labels.Set(repo.GetLabels())) { - t.Errorf("dedicated repo host is missing an expected label: found=%v, expected=%v", - repo.GetLabels(), expectedLabels) - } + // verify proper matching labels + expectedLabels := map[string]string{ + naming.LabelCluster: clusterName, + naming.LabelPGBackRest: "", + naming.LabelPGBackRestDedicated: "", + } + expectedLabelsSelector, err := metav1.LabelSelectorAsSelector( + metav1.SetAsLabelSelector(expectedLabels)) + if err != nil { + t.Error(err) + } + if !expectedLabelsSelector.Matches(labels.Set(repo.GetLabels())) { + t.Errorf("dedicated repo host is missing an expected label: found=%v, expected=%v", + repo.GetLabels(), expectedLabels) + } - template := repo.Spec.Template.DeepCopy() + template := repo.Spec.Template.DeepCopy() - // Containers and Volumes should be populated. - assert.Assert(t, len(template.Spec.Containers) != 0) - assert.Assert(t, len(template.Spec.InitContainers) != 0) - assert.Assert(t, len(template.Spec.Volumes) != 0) + // Containers and Volumes should be populated. + assert.Assert(t, len(template.Spec.Containers) != 0) + assert.Assert(t, len(template.Spec.InitContainers) != 0) + assert.Assert(t, len(template.Spec.Volumes) != 0) - // Ignore Containers and Volumes in the comparison below. - template.Spec.Containers = nil - template.Spec.InitContainers = nil - template.Spec.Volumes = nil + // Ignore Containers and Volumes in the comparison below. + template.Spec.Containers = nil + template.Spec.InitContainers = nil + template.Spec.Volumes = nil - // TODO(tjmoore4): Add additional tests to test appending existing - // topology spread constraints and spec.disableDefaultPodScheduling being - // set to true (as done in instance StatefulSet tests). - assert.Assert(t, marshalMatches(template.Spec, ` + // TODO(tjmoore4): Add additional tests to test appending existing + // topology spread constraints and spec.disableDefaultPodScheduling being + // set to true (as done in instance StatefulSet tests). + assert.Assert(t, marshalMatches(template.Spec, ` affinity: {} automountServiceAccountToken: false containers: null @@ -381,224 +381,298 @@ topologySpreadConstraints: maxSkew: 1 topologyKey: topology.kubernetes.io/zone whenUnsatisfiable: ScheduleAnyway - `)) + `)) - // verify that the repohost container exists and contains the proper env vars - var repoHostContExists bool - for _, c := range repo.Spec.Template.Spec.Containers { - if c.Name == naming.PGBackRestRepoContainerName { - repoHostContExists = true + // verify that the repohost container exists and contains the proper env vars + var repoHostContExists bool + for _, c := range repo.Spec.Template.Spec.Containers { + if c.Name == naming.PGBackRestRepoContainerName { + repoHostContExists = true + } + } + // now verify the proper env within the container + if !repoHostContExists { + t.Errorf("dedicated repo host is missing a container with name %s", + naming.PGBackRestRepoContainerName) } - } - // now verify the proper env within the container - if !repoHostContExists { - t.Errorf("dedicated repo host is missing a container with name %s", - naming.PGBackRestRepoContainerName) - } - repoHostStatus := postgresCluster.Status.PGBackRest.RepoHost - if repoHostStatus != nil { - if repoHostStatus.APIVersion != "apps/v1" || repoHostStatus.Kind != "StatefulSet" { - t.Errorf("invalid version/kind for dedicated repo host status") + repoHostStatus := postgresCluster.Status.PGBackRest.RepoHost + if repoHostStatus != nil { + if repoHostStatus.APIVersion != "apps/v1" || repoHostStatus.Kind != "StatefulSet" { + t.Errorf("invalid version/kind for dedicated repo host status") + } + } else { + t.Errorf("dedicated repo host status is missing") } - } else { - t.Errorf("dedicated repo host status is missing") - } - var foundConditionRepoHostsReady bool - for _, c := range postgresCluster.Status.Conditions { - if c.Type == "PGBackRestRepoHostReady" { - foundConditionRepoHostsReady = true - break + var foundConditionRepoHostsReady bool + for _, c := range postgresCluster.Status.Conditions { + if c.Type == "PGBackRestRepoHostReady" { + foundConditionRepoHostsReady = true + break + } + } + if !foundConditionRepoHostsReady { + t.Errorf("status condition PGBackRestRepoHostsReady is missing") } - } - if !foundConditionRepoHostsReady { - t.Errorf("status condition PGBackRestRepoHostsReady is missing") - } - assert.Check(t, wait.PollUntilContextTimeout(ctx, time.Second/2, Scale(time.Second*2), false, - func(ctx context.Context) (bool, error) { - events := &corev1.EventList{} - err := tClient.List(ctx, events, &client.MatchingFields{ - "involvedObject.kind": "PostgresCluster", - "involvedObject.name": clusterName, - "involvedObject.namespace": ns.Name, - "involvedObject.uid": clusterUID, - "reason": "RepoHostCreated", - }) - return len(events.Items) == 1, err - })) - }) + assert.Check(t, wait.PollUntilContextTimeout(ctx, time.Second/2, Scale(time.Second*2), false, + func(ctx context.Context) (bool, error) { + events := &corev1.EventList{} + err := tClient.List(ctx, events, &client.MatchingFields{ + "involvedObject.kind": "PostgresCluster", + "involvedObject.name": clusterName, + "involvedObject.namespace": ns.Name, + "involvedObject.uid": clusterUID, + "reason": "RepoHostCreated", + }) + return len(events.Items) == 1, err + })) + }) - t.Run("verify pgbackrest repo volumes", func(t *testing.T) { + t.Run("verify pgbackrest repo volumes", func(t *testing.T) { + + // get the pgBackRest repo sts using the labels we expect it to have + repoVols := &corev1.PersistentVolumeClaimList{} + if err := tClient.List(ctx, repoVols, client.InNamespace(ns.Name), + client.MatchingLabels{ + naming.LabelCluster: clusterName, + naming.LabelPGBackRest: "", + naming.LabelPGBackRestRepoVolume: "", + }); err != nil { + t.Fatal(err) + } + assert.Assert(t, len(repoVols.Items) > 0) - // get the pgBackRest repo sts using the labels we expect it to have - repoVols := &corev1.PersistentVolumeClaimList{} - if err := tClient.List(ctx, repoVols, client.InNamespace(ns.Name), - client.MatchingLabels{ - naming.LabelCluster: clusterName, - naming.LabelPGBackRest: "", - naming.LabelPGBackRestRepoVolume: "", - }); err != nil { - t.Fatal(err) - } - assert.Assert(t, len(repoVols.Items) > 0) + for _, r := range postgresCluster.Spec.Backups.PGBackRest.Repos { + if r.Volume == nil { + continue + } + var foundRepoVol bool + for _, v := range repoVols.Items { + if v.GetName() == + naming.PGBackRestRepoVolume(postgresCluster, r.Name).Name { + foundRepoVol = true + break + } + } + assert.Assert(t, foundRepoVol) + } + }) + + t.Run("verify pgbackrest configuration", func(t *testing.T) { - for _, r := range postgresCluster.Spec.Backups.PGBackRest.Repos { - if r.Volume == nil { - continue + config := &corev1.ConfigMap{} + if err := tClient.Get(ctx, types.NamespacedName{ + Name: naming.PGBackRestConfig(postgresCluster).Name, + Namespace: postgresCluster.GetNamespace(), + }, config); err != nil { + assert.NilError(t, err) } - var foundRepoVol bool - for _, v := range repoVols.Items { - if v.GetName() == - naming.PGBackRestRepoVolume(postgresCluster, r.Name).Name { - foundRepoVol = true - break + assert.Assert(t, len(config.Data) > 0) + + var instanceConfFound, dedicatedRepoConfFound bool + for k, v := range config.Data { + if v != "" { + if k == pgbackrest.CMInstanceKey { + instanceConfFound = true + } else if k == pgbackrest.CMRepoKey { + dedicatedRepoConfFound = true + } } } - assert.Assert(t, foundRepoVol) - } - }) + assert.Check(t, instanceConfFound) + assert.Check(t, dedicatedRepoConfFound) + }) - t.Run("verify pgbackrest configuration", func(t *testing.T) { + t.Run("verify pgbackrest schedule cronjob", func(t *testing.T) { - config := &corev1.ConfigMap{} - if err := tClient.Get(ctx, types.NamespacedName{ - Name: naming.PGBackRestConfig(postgresCluster).Name, - Namespace: postgresCluster.GetNamespace(), - }, config); err != nil { - assert.NilError(t, err) - } - assert.Assert(t, len(config.Data) > 0) - - var instanceConfFound, dedicatedRepoConfFound bool - for k, v := range config.Data { - if v != "" { - if k == pgbackrest.CMInstanceKey { - instanceConfFound = true - } else if k == pgbackrest.CMRepoKey { - dedicatedRepoConfFound = true - } + // set status + postgresCluster.Status = v1beta1.PostgresClusterStatus{ + Patroni: v1beta1.PatroniStatus{SystemIdentifier: "12345abcde"}, + PGBackRest: &v1beta1.PGBackRestStatus{ + Repos: []v1beta1.RepoStatus{{Name: "repo1", StanzaCreated: true}}}, } - } - assert.Check(t, instanceConfFound) - assert.Check(t, dedicatedRepoConfFound) - }) - t.Run("verify pgbackrest schedule cronjob", func(t *testing.T) { + // set conditions + clusterConditions := map[string]metav1.ConditionStatus{ + ConditionRepoHostReady: metav1.ConditionTrue, + ConditionReplicaCreate: metav1.ConditionTrue, + } - // set status - postgresCluster.Status = v1beta1.PostgresClusterStatus{ - Patroni: v1beta1.PatroniStatus{SystemIdentifier: "12345abcde"}, - PGBackRest: &v1beta1.PGBackRestStatus{ - Repos: []v1beta1.RepoStatus{{Name: "repo1", StanzaCreated: true}}}, - } + for condition, status := range clusterConditions { + meta.SetStatusCondition(&postgresCluster.Status.Conditions, metav1.Condition{ + Type: condition, Reason: "testing", Status: status}) + } - // set conditions - clusterConditions := map[string]metav1.ConditionStatus{ - ConditionRepoHostReady: metav1.ConditionTrue, - ConditionReplicaCreate: metav1.ConditionTrue, - } + requeue := r.reconcileScheduledBackups(ctx, postgresCluster, serviceAccount, fakeObservedCronJobs()) + assert.Assert(t, !requeue) - for condition, status := range clusterConditions { - meta.SetStatusCondition(&postgresCluster.Status.Conditions, metav1.Condition{ - Type: condition, Reason: "testing", Status: status}) - } + returnedCronJob := &batchv1.CronJob{} + if err := tClient.Get(ctx, types.NamespacedName{ + Name: postgresCluster.Name + "-repo1-full", + Namespace: postgresCluster.GetNamespace(), + }, returnedCronJob); err != nil { + assert.NilError(t, err) + } - requeue := r.reconcileScheduledBackups(ctx, postgresCluster, serviceAccount, fakeObservedCronJobs()) - assert.Assert(t, !requeue) + // check returned cronjob matches set spec + assert.Equal(t, returnedCronJob.Name, "hippocluster-repo1-full") + assert.Equal(t, returnedCronJob.Spec.Schedule, testCronSchedule) + assert.Equal(t, returnedCronJob.Spec.ConcurrencyPolicy, batchv1.ForbidConcurrent) + assert.Equal(t, returnedCronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Name, + "pgbackrest") + assert.Assert(t, returnedCronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].SecurityContext != &corev1.SecurityContext{}) - returnedCronJob := &batchv1.CronJob{} - if err := tClient.Get(ctx, types.NamespacedName{ - Name: postgresCluster.Name + "-repo1-full", - Namespace: postgresCluster.GetNamespace(), - }, returnedCronJob); err != nil { - assert.NilError(t, err) - } + }) - // check returned cronjob matches set spec - assert.Equal(t, returnedCronJob.Name, "hippocluster-repo1-full") - assert.Equal(t, returnedCronJob.Spec.Schedule, testCronSchedule) - assert.Equal(t, returnedCronJob.Spec.ConcurrencyPolicy, batchv1.ForbidConcurrent) - assert.Equal(t, returnedCronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Name, - "pgbackrest") - assert.Assert(t, returnedCronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].SecurityContext != &corev1.SecurityContext{}) + t.Run("verify pgbackrest schedule found", func(t *testing.T) { - }) + assert.Assert(t, backupScheduleFound(repo, "full")) - t.Run("verify pgbackrest schedule found", func(t *testing.T) { + testrepo := v1beta1.PGBackRestRepo{ + Name: "repo1", + BackupSchedules: &v1beta1.PGBackRestBackupSchedules{ + Full: &testCronSchedule, + Differential: &testCronSchedule, + Incremental: &testCronSchedule, + }} - assert.Assert(t, backupScheduleFound(repo, "full")) + assert.Assert(t, backupScheduleFound(testrepo, "full")) + assert.Assert(t, backupScheduleFound(testrepo, "diff")) + assert.Assert(t, backupScheduleFound(testrepo, "incr")) - testrepo := v1beta1.PGBackRestRepo{ - Name: "repo1", - BackupSchedules: &v1beta1.PGBackRestBackupSchedules{ - Full: &testCronSchedule, - Differential: &testCronSchedule, - Incremental: &testCronSchedule, - }} + }) - assert.Assert(t, backupScheduleFound(testrepo, "full")) - assert.Assert(t, backupScheduleFound(testrepo, "diff")) - assert.Assert(t, backupScheduleFound(testrepo, "incr")) + t.Run("verify pgbackrest schedule not found", func(t *testing.T) { - }) + assert.Assert(t, !backupScheduleFound(repo, "notabackuptype")) + + noscheduletestrepo := v1beta1.PGBackRestRepo{Name: "repo1"} + assert.Assert(t, !backupScheduleFound(noscheduletestrepo, "full")) + + }) + + t.Run("pgbackrest schedule suspended status", func(t *testing.T) { + + returnedCronJob := &batchv1.CronJob{} + if err := tClient.Get(ctx, types.NamespacedName{ + Name: postgresCluster.Name + "-repo1-full", + Namespace: postgresCluster.GetNamespace(), + }, returnedCronJob); err != nil { + assert.NilError(t, err) + } + + t.Run("pgbackrest schedule suspended false", func(t *testing.T) { + assert.Assert(t, !*returnedCronJob.Spec.Suspend) + }) + + t.Run("shutdown", func(t *testing.T) { + *postgresCluster.Spec.Shutdown = true + postgresCluster.Spec.Standby = nil + + requeue := r.reconcileScheduledBackups(ctx, + postgresCluster, serviceAccount, fakeObservedCronJobs()) + assert.Assert(t, !requeue) + + assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ + Name: postgresCluster.Name + "-repo1-full", + Namespace: postgresCluster.GetNamespace(), + }, returnedCronJob)) + + assert.Assert(t, *returnedCronJob.Spec.Suspend) + }) - t.Run("verify pgbackrest schedule not found", func(t *testing.T) { + t.Run("standby", func(t *testing.T) { + *postgresCluster.Spec.Shutdown = false + postgresCluster.Spec.Standby = &v1beta1.PostgresStandbySpec{ + Enabled: true, + } - assert.Assert(t, !backupScheduleFound(repo, "notabackuptype")) + requeue := r.reconcileScheduledBackups(ctx, + postgresCluster, serviceAccount, fakeObservedCronJobs()) + assert.Assert(t, !requeue) - noscheduletestrepo := v1beta1.PGBackRestRepo{Name: "repo1"} - assert.Assert(t, !backupScheduleFound(noscheduletestrepo, "full")) + assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ + Name: postgresCluster.Name + "-repo1-full", + Namespace: postgresCluster.GetNamespace(), + }, returnedCronJob)) + assert.Assert(t, *returnedCronJob.Spec.Suspend) + }) + }) }) - t.Run("pgbackrest schedule suspended status", func(t *testing.T) { + t.Run("run reconcile with backups not defined", func(t *testing.T) { + clusterName := "hippocluster2" + clusterUID := "hippouid2" + + ns := setupNamespace(t, tClient) + // create a PostgresCluster without backups to test with + postgresCluster := fakePostgresCluster(clusterName, ns.GetName(), clusterUID, true) + postgresCluster.Spec.Backups = v1beta1.Backups{} - returnedCronJob := &batchv1.CronJob{} - if err := tClient.Get(ctx, types.NamespacedName{ - Name: postgresCluster.Name + "-repo1-full", - Namespace: postgresCluster.GetNamespace(), - }, returnedCronJob); err != nil { - assert.NilError(t, err) + // create the 'observed' instances and set the leader + instances := &observedInstances{ + forCluster: []*Instance{{Name: "instance1", + Pods: []*corev1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{naming.LabelRole: naming.RolePatroniLeader}, + }, + Spec: corev1.PodSpec{}, + }}, + }, {Name: "instance2"}, {Name: "instance3"}}, } - t.Run("pgbackrest schedule suspended false", func(t *testing.T) { - assert.Assert(t, !*returnedCronJob.Spec.Suspend) - }) + rootCA, err := pki.NewRootCertificateAuthority() + assert.NilError(t, err) - t.Run("shutdown", func(t *testing.T) { - *postgresCluster.Spec.Shutdown = true - postgresCluster.Spec.Standby = nil + result, err := r.reconcilePGBackRest(ctx, postgresCluster, instances, rootCA, false) + if err != nil { + t.Errorf("unable to reconcile pgBackRest: %v", err) + } + assert.Equal(t, result, reconcile.Result{}) - requeue := r.reconcileScheduledBackups(ctx, - postgresCluster, serviceAccount, fakeObservedCronJobs()) - assert.Assert(t, !requeue) + t.Run("verify pgbackrest dedicated repo StatefulSet", func(t *testing.T) { - assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ - Name: postgresCluster.Name + "-repo1-full", - Namespace: postgresCluster.GetNamespace(), - }, returnedCronJob)) + // Verify the sts doesn't exist + dedicatedRepos := &appsv1.StatefulSetList{} + if err := tClient.List(ctx, dedicatedRepos, client.InNamespace(ns.Name), + client.MatchingLabels{ + naming.LabelCluster: clusterName, + naming.LabelPGBackRest: "", + naming.LabelPGBackRestDedicated: "", + }); err != nil { + t.Fatal(err) + } - assert.Assert(t, *returnedCronJob.Spec.Suspend) + assert.Equal(t, len(dedicatedRepos.Items), 0) }) - t.Run("standby", func(t *testing.T) { - *postgresCluster.Spec.Shutdown = false - postgresCluster.Spec.Standby = &v1beta1.PostgresStandbySpec{ - Enabled: true, + t.Run("verify pgbackrest repo volumes", func(t *testing.T) { + + // get the pgBackRest repo sts using the labels we expect it to have + repoVols := &corev1.PersistentVolumeClaimList{} + if err := tClient.List(ctx, repoVols, client.InNamespace(ns.Name), + client.MatchingLabels{ + naming.LabelCluster: clusterName, + naming.LabelPGBackRest: "", + naming.LabelPGBackRestRepoVolume: "", + }); err != nil { + t.Fatal(err) } - requeue := r.reconcileScheduledBackups(ctx, - postgresCluster, serviceAccount, fakeObservedCronJobs()) - assert.Assert(t, !requeue) + assert.Equal(t, len(repoVols.Items), 0) + }) - assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ - Name: postgresCluster.Name + "-repo1-full", - Namespace: postgresCluster.GetNamespace(), - }, returnedCronJob)) + t.Run("verify pgbackrest configuration", func(t *testing.T) { - assert.Assert(t, *returnedCronJob.Spec.Suspend) + config := &corev1.ConfigMap{} + err := tClient.Get(ctx, types.NamespacedName{ + Name: naming.PGBackRestConfig(postgresCluster).Name, + Namespace: postgresCluster.GetNamespace(), + }, config) + assert.Equal(t, apierrors.IsNotFound(err), true) }) }) } @@ -1641,7 +1715,7 @@ func TestGetPGBackRestResources(t *testing.T) { assert.NilError(t, err) assert.NilError(t, tClient.Create(ctx, resource)) - resources, err := r.getPGBackRestResources(ctx, tc.cluster) + resources, err := r.getPGBackRestResources(ctx, tc.cluster, true) assert.NilError(t, err) assert.Assert(t, tc.result.jobCount == len(resources.replicaCreateBackupJobs)) @@ -1878,7 +1952,7 @@ func TestReconcilePostgresClusterDataSource(t *testing.T) { pgclusterDataSource = tc.dataSource.PostgresCluster } err := r.reconcilePostgresClusterDataSource(ctx, cluster, pgclusterDataSource, - "testhash", nil, rootCA) + "testhash", nil, rootCA, true) assert.NilError(t, err) restoreConfig := &corev1.ConfigMap{} @@ -3671,3 +3745,167 @@ func TestSetScheduledJobStatus(t *testing.T) { assert.Assert(t, len(postgresCluster.Status.PGBackRest.ScheduledBackups) == 0) }) } + +func TestBackupsEnabled(t *testing.T) { + // Garbage collector cleans up test resources before the test completes + if strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") { + t.Skip("USE_EXISTING_CLUSTER: Test fails due to garbage collection") + } + + cfg, tClient := setupKubernetes(t) + require.ParallelCapacity(t, 2) + + r := &Reconciler{} + ctx, cancel := setupManager(t, cfg, func(mgr manager.Manager) { + r = &Reconciler{ + Client: mgr.GetClient(), + Recorder: mgr.GetEventRecorderFor(ControllerName), + Tracer: otel.Tracer(ControllerName), + Owner: ControllerName, + } + }) + t.Cleanup(func() { teardownManager(cancel, t) }) + + t.Run("Cluster with backups, no sts can be reconciled", func(t *testing.T) { + clusterName := "hippocluster1" + clusterUID := "hippouid1" + + ns := setupNamespace(t, tClient) + + // create a PostgresCluster to test with + postgresCluster := fakePostgresCluster(clusterName, ns.GetName(), clusterUID, true) + + backupsSpecFound, backupsReconciliationAllowed, err := r.BackupsEnabled(ctx, postgresCluster) + + assert.NilError(t, err) + assert.Assert(t, backupsSpecFound) + assert.Assert(t, backupsReconciliationAllowed) + }) + + t.Run("Cluster with backups, sts can be reconciled", func(t *testing.T) { + clusterName := "hippocluster2" + clusterUID := "hippouid2" + + ns := setupNamespace(t, tClient) + + // create a PostgresCluster to test with + postgresCluster := fakePostgresCluster(clusterName, ns.GetName(), clusterUID, true) + + // create the 'observed' instances and set the leader + instances := &observedInstances{ + forCluster: []*Instance{{Name: "instance1", + Pods: []*corev1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{naming.LabelRole: naming.RolePatroniLeader}, + }, + Spec: corev1.PodSpec{}, + }}, + }, {Name: "instance2"}, {Name: "instance3"}}, + } + + rootCA, err := pki.NewRootCertificateAuthority() + assert.NilError(t, err) + + _, err = r.reconcilePGBackRest(ctx, postgresCluster, instances, rootCA, true) + assert.NilError(t, err) + + backupsSpecFound, backupsReconciliationAllowed, err := r.BackupsEnabled(ctx, postgresCluster) + + assert.NilError(t, err) + assert.Assert(t, backupsSpecFound) + assert.Assert(t, backupsReconciliationAllowed) + }) + + t.Run("Cluster with no backups, no sts can reconcile", func(t *testing.T) { + // create a PostgresCluster to test with + clusterName := "hippocluster3" + clusterUID := "hippouid3" + + ns := setupNamespace(t, tClient) + + postgresCluster := fakePostgresCluster(clusterName, ns.GetName(), clusterUID, true) + postgresCluster.Spec.Backups = v1beta1.Backups{} + + backupsSpecFound, backupsReconciliationAllowed, err := r.BackupsEnabled(ctx, postgresCluster) + + assert.NilError(t, err) + assert.Assert(t, !backupsSpecFound) + assert.Assert(t, backupsReconciliationAllowed) + }) + + t.Run("Cluster with no backups, sts cannot be reconciled", func(t *testing.T) { + clusterName := "hippocluster4" + clusterUID := "hippouid4" + + ns := setupNamespace(t, tClient) + + // create a PostgresCluster to test with + postgresCluster := fakePostgresCluster(clusterName, ns.GetName(), clusterUID, true) + + // create the 'observed' instances and set the leader + instances := &observedInstances{ + forCluster: []*Instance{{Name: "instance1", + Pods: []*corev1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{naming.LabelRole: naming.RolePatroniLeader}, + }, + Spec: corev1.PodSpec{}, + }}, + }, {Name: "instance2"}, {Name: "instance3"}}, + } + + rootCA, err := pki.NewRootCertificateAuthority() + assert.NilError(t, err) + + _, err = r.reconcilePGBackRest(ctx, postgresCluster, instances, rootCA, true) + assert.NilError(t, err) + + postgresCluster.Spec.Backups = v1beta1.Backups{} + + backupsSpecFound, backupsReconciliationAllowed, err := r.BackupsEnabled(ctx, postgresCluster) + + assert.NilError(t, err) + assert.Assert(t, !backupsSpecFound) + assert.Assert(t, !backupsReconciliationAllowed) + }) + + t.Run("Cluster with no backups, sts, annotation can be reconciled", func(t *testing.T) { + clusterName := "hippocluster5" + clusterUID := "hippouid5" + + ns := setupNamespace(t, tClient) + + // create a PostgresCluster to test with + postgresCluster := fakePostgresCluster(clusterName, ns.GetName(), clusterUID, true) + + // create the 'observed' instances and set the leader + instances := &observedInstances{ + forCluster: []*Instance{{Name: "instance1", + Pods: []*corev1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{naming.LabelRole: naming.RolePatroniLeader}, + }, + Spec: corev1.PodSpec{}, + }}, + }, {Name: "instance2"}, {Name: "instance3"}}, + } + + rootCA, err := pki.NewRootCertificateAuthority() + assert.NilError(t, err) + + _, err = r.reconcilePGBackRest(ctx, postgresCluster, instances, rootCA, true) + assert.NilError(t, err) + + postgresCluster.Spec.Backups = v1beta1.Backups{} + annotations := map[string]string{ + naming.AuthorizeBackupRemovalAnnotation: "true", + } + postgresCluster.Annotations = annotations + + backupsSpecFound, backupsReconciliationAllowed, err := r.BackupsEnabled(ctx, postgresCluster) + + assert.NilError(t, err) + assert.Assert(t, !backupsSpecFound) + assert.Assert(t, backupsReconciliationAllowed) + }) +} diff --git a/internal/naming/annotations.go b/internal/naming/annotations.go index 5f86d45aa7..21e8bd084b 100644 --- a/internal/naming/annotations.go +++ b/internal/naming/annotations.go @@ -72,4 +72,10 @@ const ( // has schemas automatically created for the users defined in `spec.users` for all of the databases // listed for that user. AutoCreateUserSchemaAnnotation = annotationPrefix + "autoCreateUserSchema" + + // AuthorizeBackupRemovalAnnotation is an annotation used to allow users + // to delete PVC-based backups when changing from a cluster with backups + // to a cluster without backups. As usual with the operator, we do not + // touch cloud-based backups. + AuthorizeBackupRemovalAnnotation = annotationPrefix + "authorizeBackupRemoval" ) diff --git a/internal/pgbackrest/postgres.go b/internal/pgbackrest/postgres.go index 4636ee9db5..566630657b 100644 --- a/internal/pgbackrest/postgres.go +++ b/internal/pgbackrest/postgres.go @@ -26,6 +26,7 @@ import ( func PostgreSQL( inCluster *v1beta1.PostgresCluster, outParameters *postgres.Parameters, + backupsEnabled bool, ) { if outParameters.Mandatory == nil { outParameters.Mandatory = postgres.NewParameterSet() @@ -38,9 +39,15 @@ func PostgreSQL( // - https://pgbackrest.org/user-guide.html#quickstart/configure-archiving // - https://pgbackrest.org/command.html#command-archive-push // - https://www.postgresql.org/docs/current/runtime-config-wal.html - archive := `pgbackrest --stanza=` + DefaultStanzaName + ` archive-push "%p"` outParameters.Mandatory.Add("archive_mode", "on") - outParameters.Mandatory.Add("archive_command", archive) + if backupsEnabled { + archive := `pgbackrest --stanza=` + DefaultStanzaName + ` archive-push "%p"` + outParameters.Mandatory.Add("archive_command", archive) + } else { + // If backups are disabled, keep archive_mode on (to avoid a Postgres restart) + // and throw away WAL. + outParameters.Mandatory.Add("archive_command", `true`) + } // archive_timeout is used to determine at what point a WAL file is switched, // if the WAL archive has not reached its full size in # of transactions diff --git a/internal/pgbackrest/postgres_test.go b/internal/pgbackrest/postgres_test.go index da41b86281..559388e926 100644 --- a/internal/pgbackrest/postgres_test.go +++ b/internal/pgbackrest/postgres_test.go @@ -28,7 +28,7 @@ func TestPostgreSQLParameters(t *testing.T) { cluster := new(v1beta1.PostgresCluster) parameters := new(postgres.Parameters) - PostgreSQL(cluster, parameters) + PostgreSQL(cluster, parameters, true) assert.DeepEqual(t, parameters.Mandatory.AsMap(), map[string]string{ "archive_mode": "on", "archive_command": `pgbackrest --stanza=db archive-push "%p"`, @@ -39,12 +39,19 @@ func TestPostgreSQLParameters(t *testing.T) { "archive_timeout": "60s", }) + PostgreSQL(cluster, parameters, false) + assert.DeepEqual(t, parameters.Mandatory.AsMap(), map[string]string{ + "archive_mode": "on", + "archive_command": "true", + "restore_command": `pgbackrest --stanza=db archive-get %f "%p"`, + }) + cluster.Spec.Standby = &v1beta1.PostgresStandbySpec{ Enabled: true, RepoName: "repo99", } - PostgreSQL(cluster, parameters) + PostgreSQL(cluster, parameters, true) assert.DeepEqual(t, parameters.Mandatory.AsMap(), map[string]string{ "archive_mode": "on", "archive_command": `pgbackrest --stanza=db archive-push "%p"`, diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go index 0a066c076f..0e50f3f0f7 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go @@ -33,8 +33,8 @@ type PostgresClusterSpec struct { DataSource *DataSource `json:"dataSource,omitempty"` // PostgreSQL backup configuration - // +kubebuilder:validation:Required - Backups Backups `json:"backups"` + // +optional + Backups Backups `json:"backups,omitempty"` // The secret containing the Certificates and Keys to encrypt PostgreSQL // traffic will need to contain the server TLS certificate, TLS key and the @@ -322,7 +322,7 @@ func (s *PostgresClusterSpec) Default() { type Backups struct { // pgBackRest archive configuration - // +kubebuilder:validation:Required + // +optional PGBackRest PGBackRestArchive `json:"pgbackrest"` // VolumeSnapshot configuration diff --git a/testing/kuttl/e2e/optional-backups/00--cluster.yaml b/testing/kuttl/e2e/optional-backups/00--cluster.yaml new file mode 100644 index 0000000000..7b927831e0 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/00--cluster.yaml @@ -0,0 +1,15 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: created-without-backups +spec: + postgresVersion: ${KUTTL_PG_VERSION} + instances: + - name: instance1 + dataVolumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + diff --git a/testing/kuttl/e2e/optional-backups/00-assert.yaml b/testing/kuttl/e2e/optional-backups/00-assert.yaml new file mode 100644 index 0000000000..86392d0308 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/00-assert.yaml @@ -0,0 +1,38 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: created-without-backups +status: + instances: + - name: instance1 + pgbackrest: {} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + postgres-operator.crunchydata.com/role: pgdata +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + postgres-operator.crunchydata.com/role: master +status: + containerStatuses: + - ready: true + - ready: true diff --git a/testing/kuttl/e2e/optional-backups/01-errors.yaml b/testing/kuttl/e2e/optional-backups/01-errors.yaml new file mode 100644 index 0000000000..e702fcddb4 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/01-errors.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: created-without-backups-repo1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: created-without-backups-repo-host +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: created-without-backups-pgbackrest-config +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: created-without-backups-pgbackrest +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: created-without-backups-pgbackrest +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: created-without-backups-pgbackrest diff --git a/testing/kuttl/e2e/optional-backups/02-assert.yaml b/testing/kuttl/e2e/optional-backups/02-assert.yaml new file mode 100644 index 0000000000..eb3f70357f --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/02-assert.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + pod=$(kubectl get pods -o name -n "${NAMESPACE}" \ + -l postgres-operator.crunchydata.com/cluster=created-without-backups) + + kubectl exec --stdin "${pod}" --namespace "${NAMESPACE}" -c database \ + -- psql -qb --set ON_ERROR_STOP=1 --file=- <<'SQL' + DO $$ + BEGIN + ASSERT current_setting('archive_command') LIKE 'true', + format('expected "true", got %L', current_setting('archive_command')); + END $$ + SQL diff --git a/testing/kuttl/e2e/optional-backups/03-assert.yaml b/testing/kuttl/e2e/optional-backups/03-assert.yaml new file mode 100644 index 0000000000..17ca1e4062 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/03-assert.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + pod=$(kubectl get pods -o name -n "${NAMESPACE}" \ + -l postgres-operator.crunchydata.com/cluster=created-without-backups) + + kubectl exec --stdin "${pod}" --namespace "${NAMESPACE}" -c database \ + -- psql -qb --set ON_ERROR_STOP=1 \ + -c "CREATE TABLE important (data) AS VALUES ('treasure');" + + kubectl exec --stdin "${pod}" --namespace "${NAMESPACE}" -c database \ + -- psql -qb --set ON_ERROR_STOP=1 \ + -c "CHECKPOINT;" diff --git a/testing/kuttl/e2e/optional-backups/04--cluster.yaml b/testing/kuttl/e2e/optional-backups/04--cluster.yaml new file mode 100644 index 0000000000..fc39ff6ebe --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/04--cluster.yaml @@ -0,0 +1,16 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: created-without-backups +spec: + postgresVersion: ${KUTTL_PG_VERSION} + instances: + - name: instance1 + replicas: 2 + dataVolumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + diff --git a/testing/kuttl/e2e/optional-backups/05-assert.yaml b/testing/kuttl/e2e/optional-backups/05-assert.yaml new file mode 100644 index 0000000000..d346e01a04 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/05-assert.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + postgres-operator.crunchydata.com/role: replica +status: + containerStatuses: + - ready: true + - ready: true diff --git a/testing/kuttl/e2e/optional-backups/06-assert.yaml b/testing/kuttl/e2e/optional-backups/06-assert.yaml new file mode 100644 index 0000000000..c366545508 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/06-assert.yaml @@ -0,0 +1,18 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + pod=$(kubectl get pods -o name -n "${NAMESPACE}" \ + -l postgres-operator.crunchydata.com/cluster=created-without-backups \ + -l postgres-operator.crunchydata.com/role=replica) + + kubectl exec --stdin "${pod}" --namespace "${NAMESPACE}" -c database \ + -- psql -qb --set ON_ERROR_STOP=1 --file=- <<'SQL' + DO $$ + DECLARE + everything jsonb; + BEGIN + SELECT jsonb_agg(important) INTO everything FROM important; + ASSERT everything = '[{"data":"treasure"}]', format('got %L', everything); + END $$ + SQL diff --git a/testing/kuttl/e2e/optional-backups/10--cluster.yaml b/testing/kuttl/e2e/optional-backups/10--cluster.yaml new file mode 100644 index 0000000000..6da85c93f9 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/10--cluster.yaml @@ -0,0 +1,27 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: created-without-backups +spec: + postgresVersion: ${KUTTL_PG_VERSION} + instances: + - name: instance1 + replicas: 1 + dataVolumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + backups: + pgbackrest: + repos: + - name: repo1 + volume: + volumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + diff --git a/testing/kuttl/e2e/optional-backups/10-assert.yaml b/testing/kuttl/e2e/optional-backups/10-assert.yaml new file mode 100644 index 0000000000..7b740b310d --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/10-assert.yaml @@ -0,0 +1,79 @@ +# It should be possible to turn backups back on. +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: created-without-backups +status: + pgbackrest: + repoHost: + apiVersion: apps/v1 + kind: StatefulSet + ready: true + repos: + - bound: true + name: repo1 + replicaCreateBackupComplete: true + stanzaCreated: true +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + postgres-operator.crunchydata.com/role: pgdata +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: created-without-backups-repo1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: created-without-backups-repo-host +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: created-without-backups-pgbackrest-config +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: created-without-backups-pgbackrest +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: created-without-backups-pgbackrest +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: created-without-backups-pgbackrest +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + postgres-operator.crunchydata.com/patroni: created-without-backups-ha + postgres-operator.crunchydata.com/role: master +status: + containerStatuses: + - ready: true + - ready: true + - ready: true + - ready: true diff --git a/testing/kuttl/e2e/optional-backups/11-assert.yaml b/testing/kuttl/e2e/optional-backups/11-assert.yaml new file mode 100644 index 0000000000..5976d03f41 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/11-assert.yaml @@ -0,0 +1,18 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + pod=$(kubectl get pods -o name -n "${NAMESPACE}" \ + -l postgres-operator.crunchydata.com/cluster=created-without-backup \ + -l postgres-operator.crunchydata.com/instance-set=instance1 \ + -l postgres-operator.crunchydata.com/patroni=created-without-backups-ha \ + -l postgres-operator.crunchydata.com/role=master) + + kubectl exec --stdin "${pod}" --namespace "${NAMESPACE}" -c database \ + -- psql -qb --set ON_ERROR_STOP=1 --file=- <<'SQL' + DO $$ + BEGIN + ASSERT current_setting('archive_command') LIKE 'pgbackrest --stanza=db archive-push "%p"', + format('expected "pgbackrest --stanza=db archive-push \"%p\"", got %L', current_setting('archive_command')); + END $$ + SQL diff --git a/testing/kuttl/e2e/optional-backups/20--cluster.yaml b/testing/kuttl/e2e/optional-backups/20--cluster.yaml new file mode 100644 index 0000000000..8e0d01cbf8 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/20--cluster.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- command: |- + kubectl patch postgrescluster created-without-backups --type 'merge' -p '{"spec":{"backups": null}}' + namespaced: true diff --git a/testing/kuttl/e2e/optional-backups/20-assert.yaml b/testing/kuttl/e2e/optional-backups/20-assert.yaml new file mode 100644 index 0000000000..b469e277f8 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/20-assert.yaml @@ -0,0 +1,63 @@ +# It should be possible to turn backups back on. +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: created-without-backups +status: + pgbackrest: + repoHost: + apiVersion: apps/v1 + kind: StatefulSet + ready: true + repos: + - bound: true + name: repo1 + replicaCreateBackupComplete: true + stanzaCreated: true +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + postgres-operator.crunchydata.com/role: pgdata +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: created-without-backups-repo1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: created-without-backups-repo-host +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: created-without-backups-pgbackrest-config +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: created-without-backups-pgbackrest +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: created-without-backups-pgbackrest +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: created-without-backups-pgbackrest diff --git a/testing/kuttl/e2e/optional-backups/21-assert.yaml b/testing/kuttl/e2e/optional-backups/21-assert.yaml new file mode 100644 index 0000000000..5976d03f41 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/21-assert.yaml @@ -0,0 +1,18 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + pod=$(kubectl get pods -o name -n "${NAMESPACE}" \ + -l postgres-operator.crunchydata.com/cluster=created-without-backup \ + -l postgres-operator.crunchydata.com/instance-set=instance1 \ + -l postgres-operator.crunchydata.com/patroni=created-without-backups-ha \ + -l postgres-operator.crunchydata.com/role=master) + + kubectl exec --stdin "${pod}" --namespace "${NAMESPACE}" -c database \ + -- psql -qb --set ON_ERROR_STOP=1 --file=- <<'SQL' + DO $$ + BEGIN + ASSERT current_setting('archive_command') LIKE 'pgbackrest --stanza=db archive-push "%p"', + format('expected "pgbackrest --stanza=db archive-push \"%p\"", got %L', current_setting('archive_command')); + END $$ + SQL diff --git a/testing/kuttl/e2e/optional-backups/22--cluster.yaml b/testing/kuttl/e2e/optional-backups/22--cluster.yaml new file mode 100644 index 0000000000..2e25309886 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/22--cluster.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- command: kubectl annotate postgrescluster created-without-backups postgres-operator.crunchydata.com/authorizeBackupRemoval="true" + namespaced: true diff --git a/testing/kuttl/e2e/optional-backups/23-assert.yaml b/testing/kuttl/e2e/optional-backups/23-assert.yaml new file mode 100644 index 0000000000..8748ea015c --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/23-assert.yaml @@ -0,0 +1,26 @@ +# It should be possible to turn backups back on. +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: created-without-backups +status: + instances: + - name: instance1 + pgbackrest: {} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + postgres-operator.crunchydata.com/role: pgdata +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + postgres-operator.crunchydata.com/cluster: created-without-backups + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 diff --git a/testing/kuttl/e2e/optional-backups/24-errors.yaml b/testing/kuttl/e2e/optional-backups/24-errors.yaml new file mode 100644 index 0000000000..e702fcddb4 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/24-errors.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: created-without-backups-repo1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: created-without-backups-repo-host +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: created-without-backups-pgbackrest-config +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: created-without-backups-pgbackrest +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: created-without-backups-pgbackrest +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: created-without-backups-pgbackrest diff --git a/testing/kuttl/e2e/optional-backups/25-assert.yaml b/testing/kuttl/e2e/optional-backups/25-assert.yaml new file mode 100644 index 0000000000..eb3f70357f --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/25-assert.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + pod=$(kubectl get pods -o name -n "${NAMESPACE}" \ + -l postgres-operator.crunchydata.com/cluster=created-without-backups) + + kubectl exec --stdin "${pod}" --namespace "${NAMESPACE}" -c database \ + -- psql -qb --set ON_ERROR_STOP=1 --file=- <<'SQL' + DO $$ + BEGIN + ASSERT current_setting('archive_command') LIKE 'true', + format('expected "true", got %L', current_setting('archive_command')); + END $$ + SQL diff --git a/testing/kuttl/e2e/optional-backups/README.md b/testing/kuttl/e2e/optional-backups/README.md new file mode 100644 index 0000000000..92c52d4136 --- /dev/null +++ b/testing/kuttl/e2e/optional-backups/README.md @@ -0,0 +1,13 @@ +## Optional backups + +### Steps + +00-02. Create cluster without backups, check that expected K8s objects do/don't exist, e.g., repo-host sts doesn't exist; check that the archive command is `true` + +03-06. Add data and a replica; check that the data successfully replicates to the replica. + +10-11. Update cluster to add backups, check that expected K8s objects do/don't exist, e.g., repo-host sts exists; check that the archive command is set to the usual + +20-21. Update cluster to remove backups but without annotation, check that no changes were made, including to the archive command + +22-25. Annotate cluster to remove existing backups, check that expected K8s objects do/don't exist, e.g., repo-host sts doesn't exist; check that the archive command is `true`