Add functionality for operator to take a volume snapshot of the pgdata volume after a every backup. Manage snapshots so that only one ReadyToUse snapshot is kept. Add/adjust tests for volume snapshots feature. Add volume snapshot crds directory to envtest environment setup.

Author: dsessler7

Makefile

@@ -9,6 +9,9 @@ PGMONITOR_DIR ?= hack/tools/pgmonitor
 PGMONITOR_VERSION ?= v4.11.0
 QUERIES_CONFIG_DIR ?= hack/tools/queries
 
+EXTERNAL_SNAPSHOTTER_DIR ?= hack/tools/external-snapshotter
+EXTERNAL_SNAPSHOTTER_VERSION ?= v8.0.1
+
 # Buildah's "build" used to be "bud". Use the alias to be compatible for a while.
 BUILDAH_BUILD ?= buildah bud
 
@@ -52,6 +55,12 @@ get-pgmonitor:
 	cp -r '$(PGMONITOR_DIR)/postgres_exporter/common/.' '${QUERIES_CONFIG_DIR}'
 	cp '$(PGMONITOR_DIR)/postgres_exporter/linux/queries_backrest.yml' '${QUERIES_CONFIG_DIR}'
 
+.PHONY: get-external-snapshotter
+get-external-snapshotter:
+	git -C '$(dir $(EXTERNAL_SNAPSHOTTER_DIR))' clone https://github.com/kubernetes-csi/external-snapshotter.git || git -C '$(EXTERNAL_SNAPSHOTTER_DIR)' fetch origin
+	@git -C '$(EXTERNAL_SNAPSHOTTER_DIR)' checkout '$(EXTERNAL_SNAPSHOTTER_VERSION)'
+	@git -C '$(EXTERNAL_SNAPSHOTTER_DIR)' config pull.ff only
+
 .PHONY: clean
 clean: ## Clean resources
 clean: clean-deprecated
@@ -64,6 +73,7 @@ clean: clean-deprecated
 	[ ! -f hack/tools/setup-envtest ] || rm hack/tools/setup-envtest
 	[ ! -d hack/tools/envtest ] || { chmod -R u+w hack/tools/envtest && rm -r hack/tools/envtest; }
 	[ ! -d hack/tools/pgmonitor ] || rm -rf hack/tools/pgmonitor
+	[ ! -d hack/tools/external-snapshotter ] || rm -rf hack/tools/external-snapshotter
 	[ ! -n "$$(ls hack/tools)" ] || rm -r hack/tools/*
 	[ ! -d hack/.kube ] || rm -r hack/.kube
 
@@ -113,7 +123,7 @@ undeploy: ## Undeploy the PostgreSQL Operator
 
 .PHONY: deploy-dev
 deploy-dev: ## Deploy the PostgreSQL Operator locally
-deploy-dev: PGO_FEATURE_GATES ?= "TablespaceVolumes=true"
+deploy-dev: PGO_FEATURE_GATES ?= "TablespaceVolumes=true,VolumeSnapshots=true"
 deploy-dev: get-pgmonitor
 deploy-dev: build-postgres-operator
 deploy-dev: createnamespaces
@@ -190,7 +200,7 @@ check: get-pgmonitor
 check-envtest: ## Run check using envtest and a mock kube api
 check-envtest: ENVTEST_USE = $(ENVTEST) --bin-dir=$(CURDIR)/hack/tools/envtest use $(ENVTEST_K8S_VERSION)
 check-envtest: SHELL = bash
-check-envtest: get-pgmonitor tools/setup-envtest
+check-envtest: get-pgmonitor tools/setup-envtest get-external-snapshotter
 	@$(ENVTEST_USE) --print=overview && echo
 	source <($(ENVTEST_USE) --print=env) && PGO_NAMESPACE="postgres-operator" QUERIES_CONFIG_DIR="$(CURDIR)/${QUERIES_CONFIG_DIR}" \
 		$(GO_TEST) -count=1 -cover ./...
@@ -201,7 +211,7 @@ check-envtest: get-pgmonitor tools/setup-envtest
 # make check-envtest-existing PGO_TEST_TIMEOUT_SCALE=1.2
 .PHONY: check-envtest-existing
 check-envtest-existing: ## Run check using envtest and an existing kube api
-check-envtest-existing: get-pgmonitor
+check-envtest-existing: get-pgmonitor get-external-snapshotter
 check-envtest-existing: createnamespaces
 	kubectl apply --server-side -k ./config/dev
 	USE_EXISTING_CLUSTER=true PGO_NAMESPACE="postgres-operator" QUERIES_CONFIG_DIR="$(CURDIR)/${QUERIES_CONFIG_DIR}" \

config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml

@@ -4324,6 +4324,16 @@ spec:
                     required:
                     - repos
                     type: object
+                  snapshots:
+                    description: VolumeSnapshot configuration
+                    properties:
+                      volumeSnapshotClassName:
+                        description: Name of the VolumeSnapshotClass that should be
+                          used by VolumeSnapshots
+                        type: string
+                    required:
+                    - volumeSnapshotClassName
+                    type: object
                 required:
                 - pgbackrest
                 type: object

config/rbac/cluster/role.yaml

@@ -171,3 +171,14 @@ rules:
   - list
   - patch
   - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch

config/rbac/namespace/role.yaml

@@ -171,3 +171,14 @@ rules:
   - list
   - patch
   - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
+	github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0
 	github.com/onsi/ginkgo/v2 v2.17.2
 	github.com/onsi/gomega v1.33.1
 	github.com/pganalyze/pg_query_go/v5 v5.1.0

go.sum

@@ -76,6 +76,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw=
+github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=

internal/controller/postgrescluster/controller.go

@@ -29,9 +29,11 @@ import (
 	policyv1 "k8s.io/api/policy/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -60,10 +62,11 @@ const (
 
 // Reconciler holds resources for the PostgresCluster reconciler
 type Reconciler struct {
-	Client      client.Client
-	IsOpenShift bool
-	Owner       client.FieldOwner
-	PodExec     func(
+	Client          client.Client
+	DiscoveryClient *discovery.DiscoveryClient
+	IsOpenShift     bool
+	Owner           client.FieldOwner
+	PodExec         func(
 		ctx context.Context, namespace, pod, container string,
 		stdin io.Reader, stdout, stderr io.Writer, command ...string,
 	) error
@@ -345,6 +348,9 @@ func (r *Reconciler) Reconcile(
 			}
 		}
 	}
+	if err == nil {
+		err = r.reconcileVolumeSnapshots(ctx, cluster, instances, clusterVolumes)
+	}
 	if err == nil {
 		err = r.reconcilePGBouncer(ctx, cluster, instances, primaryCertificate, rootCA)
 	}
@@ -447,6 +453,14 @@ func (r *Reconciler) SetupWithManager(mgr manager.Manager) error {
 		}
 	}
 
+	if r.DiscoveryClient == nil {
+		var err error
+		r.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(mgr.GetConfig())
+		if err != nil {
+			return err
+		}
+	}
+
 	return builder.ControllerManagedBy(mgr).
 		For(&v1beta1.PostgresCluster{}).
 		Owns(&corev1.ConfigMap{}).
@@ -467,3 +481,28 @@ func (r *Reconciler) SetupWithManager(mgr manager.Manager) error {
 			r.controllerRefHandlerFuncs()). // watch all StatefulSets
 		Complete(r)
 }
+
+// GroupVersionKindExists checks to see whether a given Kind for a given
+// GroupVersion exists in the Kubernetes API Server.
+func (r *Reconciler) GroupVersionKindExists(groupVersion, kind string) (*bool, error) {
+	if r.DiscoveryClient == nil {
+		return initialize.Bool(false), nil
+	}
+
+	resourceList, err := r.DiscoveryClient.ServerResourcesForGroupVersion(groupVersion)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return initialize.Bool(false), nil
+		}
+
+		return nil, err
+	}
+
+	for _, resource := range resourceList.APIResources {
+		if resource.Kind == kind {
+			return initialize.Bool(true), nil
+		}
+	}
+
+	return initialize.Bool(false), nil
+}