MEDIUM: Remove state and data from dataplaneapi configuration file

Author: hdurand0710

.gitlab-ci.yml

@@ -1,6 +1,7 @@
 stages:
   - lint
   - build
+  - test
   - e2e
 variables:
   DOCKER_HOST: tcp://docker:2375
@@ -101,6 +102,21 @@ build:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
     - if: $CI_PIPELINE_SOURCE == 'push'
 
+test:
+  stage: test
+  image:
+    name: $CI_REGISTRY_GO/golang:$GO_VERSION
+    entrypoint: [ "" ]
+  tags:
+    - go
+  before_script:
+    - go env -w "GONOSUMDB=gitlab.int.haproxy.com/*" "GOPROXY=http://goproxy.int.haproxy.com"
+  script:
+    - go test ./...
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: "$CI_PROJECT_NAMESPACE == 'haproxy-controller' && $CI_PIPELINE_SOURCE == 'push'"
+
 .e2e:
   stage: e2e
   artifacts:

Makefile

@@ -61,3 +61,7 @@ generate:
 generate-native:
 	generate/swagger/script.sh
 	generate/post_swagger.sh
+
+.PHONY: test
+test:
+	go test ./...

cmd/dataplaneapi/main.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 	"syscall"
 
 	loads "github.com/go-openapi/loads"
@@ -78,7 +79,7 @@ func startRuntimeDebugServer() context.CancelFunc {
 	return cancelDebugServer
 }
 
-func startServer(cfg *configuration.Configuration, cancelDebugServer context.CancelFunc) (reload configuration.AtomicBool) {
+func startServer(cfg *configuration.Configuration, cancelDebugServer context.CancelFunc) (reload configuration.AtomicBool) { //nolint: maintidx
 	swaggerSpec, err := loads.Embedded(dataplaneapi.SwaggerJSON, dataplaneapi.FlatSwaggerJSON)
 	if err != nil {
 		fmt.Println(err)
@@ -128,7 +129,8 @@ func startServer(cfg *configuration.Configuration, cancelDebugServer context.Can
 		return
 	}
 
-	err = cfg.Load()
+	var loadMsg []string
+	_, err = cfg.Load()
 	if err != nil {
 		fmt.Println("configuration error:", err)
 		os.Exit(1)
@@ -148,6 +150,13 @@ func startServer(cfg *configuration.Configuration, cancelDebugServer context.Can
 		}
 	}
 
+	loadMsg, err = cfg.LoadDataplaneStorageConfig()
+	if err != nil {
+		fmt.Println("configuration error:", err)
+		os.Exit(1)
+	}
+	cfg.FlagLoadDapiStorageData = true
+
 	// incorporate changes from file to global settings
 	dataplaneapi.SyncWithFileSettings(server, cfg)
 	err = cfg.LoadRuntimeVars(dataplaneapi.SwaggerJSON, server.Host, server.Port)
@@ -173,6 +182,7 @@ func startServer(cfg *configuration.Configuration, cancelDebugServer context.Can
 				cfg.HAProxy.MapsDir = path.Join(storageDir, string(storage.MapsType))
 				cfg.HAProxy.SSLCertsDir = path.Join(storageDir, string(storage.SSLType))
 				cfg.HAProxy.GeneralStorageDir = path.Join(storageDir, string(storage.GeneralType))
+				cfg.HAProxy.DataplaneStorageDir = filepath.Clean(path.Join(storageDir, "dataplane"))
 				cfg.HAProxy.SpoeDir = path.Join(storageDir, string(storage.SpoeType))
 				cfg.HAProxy.SpoeTransactionDir = path.Join(storageDir, string(storage.SpoeTransactionsType))
 				cfg.HAProxy.BackupsDir = path.Join(storageDir, string(storage.BackupsType))
@@ -199,6 +209,13 @@ func startServer(cfg *configuration.Configuration, cancelDebugServer context.Can
 	log.Infof("Build date: %s", BuildTime)
 	log.Infof("Reload strategy: %s", cfg.HAProxy.ReloadStrategy)
 
+	// log deprecation message
+	if len(loadMsg) > 0 {
+		for _, msg := range loadMsg {
+			log.Warning(msg)
+		}
+	}
+
 	err = cfg.Save()
 	if err != nil {
 		log.Fatalf("Error saving configuration: %s", err.Error())
@@ -247,6 +264,7 @@ func startServer(cfg *configuration.Configuration, cancelDebugServer context.Can
 
 	server.ConfigureAPI()
 	dataplaneapi.SetServerStartedCallback(cfg.Notify.ServerStarted.Notify)
+
 	if err := server.Serve(); err != nil {
 		log.Fatalf("Error running HAProxy Data Plane API: %s", err.Error())
 	}

configuration/README.md

@@ -4,3 +4,71 @@
 
 example can be seen [yaml](examples/example-dataplaneapi.yaml)
 full examples of configuration also can be seen at [yaml](examples/example-full.yaml)
+
+# IMPORTANT information regarding migration from 2.x to 3.0 version
+
+Some fields in dataplane configuration file are *deprecated* starting from version 3.0.
+
+They are now moved to Data Plane API internal storage (`dataplane_storage_dir`): [read here](../storage/README.md)
+
+
+
+Those fields are the one that were not really dataplane configuration attributes but *dynamic* data (cluster, users...). They are moved to a dataplane internal storage. Hence the dataplane configuration file is not any more updated with states and data and only contains configuration.
+
+- `dataplaneapi.user` (**only cluster mode users, whose name are starting with `dpapi-c` are migrated**)
+- `cluster`
+- `service_discovery`
+- `status`
+
+Those data are moved to dataplane internal storage.
+
+
+
+## Migration from Dataplane API 2.x to 3.0 behavior
+
+Data (`user`, `cluster`, `service_discovery`, `status`) that were in Dataplane API configuration file are automatically created in:
+- `<dataplane_storage_dir>/service_discovery.json`
+- `<dataplane_storage_dir>/cluster.json`
+
+
+## General behavior regarding the deprecated section
+
+If some of those data are manually updated in Dataplane configuration file (**this should not be done, fields are deprecated**), or if it's the first start of a *3.0* dataplane API:
+- A warning log is issued (search for logs having `"[CFG DEPRECATED]`) with `[SKIP]` or `[MIGRATE]` and the category:
+  -  `[User]` or
+  - `[Cluster]` or
+  - `[Consul]` or
+  - `[AWS Region]` or
+  - `[Status]`
+
+Below an an example for cluster users:
+
+```
+time="2024-07-22T09:12:09+02:00" level=warning msg="[CFG DEPRECATED] [SKIP] [User] [dpapi-c-Abr8s1V]: already migrated. Old location [/home/helene/go/src/gitlab.int.haproxy.com/dataplaneapi-ee/.test/etc/dataplaneapi-cluster.yml] New location [/home/helene/go/src/gitlab.int.haproxy.com/dataplaneapi-ee/.test/storage/dataplane/users.json]. Use only new location"
+```
+```
+time="2024-07-22T09:12:09+02:00" level=warning msg="[CFG DEPRECATED] [MIGRATE] [User] [dpapi-c-8Mk2Z5UK]: migrating. Old location [/home/helene/go/src/gitlab.int.haproxy.com/dataplaneapi-ee/.test/etc/dataplaneapi-cluster.yml] New location [/home/helene/go/src/gitlab.int.haproxy.com/dataplaneapi-ee/.test/storage/dataplane/users.json]. Use only new location"
+```
+
+- **After migration, data are removed from the dataplane configuration file.**
+- **If a data is added again in the dataplane configuration file, and has already been migrated to the internal storage, then the value from the configuration file is ignored. Only the value from the storage is kept.**
+
+
+
+**IMPORTANT NOTE: only cluster mode users, whose name are starting with `dpapi-` are migrated**
+
+
+
+## Cluster Mode: deprecation
+
+The dataplane api configuration field:
+- `mode` (values = `single` or `cluster`)
+
+is now deprecated and this value is removed from dataplane api configuration file after migration.
+
+The way `cluster` vs `single` is now handled is as following:
+
+| Mode |     <dataplane_storage_dir>/cluster.json content    |
+|-----------|-------------------|
+| Single| `cluster` attribute is not empty|
+| Cluster| `cluster` attribute is empty |

configuration/cluster_sync.go

@@ -186,10 +186,9 @@ func (c *ClusterSync) issueRefreshRequest(url, port, basePath string, nodesPath
 		return err
 	}
 	c.cfg.Cluster.Token.Store(resp.Header.Get("X-Node-Key"))
-	err = c.cfg.Save()
-	if err != nil {
+	if err = c.cfg.SaveClusterModeData(); err != nil {
 		log.Warning(err)
-		return err
+		return fmt.Errorf("failed to save cluster mode data: %v", err)
 	}
 	c.cfg.Notify.Reload.Notify()
 	return nil
@@ -206,7 +205,7 @@ func (c *ClusterSync) monitorBootstrapKey() {
 				// do we need to delete cert here maybe?
 				log.Warningf("setting bootstrap key to empty")
 				c.cfg.Cluster.ActiveBootstrapKey.Store("")
-				err := c.cfg.Save()
+				err := c.cfg.SaveClusterModeData()
 				if err != nil {
 					log.Panic(err)
 				}
@@ -253,8 +252,11 @@ func (c *ClusterSync) monitorBootstrapKey() {
 			c.cfg.Cluster.ClusterID.Store(data["cluster-id"])
 			c.cfg.HAProxy.ClusterTLSCertDir = path.Join(data["storage-dir"], "certs-cluster")
 			c.cfg.Cluster.CertificateDir.Store(path.Join(data["storage-dir"], "certs-cluster"))
-			c.cfg.Mode.Store(ModeCluster)
-			err = c.cfg.Save()
+			err = c.cfg.SaveClusterModeData()
+			if err != nil {
+				log.Panic(err)
+			}
+
 			if err != nil {
 				log.Panic(err)
 			}
@@ -273,7 +275,7 @@ func (c *ClusterSync) monitorBootstrapKey() {
 				log.Warning(err)
 				break
 			}
-			err = c.cfg.Save()
+			err = c.cfg.SaveClusterModeData()
 			if err != nil {
 				log.Panic(err)
 			}
@@ -458,7 +460,7 @@ func (c *ClusterSync) issueJoinRequest(url, port, basePath string, registerPath
 	if err != nil {
 		return err
 	}
-	return c.cfg.Save()
+	return c.cfg.SaveClusterModeData()
 }
 
 // checkCertificate checks if we have received valid certificate or we just got CSR back
@@ -467,17 +469,17 @@ func (c *ClusterSync) issueJoinRequest(url, port, basePath string, registerPath
 // -----BEGIN CERTIFICATE----- or -----BEGIN CERTIFICATE REQUEST-----
 func (c *ClusterSync) checkCertificate(node Node) (fetched bool, err error) {
 	if !strings.HasPrefix(node.Certificate, "-----BEGIN CERTIFICATE-----") {
-		c.cfg.Status.Store("unconfigured")
+		c.cfg.Status.Store(StatusUnconfigured)
 		return false, nil
 	}
 	err = renameio.WriteFile(path.Join(c.cfg.GetClusterCertDir(), fmt.Sprintf("dataplane-%s.crt", c.cfg.Name.Load())), []byte(node.Certificate), 0o644)
 	if err != nil {
-		c.cfg.Status.Store("unconfigured")
+		c.cfg.Status.Store(StatusUnconfigured)
 		return false, err
 	}
 	c.cfg.Cluster.CertificateFetched.Store(true)
 	c.cfg.Notify.Reload.Notify()
-	c.cfg.Status.Store("active")
+	c.cfg.Status.Store(StatusActive)
 	return true, nil
 }
 
@@ -548,7 +550,7 @@ func (c *ClusterSync) fetchCert() {
 					log.Warning(err.Error())
 					break
 				}
-				err = c.cfg.Save()
+				err = c.cfg.SaveClusterModeData()
 				if err != nil {
 					log.Warning(err)
 				}