[tmpnet] Avoid serializing the node data directory

This is intended to simplify usage by CLI.

Author: maru-ava

tests/antithesis/init_db.go

@@ -45,7 +45,7 @@ func initBootstrapDB(network *tmpnet.Network, destPath string) error {
 	}
 
 	// Copy the db state from the bootstrap node to the compose volume path.
-	sourcePath := filepath.Join(network.Nodes[0].GetDataDir(), "db")
+	sourcePath := filepath.Join(network.Nodes[0].DataDir, "db")
 	if err := os.MkdirAll(destPath, perms.ReadWriteExecute); err != nil {
 		return fmt.Errorf("failed to create db path %q: %w", destPath, err)
 	}

tests/e2e/faultinjection/duplicate_node_id.go

@@ -5,7 +5,7 @@ package faultinjection
 
 import (
 	"context"
-	"fmt"
+	"path/filepath"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/stretchr/testify/require"
@@ -27,23 +27,22 @@ var _ = ginkgo.Describe("Duplicate node handling", func() {
 		network := e2e.GetEnv(tc).GetNetwork()
 
 		tc.By("creating new node")
-		node1 := e2e.AddEphemeralNode(tc, network, tmpnet.FlagsMap{})
+		node1 := e2e.AddEphemeralNode(tc, network, tmpnet.NewEphemeralNode(tmpnet.FlagsMap{}))
 		e2e.WaitForHealthy(tc, node1)
 
 		tc.By("checking that the new node is connected to its peers")
 		checkConnectedPeers(tc, network.Nodes, node1)
 
 		tc.By("creating a second new node with the same staking keypair as the first new node")
-		node1Flags := node1.Flags
-		node2Flags := tmpnet.FlagsMap{
-			config.StakingTLSKeyContentKey: node1Flags[config.StakingTLSKeyContentKey],
-			config.StakingCertContentKey:   node1Flags[config.StakingCertContentKey],
-			// Construct a unique data dir to ensure the two nodes' data will be stored
-			// separately. Usually the dir name is the node ID but in this one case the nodes have
-			// the same node ID.
-			config.DataDirKey: fmt.Sprintf("%s-second", node1Flags[config.DataDirKey]),
-		}
-		node2 := e2e.AddEphemeralNode(tc, network, node2Flags)
+		node2 := tmpnet.NewEphemeralNode(tmpnet.FlagsMap{
+			config.StakingTLSKeyContentKey: node1.Flags[config.StakingTLSKeyContentKey],
+			config.StakingCertContentKey:   node1.Flags[config.StakingCertContentKey],
+		})
+		// Construct a unique data dir to ensure the two nodes' data will be stored
+		// separately. Usually the dir name is the node ID but in this one case the nodes have
+		// the same node ID.
+		node2.DataDir = filepath.Join(network.Dir, node1.DataDir+"second")
+		_ = e2e.AddEphemeralNode(tc, network, node2)
 
 		tc.By("checking that the second new node fails to become healthy before timeout")
 		err := tmpnet.WaitForHealthy(tc.DefaultContext(), node2)

tests/e2e/p/interchain_workflow.go

@@ -108,7 +108,7 @@ var _ = e2e.DescribePChain("[Interchain Workflow]", ginkgo.Label(e2e.UsesCChainL
 		})
 
 		tc.By("adding new node and waiting for it to report healthy")
-		node := e2e.AddEphemeralNode(tc, network, tmpnet.FlagsMap{})
+		node := e2e.AddEphemeralNode(tc, network, tmpnet.NewEphemeralNode(tmpnet.FlagsMap{}))
 		e2e.WaitForHealthy(tc, node)
 
 		tc.By("retrieving new node's id and pop")

tests/e2e/p/l1.go

@@ -172,9 +172,9 @@ var _ = e2e.DescribePChain("[L1]", func() {
 		})
 
 		tc.By("creating the genesis validator")
-		subnetGenesisNode := e2e.AddEphemeralNode(tc, env.GetNetwork(), tmpnet.FlagsMap{
+		subnetGenesisNode := e2e.AddEphemeralNode(tc, env.GetNetwork(), tmpnet.NewEphemeralNode(tmpnet.FlagsMap{
 			config.TrackSubnetsKey: subnetID.String(),
-		})
+		}))
 
 		genesisNodePoP, err := subnetGenesisNode.GetProofOfPossession()
 		require.NoError(err)
@@ -348,9 +348,9 @@ var _ = e2e.DescribePChain("[L1]", func() {
 		tc.By("advancing the proposervm P-chain height", advanceProposerVMPChainHeight)
 
 		tc.By("creating the validator to register")
-		subnetRegisterNode := e2e.AddEphemeralNode(tc, env.GetNetwork(), tmpnet.FlagsMap{
+		subnetRegisterNode := e2e.AddEphemeralNode(tc, env.GetNetwork(), tmpnet.NewEphemeralNode(tmpnet.FlagsMap{
 			config.TrackSubnetsKey: subnetID.String(),
-		})
+		}))
 
 		registerNodePoP, err := subnetRegisterNode.GetProofOfPossession()
 		require.NoError(err)

tests/e2e/p/staking_rewards.go

@@ -55,9 +55,9 @@ var _ = ginkgo.Describe("[Staking Rewards]", func() {
 		})
 
 		tc.By("adding alpha node, whose uptime should result in a staking reward")
-		alphaNode := e2e.AddEphemeralNode(tc, network, tmpnet.FlagsMap{})
+		alphaNode := e2e.AddEphemeralNode(tc, network, tmpnet.NewEphemeralNode(tmpnet.FlagsMap{}))
 		tc.By("adding beta node, whose uptime should not result in a staking reward")
-		betaNode := e2e.AddEphemeralNode(tc, network, tmpnet.FlagsMap{})
+		betaNode := e2e.AddEphemeralNode(tc, network, tmpnet.NewEphemeralNode(tmpnet.FlagsMap{}))
 
 		// Wait to check health until both nodes have started to minimize the duration
 		// required for both nodes to report healthy.

tests/fixture/e2e/helpers.go

@@ -133,10 +133,9 @@ func NewEthClient(tc tests.TestContext, nodeURI tmpnet.NodeURI) ethclient.Client
 }
 
 // Adds an ephemeral node intended to be used by a single test.
-func AddEphemeralNode(tc tests.TestContext, network *tmpnet.Network, flags tmpnet.FlagsMap) *tmpnet.Node {
+func AddEphemeralNode(tc tests.TestContext, network *tmpnet.Network, node *tmpnet.Node) *tmpnet.Node {
 	require := require.New(tc)
 
-	node := tmpnet.NewEphemeralNode(flags)
 	require.NoError(network.StartNode(tc.DefaultContext(), tc.Log(), node))
 
 	tc.DeferCleanup(func() {

tests/fixture/tmpnet/network.go

@@ -240,13 +240,6 @@ func (n *Network) EnsureDefaultConfig(log logging.Logger) error {
 		n.PrimaryChainConfigs[alias].SetDefaults(chainConfig)
 	}
 
-	// Ensure nodes are configured
-	for i := range n.Nodes {
-		if err := n.EnsureNodeConfig(n.Nodes[i]); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
@@ -504,23 +497,22 @@ func (n *Network) RestartNode(ctx context.Context, log logging.Logger, node *Nod
 
 // Stops all nodes in the network.
 func (n *Network) Stop(ctx context.Context) error {
-	// Target all nodes, including the ephemeral ones
-	nodes, err := ReadNodes(n, true /* includeEphemeral */)
-	if err != nil {
+	// Ensure the node state is up-to-date
+	if err := n.readNodes(); err != nil {
 		return err
 	}
 
 	var errs []error
 
 	// Initiate stop on all nodes
-	for _, node := range nodes {
+	for _, node := range n.Nodes {
 		if err := node.InitiateStop(ctx); err != nil {
 			errs = append(errs, fmt.Errorf("failed to stop node %s: %w", node.NodeID, err))
 		}
 	}
 
 	// Wait for stop to complete on all nodes
-	for _, node := range nodes {
+	for _, node := range n.Nodes {
 		if err := node.WaitForStopped(ctx); err != nil {
 			errs = append(errs, fmt.Errorf("failed to wait for node %s to stop: %w", node.NodeID, err))
 		}
@@ -554,14 +546,9 @@ func (n *Network) EnsureNodeConfig(node *Node) error {
 		return err
 	}
 
-	if len(n.Dir) > 0 {
-		// Ensure the node's data dir is configured
-		dataDir := node.GetDataDir()
-		if len(dataDir) == 0 {
-			// NodeID will have been set by EnsureKeys
-			dataDir = filepath.Join(n.Dir, node.NodeID.String())
-			node.Flags[config.DataDirKey] = dataDir
-		}
+	// Ensure a data directory if not already set
+	if len(node.DataDir) == 0 {
+		node.DataDir = filepath.Join(n.Dir, node.NodeID.String())
 	}
 
 	return nil
@@ -768,16 +755,13 @@ func (n *Network) GetNodeURIs() []NodeURI {
 // collecting the bootstrap details for restarting a node).
 // For consumption outside of avalanchego. Needs to be kept exported.
 func (n *Network) GetBootstrapIPsAndIDs(skippedNode *Node) ([]string, []string, error) {
-	// Collect staking addresses of non-ephemeral nodes for use in bootstrapping a node
-	nodes, err := ReadNodes(n, false /* includeEphemeral */)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to read network's nodes: %w", err)
-	}
-	var (
-		bootstrapIPs = make([]string, 0, len(nodes))
-		bootstrapIDs = make([]string, 0, len(nodes))
-	)
-	for _, node := range nodes {
+	bootstrapIPs := []string{}
+	bootstrapIDs := []string{}
+	for _, node := range n.Nodes {
+		if node.IsEphemeral {
+			// Ephemeral nodes are not guaranteed to stay running
+			continue
+		}
 		if skippedNode != nil && node.NodeID == skippedNode.NodeID {
 			continue
 		}
@@ -940,12 +924,16 @@ func (n *Network) writeNodeFlags(log logging.Logger, node *Node) error {
 	// Only configure the plugin dir with a non-empty value to ensure the use of
 	// the default value (`[datadir]/plugins`) when no plugin dir is configured.
 	processConfig := node.getRuntimeConfig().Process
-	if processConfig != nil && len(processConfig.PluginDir) > 0 {
-		// Ensure the plugin directory exists or the node will fail to start
-		if err := os.MkdirAll(processConfig.PluginDir, perms.ReadWriteExecute); err != nil {
-			return fmt.Errorf("failed to create plugin dir: %w", err)
+	if processConfig != nil {
+		if len(processConfig.PluginDir) > 0 {
+			// Ensure the plugin directory exists or the node will fail to start
+			if err := os.MkdirAll(processConfig.PluginDir, perms.ReadWriteExecute); err != nil {
+				return fmt.Errorf("failed to create plugin dir: %w", err)
+			}
+			flags.SetDefault(config.PluginDirKey, processConfig.PluginDir)
 		}
-		flags.SetDefault(config.PluginDirKey, processConfig.PluginDir)
+
+		flags.SetDefault(config.DataDirKey, node.DataDir)
 	}
 
 	// Set the network and tmpnet defaults last to ensure they can be overridden

tests/fixture/tmpnet/network_config.go

@@ -57,13 +57,35 @@ func (n *Network) readNetwork() error {
 	return n.readConfig()
 }
 
-// Read the non-ephemeral nodes associated with the network from disk.
+// Read the nodes associated with the network from disk.
 func (n *Network) readNodes() error {
-	nodes, err := ReadNodes(n, false /* includeEphemeral */)
+	nodes := []*Node{}
+
+	// Node configuration is stored in child directories
+	entries, err := os.ReadDir(n.Dir)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to read dir: %w", err)
 	}
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+
+		node := NewNode()
+		dataDir := filepath.Join(n.Dir, entry.Name())
+		err := node.Read(n, dataDir)
+		if errors.Is(err, os.ErrNotExist) {
+			// If no config file exists, assume this is not the path of a node
+			continue
+		} else if err != nil {
+			return err
+		}
+
+		nodes = append(nodes, node)
+	}
+
 	n.Nodes = nodes
+
 	return nil
 }
 

tests/fixture/tmpnet/node.go

@@ -6,19 +6,14 @@ package tmpnet
 import (
 	"context"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/netip"
-	"os"
-	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/spf13/cast"
-
 	"github.com/ava-labs/avalanchego/config"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/staking"
@@ -83,25 +78,25 @@ type Node struct {
 	URI            string
 	StakingAddress netip.AddrPort
 
+	// Defaults to [network dir]/[node id] if not set
+	DataDir string
+
 	// Initialized on demand
 	runtime NodeRuntime
 
-	// Intended to be set by the network
 	network *Network
 }
 
 // Initializes a new node with only the data dir set
-func NewNode(dataDir string) *Node {
+func NewNode() *Node {
 	return &Node{
-		Flags: FlagsMap{
-			config.DataDirKey: dataDir,
-		},
+		Flags: FlagsMap{},
 	}
 }
 
 // Initializes an ephemeral node using the provided config flags
 func NewEphemeralNode(flags FlagsMap) *Node {
-	node := NewNode("")
+	node := NewNode()
 	node.Flags = flags
 	node.IsEphemeral = true
 
@@ -112,7 +107,7 @@ func NewEphemeralNode(flags FlagsMap) *Node {
 func NewNodesOrPanic(count int) []*Node {
 	nodes := make([]*Node, count)
 	for i := range nodes {
-		node := NewNode("")
+		node := NewNode()
 		if err := node.EnsureKeys(); err != nil {
 			panic(err)
 		}
@@ -121,51 +116,6 @@ func NewNodesOrPanic(count int) []*Node {
 	return nodes
 }
 
-// Reads a node's configuration from the specified directory.
-func ReadNode(dataDir string) (*Node, error) {
-	node := NewNode(dataDir)
-	return node, node.Read()
-}
-
-// Reads nodes from the specified network directory.
-func ReadNodes(network *Network, includeEphemeral bool) ([]*Node, error) {
-	nodes := []*Node{}
-
-	// Node configuration is stored in child directories
-	entries, err := os.ReadDir(network.Dir)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read dir: %w", err)
-	}
-	for _, entry := range entries {
-		if !entry.IsDir() {
-			continue
-		}
-
-		nodeDir := filepath.Join(network.Dir, entry.Name())
-		node, err := ReadNode(nodeDir)
-		if errors.Is(err, os.ErrNotExist) {
-			// If no config file exists, assume this is not the path of a node
-			continue
-		} else if err != nil {
-			return nil, err
-		}
-
-		if !includeEphemeral && node.IsEphemeral {
-			continue
-		}
-
-		if err := node.EnsureNodeID(); err != nil {
-			return nil, fmt.Errorf("failed to ensure NodeID: %w", err)
-		}
-
-		node.network = network
-
-		nodes = append(nodes, node)
-	}
-
-	return nodes, nil
-}
-
 // Retrieves the runtime for the node.
 func (n *Node) getRuntime() NodeRuntime {
 	if n.runtime == nil {
@@ -210,10 +160,6 @@ func (n *Node) readState() error {
 	return n.getRuntime().readState()
 }
 
-func (n *Node) GetDataDir() string {
-	return cast.ToString(n.Flags[config.DataDirKey])
-}
-
 func (n *Node) GetLocalURI(ctx context.Context) (string, func(), error) {
 	return n.getRuntime().GetLocalURI(ctx)
 }