Autoscaling test scale from empty with node attrs (#68730)

Autoscaling expects data tiers to be used exclusively both for node
roles and in ILM policies. This commit adds a test demonstrating that
as well as documentation for the behavior.

Author: henningandersen

docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc

@@ -7,3 +7,11 @@ the current data set. It signals that additional storage capacity is necessary
 when existing capacity has been exceeded (reactively).
 
 The reactive storage decider is enabled for all policies governing data nodes and has no configuration options.
+
+The decider relies partially on using <<data-tier-allocation,data tier preference>>
+allocation rather than node attributes. In particular, scaling a data tier into
+existence (starting the first node in a tier) will result in starting a node in
+any data tier that is empty if not using allocation based on data tier preference.
+Using the <<ilm-migrate,ILM migrate>> action to migrate between tiers is the
+preferred way of allocating to tiers and fully supports scaling a tier into
+existence.

x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.collect.List;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.autoscaling.LocalStateAutoscaling;
@@ -161,6 +162,49 @@ private void testScaleFromEmptyWarm(boolean allocatable) throws Exception {
 
     }
 
+    public void testScaleFromEmptyLegacy() {
+        internalCluster().startMasterOnlyNode();
+        internalCluster().startNode(
+            NodeRoles.onlyRole(
+                Settings.builder().put(Node.NODE_ATTRIBUTES.getKey() + "data_tier", "hot").build(),
+                DataTier.DATA_HOT_NODE_ROLE
+            )
+        );
+        putAutoscalingPolicy("hot", DataTier.DATA_HOT);
+        putAutoscalingPolicy("warm", DataTier.DATA_WARM);
+        putAutoscalingPolicy("cold", DataTier.DATA_COLD);
+
+        final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        assertAcked(
+            prepareCreate(indexName).setSettings(
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 6)
+                    .put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms")
+                    .put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "data_tier", "hot")
+                    .build()
+            )
+        );
+        refresh();
+        assertThat(capacity().results().get("warm").requiredCapacity().total().storage().getBytes(), Matchers.equalTo(0L));
+        assertThat(capacity().results().get("cold").requiredCapacity().total().storage().getBytes(), Matchers.equalTo(0L));
+
+        assertAcked(
+            client().admin()
+                .indices()
+                .updateSettings(
+                    new UpdateSettingsRequest(indexName).settings(
+                        Settings.builder().put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "data_tier", "warm")
+                    )
+                )
+                .actionGet()
+        );
+
+        assertThat(capacity().results().get("warm").requiredCapacity().total().storage().getBytes(), Matchers.greaterThan(0L));
+        // this is not desirable, but one of the caveats of not using data tiers in the ILM policy.
+        assertThat(capacity().results().get("cold").requiredCapacity().total().storage().getBytes(), Matchers.greaterThan(0L));
+    }
+
     /**
      * Verify that the list of roles includes all data roles to ensure we consider adding future data roles.
      */