Revert "Revert "Add deprecation warning for default shards (#30587)""

This reverts commit 93e3e08.

Author: jasontedor

distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/DefaultShardsIT.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+
+import static org.elasticsearch.common.logging.DeprecationLogger.WARNING_HEADER_PATTERN;
+import static org.hamcrest.Matchers.equalTo;
+
+public class DefaultShardsIT extends ESRestTestCase {
+
+    public void testDefaultShards() throws IOException {
+        final Response response = client().performRequest(new Request("PUT", "/index"));
+        final String warning = response.getHeader("Warning");
+        final Matcher matcher = WARNING_HEADER_PATTERN.matcher(warning);
+        assertTrue(matcher.matches());
+        final String message = matcher.group(1);
+        assertThat(message, equalTo("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template"));
+    }
+
+    public void testNonDefaultShards() throws IOException {
+        final Request request = new Request("PUT", "/index");
+        request.setJsonEntity("{\"settings\":{\"index.number_of_shards\":1}}");
+        final Response response = client().performRequest(request);
+        assertNull(response.getHeader("Warning"));
+    }
+
+}

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -19,11 +19,9 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.Version;
@@ -59,7 +57,6 @@
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -81,12 +78,10 @@
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
-import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -376,6 +371,9 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                 // now, put the request settings, so they override templates
                 indexSettingsBuilder.put(request.settings());
                 if (indexSettingsBuilder.get(SETTING_NUMBER_OF_SHARDS) == null) {
+                    deprecationLogger.deprecated("the default number of shards will change from [5] to [1] in 7.0.0; "
+                            + "if you wish to continue using the default of [5] shards, "
+                            + "you must manage this on the create index request or with an index template");
                     indexSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, settings.getAsInt(SETTING_NUMBER_OF_SHARDS, 5));
                 }
                 if (indexSettingsBuilder.get(SETTING_NUMBER_OF_REPLICAS) == null) {

server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java

@@ -120,6 +120,10 @@ public void testMatchTemplates() throws Exception {
 
         final ClusterState result = executeTask();
 
+        assertWarnings("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template");
+
         assertThat(result.metaData().index("test").getAliases(), hasAllKeys("alias_from_template_1", "alias_from_template_2"));
         assertThat(result.metaData().index("test").getAliases(), not(hasKey("alias_from_template_3")));
     }
@@ -134,6 +138,10 @@ public void testApplyDataFromTemplate() throws Exception {
 
         final ClusterState result = executeTask();
 
+        assertWarnings("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template");
+
         assertThat(result.metaData().index("test").getAliases(), hasKey("alias1"));
         assertThat(result.metaData().index("test").getCustoms(), hasKey("custom1"));
         assertThat(result.metaData().index("test").getSettings().get("key1"), equalTo("value1"));
@@ -148,6 +156,10 @@ public void testApplyDataFromRequest() throws Exception {
 
         final ClusterState result = executeTask();
 
+        assertWarnings("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template");
+
         assertThat(result.metaData().index("test").getAliases(), hasKey("alias1"));
         assertThat(result.metaData().index("test").getCustoms(), hasKey("custom1"));
         assertThat(result.metaData().index("test").getSettings().get("key1"), equalTo("value1"));
@@ -177,6 +189,10 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception {
 
         final ClusterState result = executeTask();
 
+        assertWarnings("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template");
+
         assertThat(result.metaData().index("test").getCustoms().get("custom1"), equalTo(mergedCustom));
         assertThat(result.metaData().index("test").getAliases().get("alias1").getSearchRouting(), equalTo("fromReq"));
         assertThat(result.metaData().index("test").getSettings().get("key1"), equalTo("reqValue"));
@@ -186,6 +202,10 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception {
     public void testDefaultSettings() throws Exception {
         final ClusterState result = executeTask();
 
+        assertWarnings("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template");
+
         assertThat(result.getMetaData().index("test").getSettings().get(SETTING_NUMBER_OF_SHARDS), equalTo("5"));
     }
 
@@ -194,6 +214,10 @@ public void testSettingsFromClusterState() throws Exception {
 
         final ClusterState result = executeTask();
 
+        assertWarnings("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template");
+
         assertThat(result.getMetaData().index("test").getSettings().get(SETTING_NUMBER_OF_SHARDS), equalTo("15"));
     }
 
@@ -246,6 +270,10 @@ public void testRequestStateOpen() throws Exception {
 
         executeTask();
 
+        assertWarnings("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template");
+
         verify(allocationService, times(1)).reroute(anyObject(), anyObject());
     }
 
@@ -255,6 +283,10 @@ public void testIndexRemovalOnFailure() throws Exception {
 
         expectThrows(RuntimeException.class, this::executeTask);
 
+        assertWarnings("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template");
+
         verify(indicesService, times(1)).removeIndex(anyObject(), anyObject(), anyObject());
     }
 
@@ -290,6 +322,10 @@ public void testValidateWaitForActiveShardsFailure() throws Exception {
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, this::executeTask);
 
+        assertWarnings("the default number of shards will change from [5] to [1] in 7.0.0; "
+                + "if you wish to continue using the default of [5] shards, "
+                + "you must manage this on the create index request or with an index template");
+
         assertThat(e.getMessage(), containsString("invalid wait_for_active_shards"));
     }
 

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java

@@ -271,7 +271,16 @@ void checkWarningHeaders(final List<String> warningHeaders) {
             final boolean matches = matcher.matches();
             if (matches) {
                 final String message = matcher.group(1);
-                if (expected.remove(message) == false) {
+                // noinspection StatementWithEmptyBody
+                if (message.equals("the default number of shards will change from [5] to [1] in 7.0.0; "
+                        + "if you wish to continue using the default of [5] shards, "
+                        + "you must manage this on the create index request or with an index template")) {
+                    /*
+                     * This warning header will come back in the vast majority of our tests that create an index. Rather than rewrite our
+                     * tests to assert this warning header, we assume that it is expected.
+                     */
+                }
+                else if (expected.remove(message) == false) {
                     unexpected.add(header);
                 }
             } else {