Validate non-secure settings are not in keystore (#42209)

Secure settings currently error if they exist inside elasticsearch.yml.
This commit adds validation that non-secure settings do not exist inside
the keystore.

closes #41831

Author: rjernst

server/src/main/java/org/elasticsearch/common/settings/Setting.java

@@ -467,6 +467,11 @@ public final String getRaw(final Settings settings) {
      * @return the raw string representation of the setting value
      */
     String innerGetRaw(final Settings settings) {
+        SecureSettings secureSettings = settings.getSecureSettings();
+        if (secureSettings != null && secureSettings.getSettingNames().contains(getKey())) {
+            throw new IllegalArgumentException("Setting [" + getKey() + "] is a non-secure setting" +
+                " and must be stored inside elasticsearch.yml, but was found inside the Elasticsearch keystore");
+        }
         return settings.get(getKey(), defaultValue.apply(settings));
     }
 

server/src/test/java/org/elasticsearch/common/settings/SettingTests.java

@@ -964,4 +964,13 @@ public void testAffixMapUpdateWithNullSettingValue() {
         assertEquals("", value);
     }
 
+    public void testNonSecureSettingInKeystore() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("foo", "bar");
+        final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        Setting<String> setting = Setting.simpleString("foo", Property.NodeScope);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> setting.get(settings));
+        assertThat(e.getMessage(), containsString("must be stored inside elasticsearch.yml"));
+    }
+
 }