Allow disabling ignore_malformed on data stream's timestamp field (#72406)

If `index.mapping.ignore_malformed` has been set to `true` then
here is no way to overwrite that to `false` for a data stream's
timestamp field.

Before this commit, validation would fail that disallow the usage
of `ignore_malformed` attribute on a data stream's timestamp field.

This commit allows the usage of `ignore_malformed` attribute,
so that `index.mapping.ignore_malformed` can be disabled for a
data stream's timestamp field. The `ignore_malformed` attribute
can only be set to false.

This allows the following index template:

```
PUT /_index_template/filebeat
{
    "index_patterns": [
        "filebeat-*"
    ],
    "template": {
        "settings": {
            "index": {
                "mapping.ignore_malformed": true
            }
        },
        "mappings": {
            "properties": {
                "@timestamp": {
                    "type": "date",
                    "ignore_malformed": false
                }
            }
        }
    },
    "data_stream": {}
}
```

Closes #71755

Author: martijnvg

x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java

@@ -167,6 +167,17 @@ public void doValidate(MappingLookup lookup) {
             configuredSettings.remove("type");
             configuredSettings.remove("meta");
             configuredSettings.remove("format");
+
+            // ignoring malformed values is disallowed (see previous check),
+            // however if `index.mapping.ignore_malformed` has been set to true then
+            // there is no way to disable ignore_malformed for the timestamp field mapper,
+            // other then not using 'index.mapping.ignore_malformed' at all.
+            // So by ignoring the ignore_malformed here, we allow index.mapping.ignore_malformed
+            // index setting to be set to true and then turned off for the timestamp field mapper.
+            // (ignore_malformed will here always be false, otherwise previous check would have failed)
+            Object value = configuredSettings.remove("ignore_malformed");
+            assert value == null || Boolean.FALSE.equals(value);
+
             // All other configured attributes are not allowed:
             if (configuredSettings.isEmpty() == false) {
                 throw new IllegalArgumentException(

x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java

@@ -6,10 +6,15 @@
  */
 package org.elasticsearch.xpack.datastreams.mapper;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperException;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MetadataMapperTestCase;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugins.Plugin;
@@ -20,6 +25,8 @@
 import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class DataStreamTimestampFieldMapperTests extends MetadataMapperTestCase {
 
@@ -145,4 +152,30 @@ public void testValidateNotDisallowedAttribute() {
         })));
         assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] has disallowed attributes: [store]"));
     }
+
+    public void testValidateDefaultIgnoreMalformed() throws Exception {
+        Settings indexSettings = Settings.builder().put(FieldMapper.IGNORE_MALFORMED_SETTING.getKey(), true).build();
+        Exception e = expectThrows(
+            IllegalArgumentException.class,
+            () -> createMapperService(Version.CURRENT, indexSettings, () -> true, timestampMapping(true, b -> {
+                b.startObject("@timestamp");
+                b.field("type", "date");
+                b.endObject();
+            }))
+        );
+        assertThat(
+            e.getMessage(),
+            equalTo("data stream timestamp field [@timestamp] has disallowed [ignore_malformed] attribute specified")
+        );
+
+        MapperService mapperService = createMapperService(Version.CURRENT, indexSettings, () -> true, timestampMapping(true, b -> {
+            b.startObject("@timestamp");
+            b.field("type", "date");
+            b.field("ignore_malformed", false);
+            b.endObject();
+        }));
+        assertThat(mapperService, notNullValue());
+        assertThat(mapperService.documentMapper().mappers().getMapper("@timestamp"), notNullValue());
+        assertThat(((DateFieldMapper) mapperService.documentMapper().mappers().getMapper("@timestamp")).getIgnoreMalformed(), is(false));
+    }
 }