[8.x] Reapply "synthetic source index setting provider should check source field mapper"

x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

@@ -40,7 +40,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             assertThat(features, Matchers.empty());
         }
         {
-            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            if (randomBoolean()) {
+                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            } else if (randomBoolean()) {
+                String mapping = """
+                    {
+                        "properties": {
+                            "field1": {
+                                "type": "keyword",
+                                "time_series_dimension": true
+                            }
+                        }
+                    }
+                    """;
+                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
+                createIndex("test-index", settings, mapping);
+            } else {
+                String mapping = """
+                    {
+                        "_source": {
+                            "mode": "synthetic"
+                        }
+                    }
+                    """;
+                createIndex("test-index", Settings.EMPTY, mapping);
+            }
             var response = getAsMap("/_license/feature_usage");
             @SuppressWarnings("unchecked")
             List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

@@ -42,7 +42,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             assertThat(features, Matchers.empty());
         }
         {
-            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            if (randomBoolean()) {
+                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            } else if (randomBoolean()) {
+                String mapping = """
+                    {
+                        "properties": {
+                            "field1": {
+                                "type": "keyword",
+                                "time_series_dimension": true
+                            }
+                        }
+                    }
+                    """;
+                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
+                createIndex("test-index", settings, mapping);
+            } else {
+                String mapping = """
+                    {
+                        "_source": {
+                            "mode": "synthetic"
+                        }
+                    }
+                    """;
+                createIndex("test-index", Settings.EMPTY, mapping);
+            }
             var response = getAsMap("/_license/feature_usage");
             @SuppressWarnings("unchecked")
             List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java

@@ -51,7 +51,10 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
         if (DiscoveryNode.isStateless(settings)) {
             return List.of(logsdbIndexModeSettingsProvider);
         }
-        return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider);
+        return List.of(
+            new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory()),
+            logsdbIndexModeSettingsProvider
+        );
     }
 
     @Override

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java

@@ -9,28 +9,41 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.MapperService;
 
+import java.io.IOException;
 import java.time.Instant;
 import java.util.List;
-import java.util.Locale;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
 
 /**
  * An index setting provider that overwrites the source mode from synthetic to stored if synthetic source isn't allowed to be used.
  */
-public class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
+final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
 
     private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceIndexSettingsProvider.class);
 
     private final SyntheticSourceLicenseService syntheticSourceLicenseService;
+    private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
 
-    public SyntheticSourceIndexSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService) {
+    SyntheticSourceIndexSettingsProvider(
+        SyntheticSourceLicenseService syntheticSourceLicenseService,
+        CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory
+    ) {
         this.syntheticSourceLicenseService = syntheticSourceLicenseService;
+        this.mapperServiceFactory = mapperServiceFactory;
     }
 
     @Override
@@ -46,19 +59,71 @@ public Settings getAdditionalIndexSettings(
         // This index name is used when validating component and index templates, we should skip this check in that case.
         // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
         boolean isTemplateValidation = "validate-index-name".equals(indexName);
-        if (newIndexHasSyntheticSourceUsage(indexTemplateAndCreateRequestSettings)
+        if (newIndexHasSyntheticSourceUsage(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
             && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) {
             LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
             // TODO: handle falling back to stored source
         }
         return Settings.EMPTY;
     }
 
-    boolean newIndexHasSyntheticSourceUsage(Settings indexTemplateAndCreateRequestSettings) {
-        // TODO: build tmp MapperService and check whether SourceFieldMapper#isSynthetic() to determine synthetic source usage.
-        // Not using IndexSettings.MODE.get() to avoid validation that may fail at this point.
-        var rawIndexMode = indexTemplateAndCreateRequestSettings.get(IndexSettings.MODE.getKey());
-        IndexMode indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null;
-        return indexMode != null && indexMode.isSyntheticSourceEnabled();
+    boolean newIndexHasSyntheticSourceUsage(
+        String indexName,
+        boolean isTimeSeries,
+        Settings indexTemplateAndCreateRequestSettings,
+        List<CompressedXContent> combinedTemplateMappings
+    ) {
+        if ("validate-index-name".equals(indexName)) {
+            // This index name is used when validating component and index templates, we should skip this check in that case.
+            // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
+            return false;
+        }
+
+        var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings);
+        try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) {
+            // combinedTemplateMappings can be null when creating system indices
+            // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
+            if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
+                combinedTemplateMappings = List.of(new CompressedXContent("{}"));
+            }
+            mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
+            return mapperService.documentMapper().sourceMapper().isSynthetic();
+        } catch (AssertionError | Exception e) {
+            // In case invalid mappings or setting are provided, then mapper service creation can fail.
+            // In that case it is ok to return false here. The index creation will fail anyway later, so need to fallback to stored source.
+            LOGGER.info(() -> Strings.format("unable to create mapper service for index [%s]", indexName), e);
+            return false;
+        }
+    }
+
+    // Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used:
+    private IndexMetadata buildIndexMetadataForMapperService(
+        String indexName,
+        boolean isTimeSeries,
+        Settings indexTemplateAndCreateRequestSettings
+    ) {
+        var tmpIndexMetadata = IndexMetadata.builder(indexName);
+
+        int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(indexTemplateAndCreateRequestSettings);
+        int dummyShards = indexTemplateAndCreateRequestSettings.getAsInt(
+            IndexMetadata.SETTING_NUMBER_OF_SHARDS,
+            dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1
+        );
+        int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+        var finalResolvedSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+            .put(indexTemplateAndCreateRequestSettings)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
+            .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
+
+        if (isTimeSeries) {
+            finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES);
+            // Avoid failing because index.routing_path is missing (in case fields are marked as dimension)
+            finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path"));
+        }
+
+        tmpIndexMetadata.settings(finalResolvedSettings);
+        return tmpIndexMetadata.build();
     }
 }

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java

@@ -16,7 +16,7 @@
 /**
  * Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
  */
-public final class SyntheticSourceLicenseService {
+final class SyntheticSourceLicenseService {
 
     private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
 
@@ -39,7 +39,7 @@ public final class SyntheticSourceLicenseService {
     private XPackLicenseState licenseState;
     private volatile boolean syntheticSourceFallback;
 
-    public SyntheticSourceLicenseService(Settings settings) {
+    SyntheticSourceLicenseService(Settings settings) {
         syntheticSourceFallback = FALLBACK_SETTING.get(settings);
     }