Skip to content

Commit 60ae0ca

Browse files
martijnvgmartijnvg
committed
Reapply "synthetic source index setting provider should check source field mapper (#113522)" (#113745)
This reverts commit e0a2d61.
1 parent acd4f07 commit 60ae0ca

File tree

6 files changed

+318
-15
lines changed

6 files changed

+318
-15
lines changed

‎x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
4040
assertThat(features, Matchers.empty());
4141
}
4242
{
43-
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
43+
if (randomBoolean()) {
44+
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
45+
} else if (randomBoolean()) {
46+
String mapping = """
47+
{
48+
"properties": {
49+
"field1": {
50+
"type": "keyword",
51+
"time_series_dimension": true
52+
}
53+
}
54+
}
55+
""";
56+
var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
57+
createIndex("test-index", settings, mapping);
58+
} else {
59+
String mapping = """
60+
{
61+
"_source": {
62+
"mode": "synthetic"
63+
}
64+
}
65+
""";
66+
createIndex("test-index", Settings.EMPTY, mapping);
67+
}
4468
var response = getAsMap("/_license/feature_usage");
4569
@SuppressWarnings("unchecked")
4670
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
4242
assertThat(features, Matchers.empty());
4343
}
4444
{
45-
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
45+
if (randomBoolean()) {
46+
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
47+
} else if (randomBoolean()) {
48+
String mapping = """
49+
{
50+
"properties": {
51+
"field1": {
52+
"type": "keyword",
53+
"time_series_dimension": true
54+
}
55+
}
56+
}
57+
""";
58+
var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
59+
createIndex("test-index", settings, mapping);
60+
} else {
61+
String mapping = """
62+
{
63+
"_source": {
64+
"mode": "synthetic"
65+
}
66+
}
67+
""";
68+
createIndex("test-index", Settings.EMPTY, mapping);
69+
}
4670
var response = getAsMap("/_license/feature_usage");
4771
@SuppressWarnings("unchecked")
4872
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,10 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
5151
if (DiscoveryNode.isStateless(settings)) {
5252
return List.of(logsdbIndexModeSettingsProvider);
5353
}
54-
return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider);
54+
return List.of(
55+
new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory()),
56+
logsdbIndexModeSettingsProvider
57+
);
5558
}
5659

5760
@Override

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java

Lines changed: 72 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,28 +9,41 @@
99

1010
import org.apache.logging.log4j.LogManager;
1111
import org.apache.logging.log4j.Logger;
12+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1213
import org.elasticsearch.cluster.metadata.Metadata;
14+
import org.elasticsearch.common.UUIDs;
1315
import org.elasticsearch.common.compress.CompressedXContent;
1416
import org.elasticsearch.common.settings.Settings;
17+
import org.elasticsearch.core.CheckedFunction;
1518
import org.elasticsearch.index.IndexMode;
1619
import org.elasticsearch.index.IndexSettingProvider;
1720
import org.elasticsearch.index.IndexSettings;
21+
import org.elasticsearch.index.IndexVersion;
22+
import org.elasticsearch.index.mapper.MapperService;
1823

24+
import java.io.IOException;
25+
import java.io.UncheckedIOException;
1926
import java.time.Instant;
2027
import java.util.List;
21-
import java.util.Locale;
28+
29+
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
2230

2331
/**
2432
* An index setting provider that overwrites the source mode from synthetic to stored if synthetic source isn't allowed to be used.
2533
*/
26-
public class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
34+
final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
2735

2836
private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceIndexSettingsProvider.class);
2937

3038
private final SyntheticSourceLicenseService syntheticSourceLicenseService;
39+
private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
3140

32-
public SyntheticSourceIndexSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService) {
41+
SyntheticSourceIndexSettingsProvider(
42+
SyntheticSourceLicenseService syntheticSourceLicenseService,
43+
CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory
44+
) {
3345
this.syntheticSourceLicenseService = syntheticSourceLicenseService;
46+
this.mapperServiceFactory = mapperServiceFactory;
3447
}
3548

3649
@Override
@@ -43,19 +56,68 @@ public Settings getAdditionalIndexSettings(
4356
Settings indexTemplateAndCreateRequestSettings,
4457
List<CompressedXContent> combinedTemplateMappings
4558
) {
46-
if (newIndexHasSyntheticSourceUsage(indexTemplateAndCreateRequestSettings)
59+
if (newIndexHasSyntheticSourceUsage(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
4760
&& syntheticSourceLicenseService.fallbackToStoredSource()) {
4861
LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
4962
// TODO: handle falling back to stored source
5063
}
5164
return Settings.EMPTY;
5265
}
5366

54-
boolean newIndexHasSyntheticSourceUsage(Settings indexTemplateAndCreateRequestSettings) {
55-
// TODO: build tmp MapperService and check whether SourceFieldMapper#isSynthetic() to determine synthetic source usage.
56-
// Not using IndexSettings.MODE.get() to avoid validation that may fail at this point.
57-
var rawIndexMode = indexTemplateAndCreateRequestSettings.get(IndexSettings.MODE.getKey());
58-
IndexMode indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null;
59-
return indexMode != null && indexMode.isSyntheticSourceEnabled();
67+
boolean newIndexHasSyntheticSourceUsage(
68+
String indexName,
69+
boolean isTimeSeries,
70+
Settings indexTemplateAndCreateRequestSettings,
71+
List<CompressedXContent> combinedTemplateMappings
72+
) {
73+
if ("validate-index-name".equals(indexName)) {
74+
// This index name is used when validating component and index templates, we should skip this check in that case.
75+
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
76+
return false;
77+
}
78+
79+
var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings);
80+
try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) {
81+
// combinedTemplateMappings can be null when creating system indices
82+
// combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
83+
if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
84+
combinedTemplateMappings = List.of(new CompressedXContent("{}"));
85+
}
86+
mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
87+
return mapperService.documentMapper().sourceMapper().isSynthetic();
88+
} catch (IOException e) {
89+
throw new UncheckedIOException(e);
90+
}
91+
}
92+
93+
// Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used:
94+
private IndexMetadata buildIndexMetadataForMapperService(
95+
String indexName,
96+
boolean isTimeSeries,
97+
Settings indexTemplateAndCreateRequestSettings
98+
) {
99+
var tmpIndexMetadata = IndexMetadata.builder(indexName);
100+
101+
int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(indexTemplateAndCreateRequestSettings);
102+
int dummyShards = indexTemplateAndCreateRequestSettings.getAsInt(
103+
IndexMetadata.SETTING_NUMBER_OF_SHARDS,
104+
dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1
105+
);
106+
int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
107+
var finalResolvedSettings = Settings.builder()
108+
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
109+
.put(indexTemplateAndCreateRequestSettings)
110+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
111+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
112+
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
113+
114+
if (isTimeSeries) {
115+
finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES);
116+
// Avoid failing because index.routing_path is missing (in case fields are marked as dimension)
117+
finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path"));
118+
}
119+
120+
tmpIndexMetadata.settings(finalResolvedSettings);
121+
return tmpIndexMetadata.build();
60122
}
61123
}

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
/**
1717
* Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
1818
*/
19-
public final class SyntheticSourceLicenseService {
19+
final class SyntheticSourceLicenseService {
2020

2121
private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
2222

@@ -39,7 +39,7 @@ public final class SyntheticSourceLicenseService {
3939
private XPackLicenseState licenseState;
4040
private volatile boolean syntheticSourceFallback;
4141

42-
public SyntheticSourceLicenseService(Settings settings) {
42+
SyntheticSourceLicenseService(Settings settings) {
4343
syntheticSourceFallback = FALLBACK_SETTING.get(settings);
4444
}
4545

0 commit comments

Comments
 (0)