Skip to content

Commit 945aaa2

Browse files
martijnvgmartijnvg
authored
[8.x] Reapply "synthetic source index setting provider should check source field mapper" (#113759) (#113828)
Backporting #113759 to 8.x branch. Originally added via #113522, but then reverted via #113745, because of mixed cluster test failures (#113730). This PR is a clean revert of the commit the reverted #113522 and one additional commit that should address the build failures report in #113730 : c7bd242 Basically create index invocation that would fail anyway should be ignored. If mapper service creation now fails, then we just assume that there is no synthetic source usage. This is ok, because the index creation would fail anyway later one. Closes #113730
1 parent a7f4e60 commit 945aaa2

File tree

6 files changed

+360
-15
lines changed

6 files changed

+360
-15
lines changed

‎x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
4040
assertThat(features, Matchers.empty());
4141
}
4242
{
43-
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
43+
if (randomBoolean()) {
44+
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
45+
} else if (randomBoolean()) {
46+
String mapping = """
47+
{
48+
"properties": {
49+
"field1": {
50+
"type": "keyword",
51+
"time_series_dimension": true
52+
}
53+
}
54+
}
55+
""";
56+
var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
57+
createIndex("test-index", settings, mapping);
58+
} else {
59+
String mapping = """
60+
{
61+
"_source": {
62+
"mode": "synthetic"
63+
}
64+
}
65+
""";
66+
createIndex("test-index", Settings.EMPTY, mapping);
67+
}
4468
var response = getAsMap("/_license/feature_usage");
4569
@SuppressWarnings("unchecked")
4670
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
4242
assertThat(features, Matchers.empty());
4343
}
4444
{
45-
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
45+
if (randomBoolean()) {
46+
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
47+
} else if (randomBoolean()) {
48+
String mapping = """
49+
{
50+
"properties": {
51+
"field1": {
52+
"type": "keyword",
53+
"time_series_dimension": true
54+
}
55+
}
56+
}
57+
""";
58+
var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
59+
createIndex("test-index", settings, mapping);
60+
} else {
61+
String mapping = """
62+
{
63+
"_source": {
64+
"mode": "synthetic"
65+
}
66+
}
67+
""";
68+
createIndex("test-index", Settings.EMPTY, mapping);
69+
}
4670
var response = getAsMap("/_license/feature_usage");
4771
@SuppressWarnings("unchecked")
4872
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,10 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
5151
if (DiscoveryNode.isStateless(settings)) {
5252
return List.of(logsdbIndexModeSettingsProvider);
5353
}
54-
return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider);
54+
return List.of(
55+
new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory()),
56+
logsdbIndexModeSettingsProvider
57+
);
5558
}
5659

5760
@Override

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java

Lines changed: 75 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,28 +9,41 @@
99

1010
import org.apache.logging.log4j.LogManager;
1111
import org.apache.logging.log4j.Logger;
12+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1213
import org.elasticsearch.cluster.metadata.Metadata;
14+
import org.elasticsearch.common.UUIDs;
1315
import org.elasticsearch.common.compress.CompressedXContent;
1416
import org.elasticsearch.common.settings.Settings;
17+
import org.elasticsearch.core.CheckedFunction;
18+
import org.elasticsearch.core.Strings;
1519
import org.elasticsearch.index.IndexMode;
1620
import org.elasticsearch.index.IndexSettingProvider;
1721
import org.elasticsearch.index.IndexSettings;
22+
import org.elasticsearch.index.IndexVersion;
23+
import org.elasticsearch.index.mapper.MapperService;
1824

25+
import java.io.IOException;
1926
import java.time.Instant;
2027
import java.util.List;
21-
import java.util.Locale;
28+
29+
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
2230

2331
/**
2432
* An index setting provider that overwrites the source mode from synthetic to stored if synthetic source isn't allowed to be used.
2533
*/
26-
public class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
34+
final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
2735

2836
private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceIndexSettingsProvider.class);
2937

3038
private final SyntheticSourceLicenseService syntheticSourceLicenseService;
39+
private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
3140

32-
public SyntheticSourceIndexSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService) {
41+
SyntheticSourceIndexSettingsProvider(
42+
SyntheticSourceLicenseService syntheticSourceLicenseService,
43+
CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory
44+
) {
3345
this.syntheticSourceLicenseService = syntheticSourceLicenseService;
46+
this.mapperServiceFactory = mapperServiceFactory;
3447
}
3548

3649
@Override
@@ -46,19 +59,71 @@ public Settings getAdditionalIndexSettings(
4659
// This index name is used when validating component and index templates, we should skip this check in that case.
4760
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
4861
boolean isTemplateValidation = "validate-index-name".equals(indexName);
49-
if (newIndexHasSyntheticSourceUsage(indexTemplateAndCreateRequestSettings)
62+
if (newIndexHasSyntheticSourceUsage(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
5063
&& syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) {
5164
LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
5265
// TODO: handle falling back to stored source
5366
}
5467
return Settings.EMPTY;
5568
}
5669

57-
boolean newIndexHasSyntheticSourceUsage(Settings indexTemplateAndCreateRequestSettings) {
58-
// TODO: build tmp MapperService and check whether SourceFieldMapper#isSynthetic() to determine synthetic source usage.
59-
// Not using IndexSettings.MODE.get() to avoid validation that may fail at this point.
60-
var rawIndexMode = indexTemplateAndCreateRequestSettings.get(IndexSettings.MODE.getKey());
61-
IndexMode indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null;
62-
return indexMode != null && indexMode.isSyntheticSourceEnabled();
70+
boolean newIndexHasSyntheticSourceUsage(
71+
String indexName,
72+
boolean isTimeSeries,
73+
Settings indexTemplateAndCreateRequestSettings,
74+
List<CompressedXContent> combinedTemplateMappings
75+
) {
76+
if ("validate-index-name".equals(indexName)) {
77+
// This index name is used when validating component and index templates, we should skip this check in that case.
78+
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
79+
return false;
80+
}
81+
82+
var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings);
83+
try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) {
84+
// combinedTemplateMappings can be null when creating system indices
85+
// combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
86+
if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
87+
combinedTemplateMappings = List.of(new CompressedXContent("{}"));
88+
}
89+
mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
90+
return mapperService.documentMapper().sourceMapper().isSynthetic();
91+
} catch (AssertionError | Exception e) {
92+
// In case invalid mappings or setting are provided, then mapper service creation can fail.
93+
// In that case it is ok to return false here. The index creation will fail anyway later, so need to fallback to stored source.
94+
LOGGER.info(() -> Strings.format("unable to create mapper service for index [%s]", indexName), e);
95+
return false;
96+
}
97+
}
98+
99+
// Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used:
100+
private IndexMetadata buildIndexMetadataForMapperService(
101+
String indexName,
102+
boolean isTimeSeries,
103+
Settings indexTemplateAndCreateRequestSettings
104+
) {
105+
var tmpIndexMetadata = IndexMetadata.builder(indexName);
106+
107+
int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(indexTemplateAndCreateRequestSettings);
108+
int dummyShards = indexTemplateAndCreateRequestSettings.getAsInt(
109+
IndexMetadata.SETTING_NUMBER_OF_SHARDS,
110+
dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1
111+
);
112+
int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
113+
var finalResolvedSettings = Settings.builder()
114+
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
115+
.put(indexTemplateAndCreateRequestSettings)
116+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
117+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
118+
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
119+
120+
if (isTimeSeries) {
121+
finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES);
122+
// Avoid failing because index.routing_path is missing (in case fields are marked as dimension)
123+
finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path"));
124+
}
125+
126+
tmpIndexMetadata.settings(finalResolvedSettings);
127+
return tmpIndexMetadata.build();
63128
}
64129
}

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
/**
1717
* Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
1818
*/
19-
public final class SyntheticSourceLicenseService {
19+
final class SyntheticSourceLicenseService {
2020

2121
private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
2222

@@ -39,7 +39,7 @@ public final class SyntheticSourceLicenseService {
3939
private XPackLicenseState licenseState;
4040
private volatile boolean syntheticSourceFallback;
4141

42-
public SyntheticSourceLicenseService(Settings settings) {
42+
SyntheticSourceLicenseService(Settings settings) {
4343
syntheticSourceFallback = FALLBACK_SETTING.get(settings);
4444
}
4545

0 commit comments

Comments
 (0)