Skip to content

Commit 9b960ef

Browse files
nik9000nik9000
committed
Add index.mode: time_series (backport of elastic#77626)
This adds a setting to enable time series mode that is hidden by the `es.index_mode_feature_flag_registered` feature flag. All it does right now is make sure you haven't configured index sorting and partitioning. This gives us a place to "hang" all of our further work on time series mode. Time series mode will entirely take over index sorting. We don't believe you'll ever be able to configure sorting yourself when you are in time series mode. We don't expect time series mode to support index partitioning when we first build it but we'd like to get there.
1 parent 70ef7f3 commit 9b960ef

File tree

11 files changed

+354
-5
lines changed

11 files changed

+354
-5
lines changed

‎qa/mixed-cluster/build.gradle

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
3939

4040
setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
4141
setting 'xpack.security.enabled', 'false'
42+
if (BuildParams.isSnapshotBuild() == false) {
43+
systemProperty 'es.index_mode_feature_flag_registered', 'true'
44+
}
4245
}
4346
}
4447

‎qa/smoke-test-multinode/build.gradle

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
* Side Public License, v 1.
77
*/
88

9+
import org.elasticsearch.gradle.internal.info.BuildParams
10+
911
apply plugin: 'elasticsearch.internal-testclusters'
1012
apply plugin: 'elasticsearch.standalone-rest-test'
1113
apply plugin: 'elasticsearch.rest-test'
@@ -25,6 +27,9 @@ testClusters.matching { it.name == "integTest" }.configureEach {
2527

2628
testClusters.all {
2729
setting 'xpack.security.enabled', 'false'
30+
if (BuildParams.isSnapshotBuild() == false) {
31+
systemProperty 'es.index_mode_feature_flag_registered', 'true'
32+
}
2833
}
2934

3035
tasks.named("integTest").configure {

‎rest-api-spec/build.gradle

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import org.elasticsearch.gradle.internal.info.BuildParams
2+
13
apply plugin: 'elasticsearch.build'
24
apply plugin: 'elasticsearch.publish'
35
apply plugin: 'elasticsearch.rest-resources'
@@ -32,6 +34,9 @@ artifacts {
3234

3335
testClusters.all {
3436
module ':modules:mapper-extras'
37+
if (BuildParams.isSnapshotBuild() == false) {
38+
systemProperty 'es.index_mode_feature_flag_registered', 'true'
39+
}
3540
}
3641

3742
tasks.named("test").configure {enabled = false }

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
enable:
2+
- skip:
3+
version: " - 7.15.99"
4+
reason: introduced in 7.16.0
5+
6+
- do:
7+
indices.create:
8+
index: test_index
9+
body:
10+
settings:
11+
index:
12+
mode: time_series
13+
number_of_replicas: 0
14+
number_of_shards: 2
15+
mappings:
16+
properties:
17+
"@timestamp":
18+
type: date
19+
metricset:
20+
type: keyword
21+
k8s:
22+
properties:
23+
pod:
24+
properties:
25+
uid:
26+
type: keyword
27+
name:
28+
type: keyword
29+
ip:
30+
type: ip
31+
network:
32+
properties:
33+
tx:
34+
type: long
35+
rx:
36+
type: long
37+
38+
---
39+
no sort field:
40+
- skip:
41+
version: " - 7.15.99"
42+
reason: introduced in 7.16.0
43+
44+
- do:
45+
catch: /\[index.mode=time_series\] is incompatible with \[index.sort.field\]/
46+
indices.create:
47+
index: test_index
48+
body:
49+
settings:
50+
index:
51+
mode: time_series
52+
sort.field: ['a']
53+
54+
---
55+
no sort order:
56+
- skip:
57+
version: " - 7.15.99"
58+
reason: introduced in 7.16.0
59+
60+
- do:
61+
catch: /\[index.mode=time_series\] is incompatible with \[index.sort.order\]/
62+
indices.create:
63+
index: test_index
64+
body:
65+
settings:
66+
index:
67+
mode: time_series
68+
sort.order: ['DESC']
69+
70+
---
71+
no sort mode:
72+
- skip:
73+
version: " - 7.15.99"
74+
reason: introduced in 7.16.0
75+
76+
- do:
77+
catch: /\[index.mode=time_series\] is incompatible with \[index.sort.mode\]/
78+
indices.create:
79+
index: test_index
80+
body:
81+
settings:
82+
index:
83+
mode: time_series
84+
sort.mode: ['MIN']
85+
86+
---
87+
no sort missing:
88+
- skip:
89+
version: " - 7.15.99"
90+
reason: introduced in 7.16.0
91+
92+
- do:
93+
catch: /\[index.mode=time_series\] is incompatible with \[index.sort.missing\]/
94+
indices.create:
95+
index: test_index
96+
body:
97+
settings:
98+
index:
99+
mode: time_series
100+
sort.missing: ['_last']
101+
102+
---
103+
no partitioning:
104+
- skip:
105+
version: " - 7.15.99"
106+
reason: introduced in 7.16.0
107+
108+
- do:
109+
catch: /\[index.mode=time_series\] is incompatible with \[index.routing_partition_size\]/
110+
indices.create:
111+
index: test_index
112+
body:
113+
settings:
114+
index:
115+
mode: time_series
116+
shards: 5
117+
routing_partition_size: 2

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import org.elasticsearch.indices.IndicesRequestCache;
3636
import org.elasticsearch.indices.ShardLimitValidator;
3737

38-
import java.util.Arrays;
3938
import java.util.Collections;
4039
import java.util.HashSet;
4140
import java.util.Map;
@@ -50,7 +49,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
5049

5150
public static final Predicate<String> INDEX_SETTINGS_KEY_PREDICATE = (s) -> s.startsWith(IndexMetadata.INDEX_SETTING_PREFIX);
5251

53-
public static final Set<Setting<?>> BUILT_IN_INDEX_SETTINGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
52+
private static final Set<Setting<?>> ALWAYS_ENABLED_BUILT_IN_INDEX_SETTINGS = org.elasticsearch.core.Set.of(
5453
MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY,
5554
MergeSchedulerConfig.AUTO_THROTTLE_SETTING,
5655
MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
@@ -178,9 +177,18 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
178177
}
179178
}
180179
}, Property.IndexScope), // this allows similarity settings to be passed
181-
Setting.groupSetting("index.analysis.", Property.IndexScope) // this allows analysis settings to be passed
180+
Setting.groupSetting("index.analysis.", Property.IndexScope)); // this allows analysis settings to be passed
182181

183-
)));
182+
public static final Set<Setting<?>> BUILT_IN_INDEX_SETTINGS = builtInIndexSettings();
183+
184+
private static Set<Setting<?>> builtInIndexSettings() {
185+
if (false == IndexSettings.isTimeSeriesModeEnabled()) {
186+
return ALWAYS_ENABLED_BUILT_IN_INDEX_SETTINGS;
187+
}
188+
Set<Setting<?>> result = new HashSet<>(ALWAYS_ENABLED_BUILT_IN_INDEX_SETTINGS);
189+
result.add(IndexSettings.MODE);
190+
return org.elasticsearch.core.Set.copyOf(result);
191+
}
184192

185193
public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS);
186194

‎server/src/main/java/org/elasticsearch/common/settings/Setting.java

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1417,7 +1417,28 @@ public static ByteSizeValue parseByteSize(String s, ByteSizeValue minValue, Byte
14171417
* @return the setting object
14181418
*/
14191419
public static <T extends Enum<T>> Setting<T> enumSetting(Class<T> clazz, String key, T defaultValue, Property... properties) {
1420-
return new Setting<>(key, defaultValue.toString(), e -> Enum.valueOf(clazz, e.toUpperCase(Locale.ROOT)), properties);
1420+
return enumSetting(clazz, key, defaultValue, s -> {}, properties);
1421+
}
1422+
1423+
/**
1424+
* Creates a setting where the allowed values are defined as enum constants. All enum constants must be uppercase.
1425+
*
1426+
* @param clazz the enum class
1427+
* @param key the key for the setting
1428+
* @param defaultValue the default value for this setting
1429+
* @param validator validator for this setting
1430+
* @param properties properties for this setting like scope, filtering...
1431+
* @param <T> the generics type parameter reflecting the actual type of the enum
1432+
* @return the setting object
1433+
*/
1434+
public static <T extends Enum<T>> Setting<T> enumSetting(
1435+
Class<T> clazz,
1436+
String key,
1437+
T defaultValue,
1438+
Validator<T> validator,
1439+
Property... properties
1440+
) {
1441+
return new Setting<>(key, defaultValue.toString(), e -> Enum.valueOf(clazz, e.toUpperCase(Locale.ROOT)), validator, properties);
14211442
}
14221443

14231444
/**

‎server/src/main/java/org/elasticsearch/index/IndexMode.java

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.index;
10+
11+
import org.elasticsearch.cluster.metadata.IndexMetadata;
12+
import org.elasticsearch.common.settings.Setting;
13+
import org.elasticsearch.common.settings.Settings;
14+
15+
import java.util.List;
16+
import java.util.Map;
17+
import java.util.Objects;
18+
import java.util.stream.Stream;
19+
20+
import static java.util.stream.Collectors.toSet;
21+
22+
/**
23+
* "Mode" that controls which behaviors and settings an index supports.
24+
*/
25+
public enum IndexMode {
26+
STANDARD {
27+
@Override
28+
void validateWithOtherSettings(Map<Setting<?>, Object> settings) {}
29+
},
30+
TIME_SERIES {
31+
@Override
32+
void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
33+
if (settings.get(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING) != Integer.valueOf(1)) {
34+
throw new IllegalArgumentException(error(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING));
35+
}
36+
for (Setting<?> unsupported : TIME_SERIES_UNSUPPORTED) {
37+
if (false == Objects.equals(unsupported.getDefault(Settings.EMPTY), settings.get(unsupported))) {
38+
throw new IllegalArgumentException(error(unsupported));
39+
}
40+
}
41+
}
42+
43+
private String error(Setting<?> unsupported) {
44+
return "[" + IndexSettings.MODE.getKey() + "=time_series] is incompatible with [" + unsupported.getKey() + "]";
45+
}
46+
};
47+
48+
private static final List<Setting<?>> TIME_SERIES_UNSUPPORTED = org.elasticsearch.core.List.of(
49+
IndexSortConfig.INDEX_SORT_FIELD_SETTING,
50+
IndexSortConfig.INDEX_SORT_ORDER_SETTING,
51+
IndexSortConfig.INDEX_SORT_MODE_SETTING,
52+
IndexSortConfig.INDEX_SORT_MISSING_SETTING
53+
);
54+
55+
static final List<Setting<?>> VALIDATE_WITH_SETTINGS = org.elasticsearch.core.List.copyOf(
56+
Stream.concat(Stream.of(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING), TIME_SERIES_UNSUPPORTED.stream()).collect(toSet())
57+
);
58+
59+
abstract void validateWithOtherSettings(Map<Setting<?>, Object> settings);
60+
}

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.apache.logging.log4j.Logger;
1111
import org.apache.logging.log4j.util.Strings;
1212
import org.apache.lucene.index.MergePolicy;
13+
import org.elasticsearch.Build;
1314
import org.elasticsearch.Version;
1415
import org.elasticsearch.cluster.metadata.IndexMetadata;
1516
import org.elasticsearch.common.logging.Loggers;
@@ -19,13 +20,16 @@
1920
import org.elasticsearch.common.settings.Settings;
2021
import org.elasticsearch.common.unit.ByteSizeUnit;
2122
import org.elasticsearch.common.unit.ByteSizeValue;
23+
import org.elasticsearch.core.Booleans;
2224
import org.elasticsearch.core.TimeValue;
2325
import org.elasticsearch.index.translog.Translog;
2426
import org.elasticsearch.ingest.IngestService;
2527
import org.elasticsearch.node.Node;
2628

2729
import java.util.Collections;
30+
import java.util.Iterator;
2831
import java.util.List;
32+
import java.util.Map;
2933
import java.util.concurrent.TimeUnit;
3034
import java.util.function.Consumer;
3135
import java.util.function.Function;
@@ -335,12 +339,59 @@ public final class IndexSettings {
335339
public static final Setting<Double> FILE_BASED_RECOVERY_THRESHOLD_SETTING
336340
= Setting.doubleSetting("index.recovery.file_based_threshold", 0.1d, 0.0d, Setting.Property.IndexScope);
337341

342+
/**
343+
* Is the {@code index.mode} enabled? It should only be enbaled if you
344+
* pass a jvm parameter or are running a snapshot build.
345+
*/
346+
private static final Boolean TIME_SERIES_MODE_FEATURE_FLAG_REGISTERED;
347+
348+
static {
349+
final String property = System.getProperty("es.index_mode_feature_flag_registered");
350+
if (Build.CURRENT.isSnapshot() && property != null) {
351+
throw new IllegalArgumentException("es.index_mode_feature_flag_registered is only supported in non-snapshot builds");
352+
}
353+
TIME_SERIES_MODE_FEATURE_FLAG_REGISTERED = Booleans.parseBoolean(property, null);
354+
}
355+
356+
public static boolean isTimeSeriesModeEnabled() {
357+
return Build.CURRENT.isSnapshot() || (TIME_SERIES_MODE_FEATURE_FLAG_REGISTERED != null && TIME_SERIES_MODE_FEATURE_FLAG_REGISTERED);
358+
}
359+
360+
/**
361+
* The {@link IndexMode "mode"} of the index.
362+
*/
363+
public static final Setting<IndexMode> MODE = Setting.enumSetting(IndexMode.class,
364+
"index.mode",
365+
IndexMode.STANDARD,
366+
new Setting.Validator<IndexMode>() {
367+
@Override
368+
public void validate(IndexMode value) {}
369+
370+
@Override
371+
public void validate(IndexMode value, Map<Setting<?>, Object> settings) {
372+
value.validateWithOtherSettings(settings);
373+
}
374+
375+
@Override
376+
public Iterator<Setting<?>> settings() {
377+
return IndexMode.VALIDATE_WITH_SETTINGS.iterator();
378+
}
379+
},
380+
Property.IndexScope,
381+
Property.Final
382+
);
383+
338384
private final Index index;
339385
private final Version version;
340386
private final Logger logger;
341387
private final String nodeName;
342388
private final Settings nodeSettings;
343389
private final int numberOfShards;
390+
/**
391+
* The {@link IndexMode "mode"} of the index.
392+
*/
393+
private final IndexMode mode;
394+
344395
// volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock
345396
private volatile Settings settings;
346397
private volatile IndexMetadata indexMetadata;
@@ -483,6 +534,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
483534
nodeName = Node.NODE_NAME_SETTING.get(settings);
484535
this.indexMetadata = indexMetadata;
485536
numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null);
537+
mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD;
486538

487539
this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings);
488540
this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
@@ -693,6 +745,13 @@ public IndexMetadata getIndexMetadata() {
693745
*/
694746
public int getNumberOfReplicas() { return settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, null); }
695747

748+
/**
749+
* "Mode" that controls which behaviors and settings an index supports.
750+
*/
751+
public IndexMode getMode() {
752+
return mode;
753+
}
754+
696755
/**
697756
* Returns the node settings. The settings returned from {@link #getSettings()} are a merged version of the
698757
* index settings and the node settings where node settings are overwritten by index settings.

0 commit comments

Comments
 (0)