Skip to content

Commit 754cda8

Browse files
markharwoodmarkharwood
authored
New queryable "_tier" metadata field (elastic#69288) (elastic#71123)
* New _tier metadata field that supports term, terms, exists and wildcard queries on the first data tier preference stated for an index. Backport of 3aee4c1 Closes elastic#68135
1 parent 69a6a8c commit 754cda8

File tree

5 files changed

+258
-1
lines changed

5 files changed

+258
-1
lines changed

‎docs/reference/mapping/fields.asciidoc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ some of these metadata fields can be customized when a mapping type is created.
1717

1818
The document's mapping type.
1919

20+
<<mapping-tier-field,`_tier`>>::
21+
22+
The current data tier preference of the index to which the document belongs.
23+
2024
<<mapping-id-field,`_id`>>::
2125

2226
The document's ID.
@@ -76,6 +80,8 @@ include::fields/id-field.asciidoc[]
7680

7781
include::fields/index-field.asciidoc[]
7882

83+
include::fields/tier-field.asciidoc[]
84+
7985
include::fields/meta-field.asciidoc[]
8086

8187
include::fields/routing-field.asciidoc[]

‎docs/reference/mapping/fields/tier-field.asciidoc

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
[[mapping-tier-field]]
2+
=== `_tier` field
3+
4+
When performing queries across multiple indexes, it is sometimes desirable to
5+
target indexes held on nodes of a given data tier (`data_hot`, `data_warm`, `data_cold` or `data_frozen`).
6+
The `_tier` field allows matching on the `tier_preference` setting of the index a document was indexed into.
7+
The preferred value is accessible in certain queries :
8+
9+
[source,console]
10+
--------------------------
11+
PUT index_1/_doc/1
12+
{
13+
"text": "Document in index 1"
14+
}
15+
16+
PUT index_2/_doc/2?refresh=true
17+
{
18+
"text": "Document in index 2"
19+
}
20+
21+
GET index_1,index_2/_search
22+
{
23+
"query": {
24+
"terms": {
25+
"_tier": ["data_hot", "data_warm"] <1>
26+
}
27+
}
28+
}
29+
--------------------------
30+
31+
<1> Querying on the `_tier` field
32+
33+
34+
Typically a query will use a `terms` query to list the tiers of interest but you can use
35+
the `_tier` field in any query that is rewritten to a `term` query, such as the
36+
`match`, `query_string`, `term`, `terms`, or `simple_query_string` query, as well as `prefix`
37+
and `wildcard` queries. However, it does not support `regexp` and `fuzzy`
38+
queries.
39+
40+
The `tier_preference` setting of the index is a comma-delimited list of tier names
41+
in order of preference i.e. the preferred tier for hosting an index is listed first followed
42+
by potentially many fall-back options. Query matching only considers the first preference
43+
(the first value of a list).
44+

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.cluster.routing.allocation.mapper;
9+
10+
import org.apache.lucene.search.MatchAllDocsQuery;
11+
import org.apache.lucene.search.MatchNoDocsQuery;
12+
import org.apache.lucene.search.Query;
13+
import org.elasticsearch.common.Strings;
14+
import org.elasticsearch.common.regex.Regex;
15+
import org.elasticsearch.index.mapper.ConstantFieldType;
16+
import org.elasticsearch.index.mapper.KeywordFieldMapper;
17+
import org.elasticsearch.index.mapper.MetadataFieldMapper;
18+
import org.elasticsearch.index.mapper.ValueFetcher;
19+
import org.elasticsearch.index.query.SearchExecutionContext;
20+
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
21+
22+
import java.util.Collections;
23+
24+
public class DataTierFieldMapper extends MetadataFieldMapper {
25+
26+
public static final String NAME = "_tier";
27+
28+
public static final String CONTENT_TYPE = "_tier";
29+
30+
public static final TypeParser PARSER = new FixedTypeParser(c -> new DataTierFieldMapper());
31+
32+
static final class DataTierFieldType extends ConstantFieldType {
33+
34+
static final DataTierFieldType INSTANCE = new DataTierFieldType();
35+
36+
private DataTierFieldType() {
37+
super(NAME, Collections.emptyMap());
38+
}
39+
40+
@Override
41+
public String typeName() {
42+
return CONTENT_TYPE;
43+
}
44+
45+
@Override
46+
public String familyTypeName() {
47+
return KeywordFieldMapper.CONTENT_TYPE;
48+
}
49+
50+
@Override
51+
protected boolean matches(String pattern, boolean caseInsensitive, SearchExecutionContext context) {
52+
if (caseInsensitive) {
53+
pattern = Strings.toLowercaseAscii(pattern);
54+
}
55+
String tierPreference = DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(context.getIndexSettings().getSettings());
56+
if (tierPreference == null) {
57+
return false;
58+
}
59+
// Tier preference can be a comma-delimited list of tiers, ordered by preference
60+
// It was decided we should only test the first of these potentially multiple preferences.
61+
String firstPreference = tierPreference.split(",")[0].trim();
62+
return Regex.simpleMatch(pattern, firstPreference);
63+
}
64+
65+
@Override
66+
public Query existsQuery(SearchExecutionContext context) {
67+
String tierPreference = DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(context.getIndexSettings().getSettings());
68+
if (tierPreference == null) {
69+
return new MatchNoDocsQuery();
70+
}
71+
return new MatchAllDocsQuery();
72+
}
73+
74+
@Override
75+
public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
76+
throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
77+
}
78+
}
79+
80+
public DataTierFieldMapper() {
81+
super(DataTierFieldType.INSTANCE);
82+
}
83+
84+
@Override
85+
protected String contentType() {
86+
return CONTENT_TYPE;
87+
}
88+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.elasticsearch.env.NodeEnvironment;
4242
import org.elasticsearch.index.IndexSettings;
4343
import org.elasticsearch.index.engine.EngineFactory;
44+
import org.elasticsearch.index.mapper.MetadataFieldMapper;
4445
import org.elasticsearch.index.shard.IndexSettingProvider;
4546
import org.elasticsearch.indices.recovery.RecoverySettings;
4647
import org.elasticsearch.license.LicenseService;
@@ -51,6 +52,7 @@
5152
import org.elasticsearch.plugins.ClusterPlugin;
5253
import org.elasticsearch.plugins.EnginePlugin;
5354
import org.elasticsearch.plugins.ExtensiblePlugin;
55+
import org.elasticsearch.plugins.MapperPlugin;
5456
import org.elasticsearch.plugins.RepositoryPlugin;
5557
import org.elasticsearch.repositories.RepositoriesService;
5658
import org.elasticsearch.repositories.Repository;
@@ -61,6 +63,7 @@
6163
import org.elasticsearch.threadpool.ThreadPool;
6264
import org.elasticsearch.watcher.ResourceWatcherService;
6365
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
66+
import org.elasticsearch.xpack.cluster.routing.allocation.mapper.DataTierFieldMapper;
6467
import org.elasticsearch.xpack.core.action.ReloadAnalyzerAction;
6568
import org.elasticsearch.xpack.core.action.TransportReloadAnalyzersAction;
6669
import org.elasticsearch.xpack.core.action.TransportXPackInfoAction;
@@ -96,7 +99,13 @@
9699
import java.util.stream.Collectors;
97100
import java.util.stream.StreamSupport;
98101

99-
public class XPackPlugin extends XPackClientPlugin implements ExtensiblePlugin, RepositoryPlugin, EnginePlugin, ClusterPlugin {
102+
public class XPackPlugin extends XPackClientPlugin
103+
implements
104+
ExtensiblePlugin,
105+
RepositoryPlugin,
106+
EnginePlugin,
107+
ClusterPlugin,
108+
MapperPlugin {
100109
private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(XPackPlugin.class);
101110

102111
public static final String ASYNC_RESULTS_INDEX = ".async-search";
@@ -238,6 +247,11 @@ private static boolean alreadyContainsXPackCustomMetadata(ClusterState clusterSt
238247
clusterState.custom(TokenMetadata.TYPE) != null;
239248
}
240249

250+
@Override
251+
public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
252+
return Collections.singletonMap(DataTierFieldMapper.NAME, DataTierFieldMapper.PARSER);
253+
}
254+
241255
@Override
242256
public Settings additionalSettings() {
243257
final String xpackInstalledNodeAttrSetting = "node.attr." + XPACK_INSTALLED_NODE_ATTR;

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.cluster.routing.allocation.mapper;
9+
10+
import org.apache.lucene.search.MatchAllDocsQuery;
11+
import org.apache.lucene.search.MatchNoDocsQuery;
12+
import org.elasticsearch.Version;
13+
import org.elasticsearch.cluster.metadata.IndexMetadata;
14+
import org.elasticsearch.common.regex.Regex;
15+
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.index.IndexSettings;
17+
import org.elasticsearch.index.mapper.MappedFieldType;
18+
import org.elasticsearch.index.mapper.MapperServiceTestCase;
19+
import org.elasticsearch.index.query.QueryShardException;
20+
import org.elasticsearch.index.query.SearchExecutionContext;
21+
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
22+
23+
import java.io.IOException;
24+
import java.util.Arrays;
25+
import java.util.function.Predicate;
26+
27+
import static java.util.Collections.emptyMap;
28+
import static org.hamcrest.Matchers.containsString;
29+
30+
public class DataTierFieldTypeTests extends MapperServiceTestCase {
31+
32+
public void testPrefixQuery() throws IOException {
33+
MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE;
34+
assertEquals(new MatchAllDocsQuery(), ft.prefixQuery("data_w", null, createContext()));
35+
assertEquals(new MatchNoDocsQuery(), ft.prefixQuery("noSuchRole", null, createContext()));
36+
}
37+
38+
public void testWildcardQuery() {
39+
MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE;
40+
assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("data_w*", null, createContext()));
41+
assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("data_warm", null, createContext()));
42+
assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("Data_Warm", null, true, createContext()));
43+
assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("Data_Warm", null, false, createContext()));
44+
assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("noSuchRole", null, createContext()));
45+
}
46+
47+
public void testTermQuery() {
48+
MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE;
49+
assertEquals(new MatchAllDocsQuery(), ft.termQuery("data_warm", createContext()));
50+
assertEquals(new MatchNoDocsQuery(), ft.termQuery("data_hot", createContext()));
51+
assertEquals(new MatchNoDocsQuery(), ft.termQuery("noSuchRole", createContext()));
52+
}
53+
54+
public void testTermsQuery() {
55+
MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE;
56+
assertEquals(new MatchAllDocsQuery(), ft.termsQuery(Arrays.asList("data_warm"), createContext()));
57+
assertEquals(new MatchNoDocsQuery(), ft.termsQuery(Arrays.asList("data_cold", "data_frozen"), createContext()));
58+
}
59+
60+
public void testRegexpQuery() {
61+
MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE;
62+
QueryShardException e = expectThrows(
63+
QueryShardException.class,
64+
() -> assertEquals(new MatchAllDocsQuery(), ft.regexpQuery("ind.x", 0, 0, 10, null, createContext()))
65+
);
66+
assertThat(e.getMessage(), containsString("Can only use regexp queries on keyword and text fields"));
67+
}
68+
69+
private SearchExecutionContext createContext() {
70+
IndexMetadata indexMetadata = IndexMetadata.builder("index")
71+
.settings(
72+
Settings.builder()
73+
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
74+
// Tier can be an ordered list of preferences - starting with primary and followed by fallbacks.
75+
.put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, "data_warm,data_hot")
76+
)
77+
.numberOfShards(1)
78+
.numberOfReplicas(0)
79+
.build();
80+
IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
81+
82+
Predicate<String> indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index");
83+
return new SearchExecutionContext(
84+
0,
85+
0,
86+
indexSettings,
87+
null,
88+
null,
89+
null,
90+
null,
91+
null,
92+
null,
93+
xContentRegistry(),
94+
writableRegistry(),
95+
null,
96+
null,
97+
System::currentTimeMillis,
98+
null,
99+
indexNameMatcher,
100+
() -> true,
101+
null,
102+
emptyMap()
103+
);
104+
}
105+
}

0 commit comments

Comments
 (0)