Skip to content

Commit 91ca9c5

Browse files
astefanastefan
authored
QL: constant_keyword support (#53241) (#53602)
(cherry picked from commit d6cd4ce)
1 parent dc2edc9 commit 91ca9c5

File tree

27 files changed

+497
-127
lines changed

27 files changed

+497
-127
lines changed

‎docs/reference/sql/language/data-types.asciidoc

+16-15
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,22 @@ s|SQL precision
1313

1414
4+h| Core types
1515

16-
| <<null-value, `null`>> | null | NULL | 0
17-
| <<boolean, `boolean`>> | boolean | BOOLEAN | 1
18-
| <<number, `byte`>> | byte | TINYINT | 3
19-
| <<number, `short`>> | short | SMALLINT | 5
20-
| <<number, `integer`>> | integer | INTEGER | 10
21-
| <<number, `long`>> | long | BIGINT | 19
22-
| <<number, `double`>> | double | DOUBLE | 15
23-
| <<number, `float`>> | float | REAL | 7
24-
| <<number, `half_float`>> | half_float | FLOAT | 3
25-
| <<number, `scaled_float`>> | scaled_float | DOUBLE | 15
26-
| <<keyword, `keyword`>> | keyword | VARCHAR | 32,766
27-
| <<text, `text`>> | text | VARCHAR | 2,147,483,647
28-
| <<binary, `binary`>> | binary | VARBINARY | 2,147,483,647
29-
| <<date, `date`>> | datetime | TIMESTAMP | 29
30-
| <<ip, `ip`>> | ip | VARCHAR | 39
16+
| <<null-value, `null`>> | null | NULL | 0
17+
| <<boolean, `boolean`>> | boolean | BOOLEAN | 1
18+
| <<number, `byte`>> | byte | TINYINT | 3
19+
| <<number, `short`>> | short | SMALLINT | 5
20+
| <<number, `integer`>> | integer | INTEGER | 10
21+
| <<number, `long`>> | long | BIGINT | 19
22+
| <<number, `double`>> | double | DOUBLE | 15
23+
| <<number, `float`>> | float | REAL | 7
24+
| <<number, `half_float`>> | half_float | FLOAT | 3
25+
| <<number, `scaled_float`>> | scaled_float | DOUBLE | 15
26+
| <<keyword, `keyword`>> | keyword | VARCHAR | 32,766
27+
| <<constant-keyword, `constant_keyword`>> | constant_keyword| VARCHAR | 32,766
28+
| <<text, `text`>> | text | VARCHAR | 2,147,483,647
29+
| <<binary, `binary`>> | binary | VARBINARY | 2,147,483,647
30+
| <<date, `date`>> | datetime | TIMESTAMP | 29
31+
| <<ip, `ip`>> | ip | VARCHAR | 39
3132

3233
4+h| Complex types
3334

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java

+2
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.util.Objects;
2929
import java.util.StringJoiner;
3030

31+
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
3132
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
3233
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
3334
import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT;
@@ -213,6 +214,7 @@ protected Object unwrapMultiValue(Object values) {
213214
protected boolean isFromDocValuesOnly(DataType dataType) {
214215
return dataType == KEYWORD // because of ignore_above.
215216
|| dataType == DATETIME
217+
|| dataType == CONSTANT_KEYWORD // because a non-existent value is considered the constant value itself
216218
|| dataType == SCALED_FLOAT; // because of scaling_factor
217219
}
218220

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java

+18-3
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.elasticsearch.index.IndexNotFoundException;
2626
import org.elasticsearch.index.IndexSettings;
2727
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
28+
import org.elasticsearch.xpack.ql.type.ConstantKeywordEsField;
2829
import org.elasticsearch.xpack.ql.type.DataType;
2930
import org.elasticsearch.xpack.ql.type.DataTypeRegistry;
3031
import org.elasticsearch.xpack.ql.type.DateEsField;
@@ -60,6 +61,7 @@
6061
import static java.util.Collections.emptyMap;
6162
import static java.util.Collections.emptySet;
6263
import static org.elasticsearch.action.ActionListener.wrap;
64+
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
6365
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
6466
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
6567
import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
@@ -298,8 +300,13 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
298300
StringBuilder errorMessage = new StringBuilder();
299301

300302
boolean hasUnmapped = types.containsKey(UNMAPPED);
303+
// a keyword field and a constant_keyword field with the same name in two different indices are considered "compatible"
304+
// since a common use case of constant_keyword field involves two indices with a field having the same name: one being
305+
// a keyword, the other being a constant_keyword
306+
boolean hasCompatibleKeywords = types.containsKey(KEYWORD.esType()) && types.containsKey(CONSTANT_KEYWORD.esType());
307+
int allowedTypesCount = (hasUnmapped ? 2 : 1) + (hasCompatibleKeywords ? 1 : 0);
301308

302-
if (types.size() > (hasUnmapped ? 2 : 1)) {
309+
if (types.size() > allowedTypesCount) {
303310
// build the error message
304311
// and create a MultiTypeField
305312

@@ -344,6 +351,11 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
344351
}
345352
}
346353

354+
// if there are both a keyword and a constant_keyword type for this field, only keep the keyword as a common compatible type
355+
if (hasCompatibleKeywords) {
356+
types.remove(CONSTANT_KEYWORD.esType());
357+
}
358+
347359
// everything checks
348360
return null;
349361
});
@@ -435,6 +447,9 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa
435447
if (esType == DATETIME) {
436448
return new DateEsField(fieldName, props, isAggregateable);
437449
}
450+
if (esType == CONSTANT_KEYWORD) {
451+
return new ConstantKeywordEsField(fieldName);
452+
}
438453
if (esType == UNSUPPORTED) {
439454
return new UnsupportedEsField(fieldName, typeName, null, props);
440455
}
@@ -501,14 +516,14 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[
501516

502517
for (Entry<String, Map<String, FieldCapabilities>> entry : sortedFields) {
503518
String fieldName = entry.getKey();
504-
Map<String, FieldCapabilities> types = entry.getValue();
505519

506520
// ignore size added by the mapper plugin
507521
if (FIELD_NAMES_BLACKLIST.contains(fieldName)) {
508522
continue;
509523
}
510524

511-
// apply verification
525+
Map<String, FieldCapabilities> types = new LinkedHashMap<>(entry.getValue());
526+
// apply verification and possibly remove the "duplicate" CONSTANT_KEYWORD field type
512527
final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types);
513528

514529
// filter meta fields and unmapped

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.ql.type;
8+
9+
import java.util.Collections;
10+
11+
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
12+
13+
/**
14+
* SQL-related information about an index field with a constant_keyword type
15+
*/
16+
public class ConstantKeywordEsField extends KeywordEsField {
17+
18+
public ConstantKeywordEsField(String name) {
19+
super(name, CONSTANT_KEYWORD, Collections.emptyMap(), true, Short.MAX_VALUE, false, false);
20+
}
21+
22+
}

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java

+6-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
2323
import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
24+
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
2425
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
2526
import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
2627
import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -59,9 +60,12 @@ public static DataType commonType(DataType left, DataType right) {
5960
return left;
6061
}
6162
if (isString(left) && isString(right)) {
62-
if (left == TEXT) {
63+
if (left == TEXT || right == TEXT) {
6364
return TEXT;
6465
}
66+
if (left == KEYWORD) {
67+
return KEYWORD;
68+
}
6569
return right;
6670
}
6771
if (left.isNumeric() && right.isNumeric()) {
@@ -120,7 +124,7 @@ public static Converter converterFor(DataType from, DataType to) {
120124
return DefaultConverter.TO_NULL;
121125
}
122126
// proper converters
123-
if (to == KEYWORD || to == TEXT) {
127+
if (to == KEYWORD || to == TEXT || to == CONSTANT_KEYWORD) {
124128
return conversionToString(from);
125129
}
126130
if (to == LONG) {

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java

+20-18
Original file line numberDiff line numberDiff line change
@@ -20,33 +20,34 @@
2020
public final class DataTypes {
2121

2222
// @formatter:off
23-
public static final DataType UNSUPPORTED = new DataType("UNSUPPORTED", null, 0, false, false, false);
23+
public static final DataType UNSUPPORTED = new DataType("UNSUPPORTED", null, 0, false, false, false);
2424

25-
public static final DataType NULL = new DataType("null", 0, false, false, false);
25+
public static final DataType NULL = new DataType("null", 0, false, false, false);
2626

27-
public static final DataType BOOLEAN = new DataType("boolean", 1, false, false, false);
27+
public static final DataType BOOLEAN = new DataType("boolean", 1, false, false, false);
2828
// integer numeric
29-
public static final DataType BYTE = new DataType("byte", Byte.BYTES, true, false, true);
30-
public static final DataType SHORT = new DataType("short", Short.BYTES, true, false, true);
31-
public static final DataType INTEGER = new DataType("integer", Integer.BYTES, true, false, true);
32-
public static final DataType LONG = new DataType("long", Long.BYTES, true, false, true);
29+
public static final DataType BYTE = new DataType("byte", Byte.BYTES, true, false, true);
30+
public static final DataType SHORT = new DataType("short", Short.BYTES, true, false, true);
31+
public static final DataType INTEGER = new DataType("integer", Integer.BYTES, true, false, true);
32+
public static final DataType LONG = new DataType("long", Long.BYTES, true, false, true);
3333
// decimal numeric
34-
public static final DataType DOUBLE = new DataType("double", Double.BYTES, false, true, true);
35-
public static final DataType FLOAT = new DataType("float", Float.BYTES, false, true, true);
36-
public static final DataType HALF_FLOAT = new DataType("half_float", Float.BYTES, false, true, true);
37-
public static final DataType SCALED_FLOAT = new DataType("scaled_float", Long.BYTES, false, true, true);
34+
public static final DataType DOUBLE = new DataType("double", Double.BYTES, false, true, true);
35+
public static final DataType FLOAT = new DataType("float", Float.BYTES, false, true, true);
36+
public static final DataType HALF_FLOAT = new DataType("half_float", Float.BYTES, false, true, true);
37+
public static final DataType SCALED_FLOAT = new DataType("scaled_float", Long.BYTES, false, true, true);
3838
// string
39-
public static final DataType KEYWORD = new DataType("keyword", Integer.MAX_VALUE, false, false, true);
40-
public static final DataType TEXT = new DataType("text", Integer.MAX_VALUE, false, false, false);
39+
public static final DataType KEYWORD = new DataType("keyword", Integer.MAX_VALUE, false, false, true);
40+
public static final DataType TEXT = new DataType("text", Integer.MAX_VALUE, false, false, false);
41+
public static final DataType CONSTANT_KEYWORD = new DataType("constant_keyword", Integer.MAX_VALUE, false, false, true);
4142
// date
42-
public static final DataType DATETIME = new DataType("DATETIME", "date", Long.BYTES, false, false, true);
43+
public static final DataType DATETIME = new DataType("DATETIME", "date", Long.BYTES, false, false, true);
4344
// ip
4445
public static final DataType IP = new DataType("ip", 45, false, false, true);
4546
// binary
46-
public static final DataType BINARY = new DataType("binary", Integer.MAX_VALUE, false, false, true);
47+
public static final DataType BINARY = new DataType("binary", Integer.MAX_VALUE, false, false, true);
4748
// complex types
48-
public static final DataType OBJECT = new DataType("object", 0, false, false, false);
49-
public static final DataType NESTED = new DataType("nested", 0, false, false, false);
49+
public static final DataType OBJECT = new DataType("object", 0, false, false, false);
50+
public static final DataType NESTED = new DataType("nested", 0, false, false, false);
5051
//@formatter:on
5152

5253
private static final Collection<DataType> TYPES = unmodifiableList(Arrays.asList(
@@ -63,6 +64,7 @@ public final class DataTypes {
6364
SCALED_FLOAT,
6465
KEYWORD,
6566
TEXT,
67+
CONSTANT_KEYWORD,
6668
DATETIME,
6769
IP,
6870
BINARY,
@@ -134,7 +136,7 @@ public static boolean isUnsupported(DataType from) {
134136
}
135137

136138
public static boolean isString(DataType t) {
137-
return t == KEYWORD || t == TEXT;
139+
return t == KEYWORD || t == TEXT || t == CONSTANT_KEYWORD;
138140
}
139141

140142
public static boolean isPrimitive(DataType t) {

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,12 @@ public KeywordEsField(String name, Map<String, EsField> properties, boolean hasD
2929

3030
public KeywordEsField(String name, Map<String, EsField> properties, boolean hasDocValues, int precision,
3131
boolean normalized, boolean isAlias) {
32-
super(name, KEYWORD, properties, hasDocValues, isAlias);
32+
this(name, KEYWORD, properties, hasDocValues, precision, normalized, isAlias);
33+
}
34+
35+
protected KeywordEsField(String name, DataType esDataType, Map<String, EsField> properties, boolean hasDocValues, int precision,
36+
boolean normalized, boolean isAlias) {
37+
super(name, esDataType, properties, hasDocValues, isAlias);
3338
this.precision = precision;
3439
this.normalized = normalized;
3540
}

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import java.util.Map;
1212
import java.util.function.Function;
1313

14+
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
1415
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
1516
import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
1617

@@ -44,7 +45,7 @@ public Exact getExactInfo() {
4445
private Tuple<EsField, String> findExact() {
4546
EsField field = null;
4647
for (EsField property : getProperties().values()) {
47-
if (property.getDataType() == KEYWORD && property.getExactInfo().hasExact()) {
48+
if ((property.getDataType() == KEYWORD || property.getDataType() == CONSTANT_KEYWORD) && property.getExactInfo().hasExact()) {
4849
if (field != null) {
4950
return new Tuple<>(null, "Multiple exact keyword candidates available for [" + getName() +
5051
"]; specify which one to use");

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import java.util.Map.Entry;
1515

1616
import static java.util.Collections.emptyMap;
17+
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
1718
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
1819
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
1920
import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED;
@@ -89,6 +90,8 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje
8990
int length = intSetting(content.get("ignore_above"), Short.MAX_VALUE);
9091
boolean normalized = Strings.hasText(textSetting(content.get("normalizer"), null));
9192
field = new KeywordEsField(name, properties, docValues, length, normalized);
93+
} else if (esDataType == CONSTANT_KEYWORD) {
94+
field = new ConstantKeywordEsField(name);
9295
} else if (esDataType == DATETIME) {
9396
field = new DateEsField(name, properties, docValues);
9497
} else if (esDataType == UNSUPPORTED) {

‎x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java

+7
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor;
1818
import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
1919
import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
20+
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
2021
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
2122
import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
2223
import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
@@ -360,6 +361,7 @@ public void testCommonType() {
360361
assertEquals(BOOLEAN, commonType(BOOLEAN, BOOLEAN));
361362
assertEquals(NULL, commonType(NULL, NULL));
362363
assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
364+
assertEquals(DOUBLE, commonType(DOUBLE, CONSTANT_KEYWORD));
363365
assertEquals(LONG, commonType(TEXT, LONG));
364366
assertEquals(SHORT, commonType(SHORT, BYTE));
365367
assertEquals(FLOAT, commonType(BYTE, FLOAT));
@@ -369,6 +371,11 @@ public void testCommonType() {
369371
// strings
370372
assertEquals(TEXT, commonType(TEXT, KEYWORD));
371373
assertEquals(TEXT, commonType(KEYWORD, TEXT));
374+
assertEquals(TEXT, commonType(TEXT, CONSTANT_KEYWORD));
375+
assertEquals(TEXT, commonType(CONSTANT_KEYWORD, TEXT));
376+
assertEquals(KEYWORD, commonType(KEYWORD, CONSTANT_KEYWORD));
377+
assertEquals(KEYWORD, commonType(CONSTANT_KEYWORD, KEYWORD));
378+
assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, CONSTANT_KEYWORD));
372379
}
373380

374381
public void testEsDataTypes() {

0 commit comments

Comments
 (0)