Skip to content

Commit 7adb286

Browse files
astefanastefan
authored
SQL: change the way unsupported data types fields are handled (#50823)
The hierarchy of fields/sub-fields under a field that is of an unsupported data type will be marked as unsupported as well. Until this change, the behavior was to set the unsupported data type field's hierarchy as empty. Example, considering the following hierarchy of fields/subfields a -> b -> c -> d, if b would be of type "foo", then b, c and d will be marked as unsupported.
1 parent 3fba171 commit 7adb286

File tree

10 files changed

+209
-23
lines changed

10 files changed

+209
-23
lines changed

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -358,11 +358,12 @@ private static EsField createField(String fieldName, Map<String, Map<String, Fie
358358

359359
int dot = fieldName.lastIndexOf('.');
360360
String fullFieldName = fieldName;
361+
EsField parent = null;
361362

362363
if (dot >= 0) {
363364
String parentName = fieldName.substring(0, dot);
364365
fieldName = fieldName.substring(dot + 1);
365-
EsField parent = flattedMapping.get(parentName);
366+
parent = flattedMapping.get(parentName);
366367
if (parent == null) {
367368
Map<String, FieldCapabilities> map = globalCaps.get(parentName);
368369
Function<String, EsField> fieldFunction;
@@ -387,7 +388,22 @@ private static EsField createField(String fieldName, Map<String, Map<String, Fie
387388
}
388389

389390
EsField esField = field.apply(fieldName);
390-
391+
392+
if (parent != null && parent instanceof UnsupportedEsField) {
393+
UnsupportedEsField unsupportedParent = (UnsupportedEsField) parent;
394+
String inherited = unsupportedParent.getInherited();
395+
String type = unsupportedParent.getOriginalType();
396+
397+
if (inherited == null) {
398+
// mark the sub-field as unsupported, just like its parent, setting the first unsupported parent as the current one
399+
esField = new UnsupportedEsField(esField.getName(), type, unsupportedParent.getName(), esField.getProperties());
400+
} else {
401+
// mark the sub-field as unsupported, just like its parent, but setting the first unsupported parent
402+
// as the parent's first unsupported grandparent
403+
esField = new UnsupportedEsField(esField.getName(), type, inherited, esField.getProperties());
404+
}
405+
}
406+
391407
parentProps.put(fieldName, esField);
392408
flattedMapping.put(fullFieldName, esField);
393409

@@ -408,7 +424,7 @@ private static EsField createField(String fieldName, String typeName, Map<String
408424
case DATETIME:
409425
return new DateEsField(fieldName, props, isAggregateable);
410426
case UNSUPPORTED:
411-
return new UnsupportedEsField(fieldName, typeName);
427+
return new UnsupportedEsField(fieldName, typeName, null, props);
412428
default:
413429
return new EsField(fieldName, esType, props, isAggregateable, isAlias);
414430
}

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ private static void walkMapping(String name, Object value, Map<String, EsField>
7373
properties = Collections.emptyMap();
7474
}
7575
} else {
76-
properties = Collections.emptyMap();
76+
properties = fromEs(content);
7777
}
7878
boolean docValues = boolSetting(content.get("doc_values"), esDataType.defaultDocValues);
7979
final EsField field;
@@ -91,7 +91,8 @@ private static void walkMapping(String name, Object value, Map<String, EsField>
9191
break;
9292
case UNSUPPORTED:
9393
String type = content.get("type").toString();
94-
field = new UnsupportedEsField(name, type);
94+
field = new UnsupportedEsField(name, type, null, properties);
95+
propagateUnsupportedType(name, type, properties);
9596
break;
9697
default:
9798
field = new EsField(name, esDataType, properties, docValues);
@@ -113,4 +114,21 @@ private static boolean boolSetting(Object value, boolean defaultValue) {
113114
private static int intSetting(Object value, int defaultValue) {
114115
return value == null ? defaultValue : Integer.parseInt(value.toString());
115116
}
117+
118+
private static void propagateUnsupportedType(String inherited, String originalType, Map<String, EsField> properties) {
119+
if (properties != null && properties.isEmpty() == false) {
120+
for (Entry<String, EsField> entry : properties.entrySet()) {
121+
EsField field = entry.getValue();
122+
UnsupportedEsField u;
123+
if (field instanceof UnsupportedEsField) {
124+
u = (UnsupportedEsField) field;
125+
u = new UnsupportedEsField(u.getName(), originalType, inherited, u.getProperties());
126+
} else {
127+
u = new UnsupportedEsField(field.getName(), originalType, inherited, field.getProperties());
128+
}
129+
entry.setValue(u);
130+
propagateUnsupportedType(inherited, originalType, u.getProperties());
131+
}
132+
}
133+
}
116134
}

‎x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/UnsupportedEsField.java

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,24 +5,40 @@
55
*/
66
package org.elasticsearch.xpack.ql.type;
77

8-
import java.util.Collections;
8+
import java.util.Map;
99
import java.util.Objects;
10+
import java.util.TreeMap;
1011

1112
/**
12-
* SQL-related information about an index field that cannot be supported by SQL
13+
* SQL-related information about an index field that cannot be supported by SQL.
14+
* All the subfields (properties) of an unsupported type should also be unsupported.
1315
*/
1416
public class UnsupportedEsField extends EsField {
1517

16-
private String originalType;
17-
18+
private final String originalType;
19+
private final String inherited; // for fields belonging to parents (or grandparents) that have an unsupported type
20+
1821
public UnsupportedEsField(String name, String originalType) {
19-
super(name, DataType.UNSUPPORTED, Collections.emptyMap(), false);
22+
this(name, originalType, null, new TreeMap<>());
23+
}
24+
25+
public UnsupportedEsField(String name, String originalType, String inherited, Map<String, EsField> properties) {
26+
super(name, DataType.UNSUPPORTED, properties, false);
2027
this.originalType = originalType;
28+
this.inherited = inherited;
2129
}
2230

2331
public String getOriginalType() {
2432
return originalType;
2533
}
34+
35+
public String getInherited() {
36+
return inherited;
37+
}
38+
39+
public boolean hasInherited() {
40+
return inherited != null;
41+
}
2642

2743
@Override
2844
public boolean equals(Object o) {
@@ -36,11 +52,12 @@ public boolean equals(Object o) {
3652
return false;
3753
}
3854
UnsupportedEsField that = (UnsupportedEsField) o;
39-
return Objects.equals(originalType, that.originalType);
55+
return Objects.equals(originalType, that.originalType)
56+
&& Objects.equals(inherited, that.inherited);
4057
}
4158

4259
@Override
4360
public int hashCode() {
44-
return Objects.hash(super.hashCode(), originalType);
61+
return Objects.hash(super.hashCode(), originalType, inherited);
4562
}
4663
}

‎x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,10 @@ public void testUnsupportedTypes() {
188188
Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
189189
EsField dt = mapping.get("range");
190190
assertThat(dt.getDataType().typeName(), is("unsupported"));
191+
dt = mapping.get("time_frame");
192+
assertThat(dt.getDataType().typeName(), is("unsupported"));
193+
dt = mapping.get("flat");
194+
assertThat(dt.getDataType().typeName(), is("unsupported"));
191195
}
192196

193197
public static Map<String, EsField> loadMapping(String name) {

‎x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,26 @@
77
"unsupported" : { "type" : "ip_range" },
88
"date" : { "type" : "date"},
99
"shape": { "type" : "geo_shape" },
10+
"x" : {
11+
"type" : "text",
12+
"fields" : {
13+
"y" : {
14+
"type" : "foobar",
15+
"fields" : {
16+
"z" : {
17+
"properties" : {
18+
"v" : {
19+
"type" : "keyword"
20+
},
21+
"w" : {
22+
"type" : "foo"
23+
}
24+
}
25+
}
26+
}
27+
}
28+
}
29+
},
1030
"some" : {
1131
"properties" : {
1232
"dotted" : {

‎x-pack/plugin/ql/src/test/resources/mapping-unsupported.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
"time_frame" : {
77
"type" : "date_range",
88
"format" : "yyyy-MM-dd"
9+
},
10+
"flat" : {
11+
"type" : "flattened"
912
}
1013
}
1114
}

‎x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,14 @@ private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute na
217217
// unsupported types
218218
else if (DataTypes.isUnsupported(fa.dataType())) {
219219
UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
220-
named = u.withUnresolvedMessage(
221-
"Cannot use field [" + fa.name() + "] type [" + unsupportedField.getOriginalType() + "] as is unsupported");
220+
if (unsupportedField.hasInherited()) {
221+
named = u.withUnresolvedMessage(
222+
"Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "] "
223+
+ "in hierarchy (field [" + unsupportedField.getInherited() + "])");
224+
} else {
225+
named = u.withUnresolvedMessage(
226+
"Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]");
227+
}
222228
}
223229
// compound fields
224230
else if (allowCompound == false && fa.dataType().isPrimitive() == false) {

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -482,19 +482,33 @@ public void testGroupByScalarFunctionWithAggOnTarget() {
482482
}
483483

484484
public void testUnsupportedType() {
485-
assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
485+
assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
486486
error("SELECT unsupported FROM test"));
487487
}
488488

489489
public void testUnsupportedStarExpansion() {
490-
assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
490+
assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
491491
error("SELECT unsupported.* FROM test"));
492492
}
493493

494494
public void testUnsupportedTypeInFilter() {
495-
assertEquals("1:26: Cannot use field [unsupported] type [ip_range] as is unsupported",
495+
assertEquals("1:26: Cannot use field [unsupported] with unsupported type [ip_range]",
496496
error("SELECT * FROM test WHERE unsupported > 1"));
497497
}
498+
499+
public void testValidRootFieldWithUnsupportedChildren() {
500+
accept("SELECT x FROM test");
501+
}
502+
503+
public void testUnsupportedTypeInHierarchy() {
504+
assertEquals("1:8: Cannot use field [x.y.z.w] with unsupported type [foobar] in hierarchy (field [y])",
505+
error("SELECT x.y.z.w FROM test"));
506+
assertEquals("1:8: Cannot use field [x.y.z.v] with unsupported type [foobar] in hierarchy (field [y])",
507+
error("SELECT x.y.z.v FROM test"));
508+
assertEquals("1:8: Cannot use field [x.y.z] with unsupported type [foobar] in hierarchy (field [y])",
509+
error("SELECT x.y.z.* FROM test"));
510+
assertEquals("1:8: Cannot use field [x.y] with unsupported type [foobar]", error("SELECT x.y FROM test"));
511+
}
498512

499513
public void testTermEqualitOnInexact() {
500514
assertEquals("1:26: [text = 'value'] cannot operate on first argument field of data type [text]: " +
@@ -509,12 +523,12 @@ public void testTermEqualityOnAmbiguous() {
509523
}
510524

511525
public void testUnsupportedTypeInFunction() {
512-
assertEquals("1:12: Cannot use field [unsupported] type [ip_range] as is unsupported",
526+
assertEquals("1:12: Cannot use field [unsupported] with unsupported type [ip_range]",
513527
error("SELECT ABS(unsupported) FROM test"));
514528
}
515529

516530
public void testUnsupportedTypeInOrder() {
517-
assertEquals("1:29: Cannot use field [unsupported] type [ip_range] as is unsupported",
531+
assertEquals("1:29: Cannot use field [unsupported] with unsupported type [ip_range]",
518532
error("SELECT * FROM test ORDER BY unsupported"));
519533
}
520534

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,94 @@ public void testMetaFieldsAreIgnored() throws Exception {
128128
assertEquals(DataType.INTEGER, esIndex.mapping().get("_meta_field").getDataType());
129129
assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
130130
}
131+
132+
public void testFlattenedHiddenSubfield() throws Exception {
133+
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
134+
addFieldCaps(fieldCaps, "some_field", "flattened", false, false);
135+
addFieldCaps(fieldCaps, "some_field._keyed", "flattened", false, false);
136+
addFieldCaps(fieldCaps, "another_field", "object", true, false);
137+
addFieldCaps(fieldCaps, "another_field._keyed", "keyword", true, false);
138+
addFieldCaps(fieldCaps, "nested_field", "object", false, false);
139+
addFieldCaps(fieldCaps, "nested_field.sub_field", "flattened", true, true);
140+
addFieldCaps(fieldCaps, "nested_field.sub_field._keyed", "flattened", true, true);
141+
addFieldCaps(fieldCaps, "text", "keyword", true, true);
142+
143+
String wildcard = "*";
144+
IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
145+
assertTrue(resolution.isValid());
146+
147+
EsIndex esIndex = resolution.get();
148+
assertEquals(wildcard, esIndex.name());
149+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
150+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getProperties().get("_keyed").getDataType());
151+
assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
152+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field").getDataType());
153+
assertEquals(DataType.UNSUPPORTED,
154+
esIndex.mapping().get("nested_field").getProperties().get("sub_field").getProperties().get("_keyed").getDataType());
155+
assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
156+
assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
157+
assertEquals(DataType.KEYWORD, esIndex.mapping().get("another_field").getProperties().get("_keyed").getDataType());
158+
}
159+
160+
public void testPropagateUnsupportedTypeToSubFields() throws Exception {
161+
// generate a field type having the name of the format "foobar43"
162+
String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
163+
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
164+
addFieldCaps(fieldCaps, "a", "text", false, false);
165+
addFieldCaps(fieldCaps, "a.b", esFieldType, false, false);
166+
addFieldCaps(fieldCaps, "a.b.c", "object", true, false);
167+
addFieldCaps(fieldCaps, "a.b.c.d", "keyword", true, false);
168+
addFieldCaps(fieldCaps, "a.b.c.e", "foo", true, true);
169+
170+
String wildcard = "*";
171+
IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
172+
assertTrue(resolution.isValid());
173+
174+
EsIndex esIndex = resolution.get();
175+
assertEquals(wildcard, esIndex.name());
176+
assertEquals(DataType.TEXT, esIndex.mapping().get("a").getDataType());
177+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getDataType());
178+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c").getDataType());
179+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
180+
.getProperties().get("d").getDataType());
181+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
182+
.getProperties().get("e").getDataType());
183+
}
184+
185+
public void testRandomMappingFieldTypeMappedAsUnsupported() throws Exception {
186+
// generate a field type having the name of the format "foobar43"
187+
String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
188+
189+
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
190+
addFieldCaps(fieldCaps, "some_field", esFieldType, false, false);
191+
addFieldCaps(fieldCaps, "another_field", "object", true, false);
192+
addFieldCaps(fieldCaps, "another_field._foo", esFieldType, true, false);
193+
addFieldCaps(fieldCaps, "nested_field", "object", false, false);
194+
addFieldCaps(fieldCaps, "nested_field.sub_field1", esFieldType, true, true);
195+
addFieldCaps(fieldCaps, "nested_field.sub_field1.bar", esFieldType, true, true);
196+
addFieldCaps(fieldCaps, "nested_field.sub_field2", esFieldType, true, true);
197+
// even if this is of a supported type, because it belongs to an UNSUPPORTED type parent, it should also be UNSUPPORTED
198+
addFieldCaps(fieldCaps, "nested_field.sub_field2.bar", "keyword", true, true);
199+
addFieldCaps(fieldCaps, "text", "keyword", true, true);
200+
201+
String wildcard = "*";
202+
IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
203+
assertTrue(resolution.isValid());
204+
205+
EsIndex esIndex = resolution.get();
206+
assertEquals(wildcard, esIndex.name());
207+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
208+
assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
209+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getDataType());
210+
assertEquals(DataType.UNSUPPORTED,
211+
esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getProperties().get("bar").getDataType());
212+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getDataType());
213+
assertEquals(DataType.UNSUPPORTED,
214+
esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getProperties().get("bar").getDataType());
215+
assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
216+
assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
217+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("another_field").getProperties().get("_foo").getDataType());
218+
}
131219

132220
public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception {
133221
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
@@ -330,7 +418,7 @@ private static <K, V> void assertEqualsMaps(Map<K, V> left, Map<K, V> right) {
330418
private void addFieldCaps(Map<String, Map<String, FieldCapabilities>> fieldCaps, String name, String type, boolean isSearchable,
331419
boolean isAggregatable) {
332420
Map<String, FieldCapabilities> cap = new HashMap<>();
333-
cap.put(name, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap()));
421+
cap.put(type, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap()));
334422
fieldCaps.put(name, cap);
335423
}
336424
}

0 commit comments

Comments
 (0)