Skip to content

Commit 6c04815

Browse files
astefanastefan
authored
SQL: change the way unsupported data types fields are handled (#50823)
The hierarchy of fields/sub-fields under a field that is of an unsupported data type will be marked as unsupported as well. Until this change, the behavior was to set the unsupported data type field's hierarchy as empty. Example, considering the following hierarchy of fields/sub-fields a -> b -> c -> d, if b would be of type "foo", then b, c and d will be marked as unsupported. (cherry picked from commit 7adb286)
1 parent 71cb424 commit 6c04815

File tree

10 files changed

+209
-23
lines changed

10 files changed

+209
-23
lines changed

‎x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -218,8 +218,14 @@ private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute na
218218
// unsupported types
219219
else if (DataTypes.isUnsupported(fa.dataType())) {
220220
UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
221-
named = u.withUnresolvedMessage(
222-
"Cannot use field [" + fa.name() + "] type [" + unsupportedField.getOriginalType() + "] as is unsupported");
221+
if (unsupportedField.hasInherited()) {
222+
named = u.withUnresolvedMessage(
223+
"Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "] "
224+
+ "in hierarchy (field [" + unsupportedField.getInherited() + "])");
225+
} else {
226+
named = u.withUnresolvedMessage(
227+
"Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]");
228+
}
223229
}
224230
// compound fields
225231
else if (allowCompound == false && fa.dataType().isPrimitive() == false) {

‎x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -356,11 +356,12 @@ private static EsField createField(String fieldName, Map<String, Map<String, Fie
356356

357357
int dot = fieldName.lastIndexOf('.');
358358
String fullFieldName = fieldName;
359+
EsField parent = null;
359360

360361
if (dot >= 0) {
361362
String parentName = fieldName.substring(0, dot);
362363
fieldName = fieldName.substring(dot + 1);
363-
EsField parent = flattedMapping.get(parentName);
364+
parent = flattedMapping.get(parentName);
364365
if (parent == null) {
365366
Map<String, FieldCapabilities> map = globalCaps.get(parentName);
366367
Function<String, EsField> fieldFunction;
@@ -385,7 +386,22 @@ private static EsField createField(String fieldName, Map<String, Map<String, Fie
385386
}
386387

387388
EsField esField = field.apply(fieldName);
388-
389+
390+
if (parent != null && parent instanceof UnsupportedEsField) {
391+
UnsupportedEsField unsupportedParent = (UnsupportedEsField) parent;
392+
String inherited = unsupportedParent.getInherited();
393+
String type = unsupportedParent.getOriginalType();
394+
395+
if (inherited == null) {
396+
// mark the sub-field as unsupported, just like its parent, setting the first unsupported parent as the current one
397+
esField = new UnsupportedEsField(esField.getName(), type, unsupportedParent.getName(), esField.getProperties());
398+
} else {
399+
// mark the sub-field as unsupported, just like its parent, but setting the first unsupported parent
400+
// as the parent's first unsupported grandparent
401+
esField = new UnsupportedEsField(esField.getName(), type, inherited, esField.getProperties());
402+
}
403+
}
404+
389405
parentProps.put(fieldName, esField);
390406
flattedMapping.put(fullFieldName, esField);
391407

@@ -406,7 +422,7 @@ private static EsField createField(String fieldName, String typeName, Map<String
406422
case DATETIME:
407423
return new DateEsField(fieldName, props, isAggregateable);
408424
case UNSUPPORTED:
409-
return new UnsupportedEsField(fieldName, typeName);
425+
return new UnsupportedEsField(fieldName, typeName, null, props);
410426
default:
411427
return new EsField(fieldName, esType, props, isAggregateable, isAlias);
412428
}

‎x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/Types.java

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ private static void walkMapping(String name, Object value, Map<String, EsField>
7373
properties = Collections.emptyMap();
7474
}
7575
} else {
76-
properties = Collections.emptyMap();
76+
properties = fromEs(content);
7777
}
7878
boolean docValues = boolSetting(content.get("doc_values"), esDataType.defaultDocValues);
7979
final EsField field;
@@ -91,7 +91,8 @@ private static void walkMapping(String name, Object value, Map<String, EsField>
9191
break;
9292
case UNSUPPORTED:
9393
String type = content.get("type").toString();
94-
field = new UnsupportedEsField(name, type);
94+
field = new UnsupportedEsField(name, type, null, properties);
95+
propagateUnsupportedType(name, type, properties);
9596
break;
9697
default:
9798
field = new EsField(name, esDataType, properties, docValues);
@@ -113,4 +114,21 @@ private static boolean boolSetting(Object value, boolean defaultValue) {
113114
private static int intSetting(Object value, int defaultValue) {
114115
return value == null ? defaultValue : Integer.parseInt(value.toString());
115116
}
117+
118+
private static void propagateUnsupportedType(String inherited, String originalType, Map<String, EsField> properties) {
119+
if (properties != null && properties.isEmpty() == false) {
120+
for (Entry<String, EsField> entry : properties.entrySet()) {
121+
EsField field = entry.getValue();
122+
UnsupportedEsField u;
123+
if (field instanceof UnsupportedEsField) {
124+
u = (UnsupportedEsField) field;
125+
u = new UnsupportedEsField(u.getName(), originalType, inherited, u.getProperties());
126+
} else {
127+
u = new UnsupportedEsField(field.getName(), originalType, inherited, field.getProperties());
128+
}
129+
entry.setValue(u);
130+
propagateUnsupportedType(inherited, originalType, u.getProperties());
131+
}
132+
}
133+
}
116134
}

‎x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/UnsupportedEsField.java

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,24 +5,40 @@
55
*/
66
package org.elasticsearch.xpack.sql.type;
77

8-
import java.util.Collections;
8+
import java.util.Map;
99
import java.util.Objects;
10+
import java.util.TreeMap;
1011

1112
/**
12-
* SQL-related information about an index field that cannot be supported by SQL
13+
* SQL-related information about an index field that cannot be supported by SQL.
14+
* All the subfields (properties) of an unsupported type should also be unsupported.
1315
*/
1416
public class UnsupportedEsField extends EsField {
1517

16-
private String originalType;
17-
18+
private final String originalType;
19+
private final String inherited; // for fields belonging to parents (or grandparents) that have an unsupported type
20+
1821
public UnsupportedEsField(String name, String originalType) {
19-
super(name, DataType.UNSUPPORTED, Collections.emptyMap(), false);
22+
this(name, originalType, null, new TreeMap<>());
23+
}
24+
25+
public UnsupportedEsField(String name, String originalType, String inherited, Map<String, EsField> properties) {
26+
super(name, DataType.UNSUPPORTED, properties, false);
2027
this.originalType = originalType;
28+
this.inherited = inherited;
2129
}
2230

2331
public String getOriginalType() {
2432
return originalType;
2533
}
34+
35+
public String getInherited() {
36+
return inherited;
37+
}
38+
39+
public boolean hasInherited() {
40+
return inherited != null;
41+
}
2642

2743
@Override
2844
public boolean equals(Object o) {
@@ -36,11 +52,12 @@ public boolean equals(Object o) {
3652
return false;
3753
}
3854
UnsupportedEsField that = (UnsupportedEsField) o;
39-
return Objects.equals(originalType, that.originalType);
55+
return Objects.equals(originalType, that.originalType)
56+
&& Objects.equals(inherited, that.inherited);
4057
}
4158

4259
@Override
4360
public int hashCode() {
44-
return Objects.hash(super.hashCode(), originalType);
61+
return Objects.hash(super.hashCode(), originalType, inherited);
4562
}
4663
}

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -482,19 +482,33 @@ public void testGroupByScalarFunctionWithAggOnTarget() {
482482
}
483483

484484
public void testUnsupportedType() {
485-
assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
485+
assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
486486
error("SELECT unsupported FROM test"));
487487
}
488488

489489
public void testUnsupportedStarExpansion() {
490-
assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
490+
assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
491491
error("SELECT unsupported.* FROM test"));
492492
}
493493

494494
public void testUnsupportedTypeInFilter() {
495-
assertEquals("1:26: Cannot use field [unsupported] type [ip_range] as is unsupported",
495+
assertEquals("1:26: Cannot use field [unsupported] with unsupported type [ip_range]",
496496
error("SELECT * FROM test WHERE unsupported > 1"));
497497
}
498+
499+
public void testValidRootFieldWithUnsupportedChildren() {
500+
accept("SELECT x FROM test");
501+
}
502+
503+
public void testUnsupportedTypeInHierarchy() {
504+
assertEquals("1:8: Cannot use field [x.y.z.w] with unsupported type [foobar] in hierarchy (field [y])",
505+
error("SELECT x.y.z.w FROM test"));
506+
assertEquals("1:8: Cannot use field [x.y.z.v] with unsupported type [foobar] in hierarchy (field [y])",
507+
error("SELECT x.y.z.v FROM test"));
508+
assertEquals("1:8: Cannot use field [x.y.z] with unsupported type [foobar] in hierarchy (field [y])",
509+
error("SELECT x.y.z.* FROM test"));
510+
assertEquals("1:8: Cannot use field [x.y] with unsupported type [foobar]", error("SELECT x.y FROM test"));
511+
}
498512

499513
public void testTermEqualitOnInexact() {
500514
assertEquals("1:26: [text = 'value'] cannot operate on first argument field of data type [text]: " +
@@ -509,12 +523,12 @@ public void testTermEqualityOnAmbiguous() {
509523
}
510524

511525
public void testUnsupportedTypeInFunction() {
512-
assertEquals("1:12: Cannot use field [unsupported] type [ip_range] as is unsupported",
526+
assertEquals("1:12: Cannot use field [unsupported] with unsupported type [ip_range]",
513527
error("SELECT ABS(unsupported) FROM test"));
514528
}
515529

516530
public void testUnsupportedTypeInOrder() {
517-
assertEquals("1:29: Cannot use field [unsupported] type [ip_range] as is unsupported",
531+
assertEquals("1:29: Cannot use field [unsupported] with unsupported type [ip_range]",
518532
error("SELECT * FROM test ORDER BY unsupported"));
519533
}
520534

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,94 @@ public void testMetaFieldsAreIgnored() throws Exception {
124124
assertEquals(DataType.INTEGER, esIndex.mapping().get("_meta_field").getDataType());
125125
assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
126126
}
127+
128+
public void testFlattenedHiddenSubfield() throws Exception {
129+
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
130+
addFieldCaps(fieldCaps, "some_field", "flattened", false, false);
131+
addFieldCaps(fieldCaps, "some_field._keyed", "flattened", false, false);
132+
addFieldCaps(fieldCaps, "another_field", "object", true, false);
133+
addFieldCaps(fieldCaps, "another_field._keyed", "keyword", true, false);
134+
addFieldCaps(fieldCaps, "nested_field", "object", false, false);
135+
addFieldCaps(fieldCaps, "nested_field.sub_field", "flattened", true, true);
136+
addFieldCaps(fieldCaps, "nested_field.sub_field._keyed", "flattened", true, true);
137+
addFieldCaps(fieldCaps, "text", "keyword", true, true);
138+
139+
String wildcard = "*";
140+
IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
141+
assertTrue(resolution.isValid());
142+
143+
EsIndex esIndex = resolution.get();
144+
assertEquals(wildcard, esIndex.name());
145+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
146+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getProperties().get("_keyed").getDataType());
147+
assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
148+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field").getDataType());
149+
assertEquals(DataType.UNSUPPORTED,
150+
esIndex.mapping().get("nested_field").getProperties().get("sub_field").getProperties().get("_keyed").getDataType());
151+
assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
152+
assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
153+
assertEquals(DataType.KEYWORD, esIndex.mapping().get("another_field").getProperties().get("_keyed").getDataType());
154+
}
155+
156+
public void testPropagateUnsupportedTypeToSubFields() throws Exception {
157+
// generate a field type having the name of the format "foobar43"
158+
String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
159+
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
160+
addFieldCaps(fieldCaps, "a", "text", false, false);
161+
addFieldCaps(fieldCaps, "a.b", esFieldType, false, false);
162+
addFieldCaps(fieldCaps, "a.b.c", "object", true, false);
163+
addFieldCaps(fieldCaps, "a.b.c.d", "keyword", true, false);
164+
addFieldCaps(fieldCaps, "a.b.c.e", "foo", true, true);
165+
166+
String wildcard = "*";
167+
IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
168+
assertTrue(resolution.isValid());
169+
170+
EsIndex esIndex = resolution.get();
171+
assertEquals(wildcard, esIndex.name());
172+
assertEquals(DataType.TEXT, esIndex.mapping().get("a").getDataType());
173+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getDataType());
174+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c").getDataType());
175+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
176+
.getProperties().get("d").getDataType());
177+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
178+
.getProperties().get("e").getDataType());
179+
}
180+
181+
public void testRandomMappingFieldTypeMappedAsUnsupported() throws Exception {
182+
// generate a field type having the name of the format "foobar43"
183+
String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
184+
185+
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
186+
addFieldCaps(fieldCaps, "some_field", esFieldType, false, false);
187+
addFieldCaps(fieldCaps, "another_field", "object", true, false);
188+
addFieldCaps(fieldCaps, "another_field._foo", esFieldType, true, false);
189+
addFieldCaps(fieldCaps, "nested_field", "object", false, false);
190+
addFieldCaps(fieldCaps, "nested_field.sub_field1", esFieldType, true, true);
191+
addFieldCaps(fieldCaps, "nested_field.sub_field1.bar", esFieldType, true, true);
192+
addFieldCaps(fieldCaps, "nested_field.sub_field2", esFieldType, true, true);
193+
// even if this is of a supported type, because it belongs to an UNSUPPORTED type parent, it should also be UNSUPPORTED
194+
addFieldCaps(fieldCaps, "nested_field.sub_field2.bar", "keyword", true, true);
195+
addFieldCaps(fieldCaps, "text", "keyword", true, true);
196+
197+
String wildcard = "*";
198+
IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
199+
assertTrue(resolution.isValid());
200+
201+
EsIndex esIndex = resolution.get();
202+
assertEquals(wildcard, esIndex.name());
203+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
204+
assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
205+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getDataType());
206+
assertEquals(DataType.UNSUPPORTED,
207+
esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getProperties().get("bar").getDataType());
208+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getDataType());
209+
assertEquals(DataType.UNSUPPORTED,
210+
esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getProperties().get("bar").getDataType());
211+
assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
212+
assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
213+
assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("another_field").getProperties().get("_foo").getDataType());
214+
}
127215

128216
public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception {
129217
Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
@@ -323,7 +411,7 @@ private static <K, V> void assertEqualsMaps(Map<K, V> left, Map<K, V> right) {
323411
private void addFieldCaps(Map<String, Map<String, FieldCapabilities>> fieldCaps, String name, String type, boolean isSearchable,
324412
boolean isAggregatable) {
325413
Map<String, FieldCapabilities> cap = new HashMap<>();
326-
cap.put(name, new FieldCapabilities(name, type, isSearchable, isAggregatable));
414+
cap.put(type, new FieldCapabilities(name, type, isSearchable, isAggregatable));
327415
fieldCaps.put(name, cap);
328416
}
329417
}

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -463,7 +463,7 @@ public void testSysColumnsNoArg() throws Exception {
463463

464464
public void testSysColumnsWithCatalogWildcard() throws Exception {
465465
executeCommand("SYS COLUMNS CATALOG 'cluster' TABLE LIKE 'test' LIKE '%'", emptyList(), r -> {
466-
assertEquals(14, r.size());
466+
assertEquals(15, r.size());
467467
assertEquals(CLUSTER_NAME, r.column(0));
468468
assertEquals("test", r.column(2));
469469
assertEquals("bool", r.column(3));
@@ -476,7 +476,7 @@ public void testSysColumnsWithCatalogWildcard() throws Exception {
476476

477477
public void testSysColumnsWithMissingCatalog() throws Exception {
478478
executeCommand("SYS COLUMNS TABLE LIKE 'test' LIKE '%'", emptyList(), r -> {
479-
assertEquals(14, r.size());
479+
assertEquals(15, r.size());
480480
assertEquals(CLUSTER_NAME, r.column(0));
481481
assertEquals("test", r.column(2));
482482
assertEquals("bool", r.column(3));
@@ -489,7 +489,7 @@ public void testSysColumnsWithMissingCatalog() throws Exception {
489489

490490
public void testSysColumnsWithNullCatalog() throws Exception {
491491
executeCommand("SYS COLUMNS CATALOG ? TABLE LIKE 'test' LIKE '%'", singletonList(new SqlTypedParamValue("keyword", null)), r -> {
492-
assertEquals(14, r.size());
492+
assertEquals(15, r.size());
493493
assertEquals(CLUSTER_NAME, r.column(0));
494494
assertEquals("test", r.column(2));
495495
assertEquals("bool", r.column(3));

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,10 @@ public void testUnsupportedTypes() {
188188
Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
189189
EsField dt = mapping.get("range");
190190
assertThat(dt.getDataType().typeName, is("unsupported"));
191+
dt = mapping.get("time_frame");
192+
assertThat(dt.getDataType().typeName, is("unsupported"));
193+
dt = mapping.get("flat");
194+
assertThat(dt.getDataType().typeName, is("unsupported"));
191195
}
192196

193197
public static Map<String, EsField> loadMapping(String name) {

0 commit comments

Comments
 (0)